End DCS/OSC strings at any ESC, default the external terminal to Terminal.app

A string with no ST swallowed every later sequence, RIS included, and
Clear Screen could not recover the tab. Any ESC now ends it, as in xterm,
and the sequence it starts is dispatched. The shipped setting needed Kitty.

Also: REP, SCOSC/SCORC, DECOM-relative CPR, `:` subparams and more DECRQM
modes answered; new rows inserted in one edit; controls stripped from
pastes; wide-char mouse columns.

tests/test_commands.py

@@ -4,7 +4,7 @@ import _bootstrap  # sets sys.path for the imports below
 from _bootstrap import FakeProcess
 
 import sublime
-from vterm.commands import VtermPasteSelectionCommand, VtermSelectThemeCommand, selection_text
+from vterm.commands import VtermPasteSelectionCommand, VtermPasteTextCommand, VtermSelectThemeCommand, selection_text
 from vterm.emulator import VtScreen
 from vterm.render import CONTINUATION
 from vterm.scheme import get_scheme_manager
@@ -245,3 +245,17 @@ view = FakeView("abcdef", [sublime.Region(2, 2)])
 assert paste(view) == [], view.commands
 
 print("selection tests passed (sentinel stripping, paste, empty no-op)")
+
+
+# a paste is a trust boundary: C0 controls bar tab/LF/CR go, so a pasted
+# ESC[201~ cannot close the bracket and type the rest into the app
+terminal = fake_terminal("default")
+terminal.process = terminal.screen._process
+terminal.screen.set_mode(2004, private=True)
+terminal.view.run_command = lambda name, args=None: None
+command = VtermPasteTextCommand.__new__(VtermPasteTextCommand)
+command.view = terminal.view
+command.run(None, "a\x1b[201~b\x00\tc\nd")
+assert terminal.process.written == "\x1b[200~a[201~b\tc\rd\x1b[201~", repr(terminal.process.written)
+
+print("paste tests passed (controls stripped, bracket kept whole)")

tests/test_emulator.py

@@ -454,8 +454,67 @@ def run_clear_cases():
     print("all clear-screen tests passed")
 
 
+def run_parser_edge_cases():
+    """Sequences the stock collector misread: intermediates, `:` subparams, REP, marks."""
+    for char_by_char in (False, True):
+        tag = "char-by-char" if char_by_char else "chunked"
+
+        # DECSTR (`reset` sends it): `!` is an intermediate, the `p` must not be drawn
+        proc, screen, stream = make(20, 4)
+        feed(stream, "\x1b[!pX", char_by_char)
+        assert screen.display[0].rstrip() == "X", (tag, repr(screen.display[0]))
+
+        # `4:3` is curly underline, not SGR 43 (yellow background)
+        proc, screen, stream = make(20, 4)
+        feed(stream, "\x1b[4:3mX\x1b[38:2:1:2:3mY", char_by_char)
+        cell = screen.buffer[0][0]
+        assert (cell.bg, cell.underscore) == ("default", True), (tag, cell)
+        assert screen.buffer[0][1].data == "Y", (tag, screen.buffer[0][1])
+
+        # REP repeats the last printable; ncurses emits it for runs
+        proc, screen, stream = make(20, 4)
+        feed(stream, "a\x1b[3b\x1b[31mb\x1b[b", char_by_char)
+        assert screen.display[0].rstrip() == "aaaabb", (tag, repr(screen.display[0]))
+        assert screen.buffer[0][5].fg == "red", (tag, screen.buffer[0][5])
+
+        # a Thai vowel sign has combining class 0 but is still a mark: merged, not dropped
+        proc, screen, stream = make(20, 4)
+        feed(stream, "\u0e01\u0e31x", char_by_char)
+        assert screen.buffer[0][0].data == "\u0e01\u0e31", (tag, screen.buffer[0][0])
+        assert screen.buffer[0][1].data == "x", (tag, screen.buffer[0][1])
+
+        # DECRQM: the modes we honour answer set/reset, never "unrecognized"
+        proc, screen, stream = make()
+        feed(stream, "\x1b[?2031$p\x1b[?1h\x1b[?1$p\x1b[4$p\x1b[20$p", char_by_char)
+        assert proc.written == "\x1b[?2031;2$y\x1b[?1;1$y\x1b[4;2$y\x1b[20;2$y", (tag, proc.written)
+
+        # RIS on the alternate screen lands back on a cleared primary screen
+        proc, screen, stream = make(20, 4)
+        feed(stream, "primary\x1b[?1049halt\x1bc", char_by_char)
+        assert not screen.alternate_buffer_mode and not screen.savepoints, tag
+        assert screen.display[0].strip() == "", (tag, repr(screen.display[0]))
+        feed(stream, "\r\n" * 4, char_by_char)
+        assert len(screen.history) == 1, (tag, len(screen.history))
+
+        # SCOSC/SCORC, and CPR relative to the region under DECOM
+        proc, screen, stream = make(20, 4)
+        feed(stream, "ab\x1b[s\x1b[3;3H\x1b[uX\x1b[2;3r\x1b[?6h\x1b[6n", char_by_char)
+        assert screen.display[0].rstrip() == "abX", (tag, repr(screen.display[0]))
+        assert proc.written == "\x1b[1;1R", (tag, proc.written)
+
+        # a string with no terminator ends at the next ESC, as in xterm; a stray
+        # ESC P from a binary must not swallow the rest of the session
+        proc, screen, stream = make(20, 4)
+        feed(stream, "\x1bPgarbage\x1bcX\x1b]0;title\x1b[1mY", char_by_char)
+        assert screen.display[0].rstrip() == "XY", (tag, repr(screen.display[0]))
+        assert screen.title == "title" and screen.buffer[0][1].bold, (tag, screen.title)
+
+    print("all parser edge-case tests passed")
+
+
 run_case(char_by_char=False)
 run_case(char_by_char=True)
+run_parser_edge_cases()
 run_clear_cases()
 run_resize_cases()
 run_private_csi_cases()

tests/test_render.py

@@ -357,7 +357,13 @@ command = make_command(terminal)
 stream.feed("".join("\x1b[31mr{}\x1b[0m\r\n".format(i) for i in range(300)))
 command.update_lines(None, terminal)
 terminal.screen.dirty.clear()
-assert set(command.state.archived_chunks) == {0}, command.state.archived_chunks
+# rows new to the view went in as one insert, coloured straight onto their
+# chunk keys; only row 0 and the two coloured screen rows have a key of their own
+assert set(command.state.archived_chunks) == {0, 1}, command.state.archived_chunks
+assert len(terminal.view.regions["vterm@1@vterm.red.default"][1]) == terminal.offset - CHUNK
+row_keys = [k for k in terminal.view.regions if k.startswith("vterm#")]
+assert len(row_keys) == 3, row_keys
+assert terminal.view.text.split("\n")[:3] == ["r0", "r1", "r2"], terminal.view.text[:20]
 command.trim_history(None, terminal)
 assert "vterm@0@vterm.red.default" not in terminal.view.regions, "trimmed chunk key left behind"
 assert 0 not in command.state.archived_chunks
@@ -665,7 +671,15 @@ terminal.view.window_to_text = lambda xy: terminal.view.text_point(2, 4)
 mouse_release.run(None, event)
 assert terminal.process.written == "\x1b[<0;1;1M\x1b[<32;5;3M\x1b[<0;5;3m", repr(terminal.process.written)
 
-print("click tests passed (first click focuses only; press, drag and release report)")
+# the column is counted in cells: a wide char before the point takes two
+terminal.process.writes.clear()
+terminal.view.text = "\u65e5\u672cx   \nr1    \nr2    \nr3    \nr4    \n"
+terminal.view.window_to_text = lambda xy: terminal.view.text_point(0, 2)
+mouse_press.run(None, event)
+mouse_release.run(None, event)
+assert terminal.process.written == "\x1b[<0;5;1M\x1b[<0;5;1m", repr(terminal.process.written)
+
+print("click tests passed (first click focuses only; press, drag and release report; wide chars)")
 
 
 # entering the alt screen folds the scrollback away (kitty hides it); leaving
@@ -751,4 +765,12 @@ line[0] = Char("r", "red", "default", False, False, False, False, True)
 runs = [r for r in segment_buffer_line(line) if r[1] != r[2]]
 assert runs == [("r", 0, 1, "reverse_default", "red", "")], runs
 
-print("segmenter tests passed (leading gap, wide char, orphan filler, reverse)")
+# a cell can hold two code points (a mark NFC could not compose): the spans
+# that follow are offsets into the row text, so they move by two
+line = StaticDefaultDict(Char(" "))
+line[0] = Char("q\u0301", "default", "default")
+line[1] = Char("x", "blue", "default")
+runs = [r for r in segment_buffer_line(line) if r[1] != r[2]]
+assert runs == [("q\u0301", 0, 2, "default", "default", ""), ("x", 2, 3, "blue", "default", "")], runs
+
+print("segmenter tests passed (leading gap, wide char, orphan filler, reverse, two-code-point cell)")