Revert some changes to make branch surface smaller.

It was that way before, it does not break things,
and people could rely on that.

renpy/__init__.py

@@ -421,6 +421,7 @@ def import_all():
     import renpy.easy
     import renpy.encryption
     import renpy.execution
+    import renpy.tokenizer
     import renpy.lexer
     import renpy.loadsave
     import renpy.savelocation
@@ -742,6 +743,7 @@ if typing.TYPE_CHECKING:
     from . import substitutions as substitutions
     from . import test as test
     from . import text as text
+    from . import tokenizer as tokenizer
     from . import translation as translation
     from . import uguu as uguu
     from . import ui as ui

renpy/add_from.py

@@ -19,26 +19,23 @@
 # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
-from __future__ import division, absolute_import, with_statement, print_function, unicode_literals
-from renpy.compat import PY2, basestring, bchr, bord, chr, open, pystr, range, round, str, tobytes, unicode # *
 
-
-
-import collections
+from collections import defaultdict
+from pathlib import Path
 import renpy
 import os
 
 # A map from filename to position, target label pairs.
-missing = collections.defaultdict(list)
+missing = defaultdict[str, list[tuple[int, str]]](list)
 
 
-def report_missing(target, filename, position):
+def report_missing(target: str, filename: str, number: int):
     """
     Reports that the call statement ending at `position` in `filename`
     is missing a from clause.
     """
 
-    missing[filename].append((position, target))
+    missing[filename].append((number, target))
 
 
 # Labels that we've created while running add_from.
@@ -69,44 +66,43 @@ def generate_label(target):
     return label
 
 
-def process_file(fn):
+def process_file(fn: str, full_fn: str):
     """
     Adds missing from clauses to `fn`.
     """
 
-    if not os.path.exists(fn):
+    path = Path(full_fn)
+    if not path.exists():
+        return
+
+    if (lines := renpy.scriptedit.ensure_loaded(full_fn)) is None:
         return
 
     edits = missing[fn]
     edits.sort()
+    edits = iter(edits)
+    next_edit, target = next(edits)
 
-    with open(fn, "rb") as f:
-        data = f.read().decode("utf-8")
+    new_lines = []
+    for lineno, line in lines.items():
+        if lineno == next_edit:
+            code = f" from {generate_label(target)}"
+            full_text = line.with_added_code(code)
+        else:
+            full_text = line.full_text
 
-    # How much of the input has been consumed.
-    consumed = 0
+        new_lines.append(full_text)
 
-    # The output.
-    output = u""
+    new_path = path.with_suffix(f"{path.suffix}.new")
+    with new_path.open("w", encoding="utf-8") as f:
+        f.write("\ufeff")
+        f.writelines(new_lines)
 
-    for position, target in edits:
-        output += data[consumed:position]
-        consumed = position
+    bak_path = path.with_suffix(f"{path.suffix}.bak")
+    bak_path.unlink(missing_ok=True)
 
-        output += " from {}".format(generate_label(target))
-
-    output += data[consumed:]
-
-    with open(fn + ".new", "wb") as f:
-        f.write(output.encode("utf-8"))
-
-    try:
-        os.unlink(fn + ".bak")
-    except Exception:
-        pass
-
-    os.rename(fn, fn + ".bak")
-    os.rename(fn + ".new", fn)
+    path.rename(bak_path)
+    new_path.rename(path)
 
 
 def add_from():
@@ -114,8 +110,9 @@ def add_from():
     renpy.arguments.takes_no_arguments("Adds from clauses to call statements that are missing them.")
 
     for fn in missing:
-        if fn.startswith(renpy.config.gamedir):
-            process_file(fn)
+        full_fn = renpy.parser.unelide_filename(fn)
+        if full_fn.startswith(renpy.config.gamedir):
+            process_file(fn, full_fn)
 
     return False
 

renpy/common/_errorhandling.rpym

@@ -527,7 +527,7 @@ init python:
             rv.append(re.sub(
                 r'(File "(.*)", line (\d+))',
                 r'{a=edit:\3:\2}\1{/a}',
-                lines.pop(0)))
+                lines.pop(0).replace("{", "{{")))
 
             error_rv = []
             error_len = 0
@@ -544,7 +544,7 @@ init python:
 
                     prev_line = error_rv[-1]
 
-                    line = prev_line[:idx]
+                    line = prev_line[:idx].replace("{", "{{")
 
                     if count == 1:
                         if len(prev_line) < idx + count:
@@ -557,7 +557,7 @@ init python:
                     line += "{b}{color=#a00}"
                     line += mark
                     line += "{/color}{/b}"
-                    line += prev_line[idx + count:]
+                    line += prev_line[idx + count:].replace("{", "{{")
 
                     error_rv[-1] = line
                     continue

renpy/parser.py

@@ -22,9 +22,6 @@
 # This module contains the parser for the Ren'Py script language. It's
 # called when parsing is necessary, and creates an AST from the script.
 
-from __future__ import division, absolute_import, with_statement, print_function, unicode_literals # type: ignore
-from renpy.compat import PY2, basestring, bchr, bord, chr, open, pystr, range, round, str, tobytes, unicode # *
-
 import collections
 import time
 
@@ -34,8 +31,6 @@ import renpy.ast as ast
 from renpy.parameter import EMPTY_ARGUMENTS, Parameter
 
 from renpy.lexer import (
-    list_logical_lines,
-    group_logical_lines,
     ParseError,
     Lexer,
 
@@ -290,7 +285,7 @@ def parse_menu(stmtl, loc, arguments):
         # A string on a line by itself is a caption.
         if l.eol():
 
-            if l.subblock:
+            if l.has_block():
                 l.error("Line is followed by a block, despite not being a menu choice. Did you forget a colon at the end of the line?")
 
             if label and say_ast:
@@ -783,11 +778,14 @@ def call_statement(l, loc):
         name = l.require(l.label_name_declare)
         rv.append(ast.Label(loc, name, [], None))
     else:
-        if renpy.scriptedit.lines and (loc in renpy.scriptedit.lines):
+        if line := renpy.scriptedit.lines.get(l.filename, {}).get(l.number):
             if expression:
-                renpy.add_from.report_missing("expression", renpy.lexer.original_filename, renpy.scriptedit.lines[loc].end)
-            else:
-                renpy.add_from.report_missing(target, renpy.lexer.original_filename, renpy.scriptedit.lines[loc].end)
+                target = "expression"
+
+            renpy.add_from.report_missing(
+                target,
+                line.filename,
+                line.number)
 
     rv.append(ast.Pass(loc))
 
@@ -1669,7 +1667,7 @@ def parse_block(l):
     return rv
 
 
-def parse(fn, filedata=None, linenumber=1):
+def parse(fn: str, filedata: str | None = None, linenumber: int = 1) -> list[ast.Node] | None:
     """
     Parses a Ren'Py script contained within the file `fn`.
 
@@ -1685,13 +1683,19 @@ def parse(fn, filedata=None, linenumber=1):
     renpy.game.exception_info = 'While parsing ' + fn + '.'
 
     try:
-        lines = list_logical_lines(fn, filedata, linenumber)
-        nested = group_logical_lines(lines)
+        if filedata is None:
+            tok = renpy.tokenizer.from_file(fn)
+        else:
+            tok = renpy.tokenizer.from_string(
+                filedata, fn, lineno_offset=linenumber - 1)
+
+        lines = list(tok.logical_lines())
+
     except ParseError as e:
         parse_errors.append(e.message)
         return None
 
-    l = Lexer(nested)
+    l = Lexer(lines)
 
     rv = parse_block(l)
 

renpy/scriptedit.py

@@ -22,237 +22,323 @@
 # This file contains code to add and remove statements from the AST
 # and the textual representation of Ren'Py code.
 
-from __future__ import division, absolute_import, with_statement, print_function, unicode_literals
-from renpy.compat import PY2, basestring, bchr, bord, chr, open, pystr, range, round, str, tobytes, unicode # *
-
-
 
 import renpy
 import re
-import codecs
 
-# A map from line loc (elided filename, line) to the Line object representing
-# that line.
-lines = { }
+from dataclasses import dataclass
+from pathlib import Path
 
-# The set of files that have been loaded.
-files = set()
+import renpy.tokenizer
 
-
-class Line(object):
+@dataclass(eq=False)
+class Line:
     """
-    Represents a logical line in a file.
+    Represent a logical line of Ren'Py code.
+    In contrast to renpy.tokenizer.Line, this class represents
+    lines exactly as they appear in the file, including comments.
+    Also, it includes
     """
 
-    def __init__(self, filename, number, start):
+    # The full path to the file with the line in it.
+    filename: str
 
-        filename = filename.replace("\\", "/")
+    # The line number where logical line starts.
+    number: int
 
-        # The full path to the file with the line in it.
-        self.filename = filename
+    # The lines of the logical line.
+    lines: tuple[str, ...]
 
-        # The line number.
-        self.number = number
+    # The comments on the lines.
+    comments: tuple[str, ...]
 
-        # The offset inside the file at which the line starts.
-        self.start = start
+    _text: str | None = None
+    _full_text: str | None = None
 
-        # The offset inside the file at which the line ends.
-        self.end = start
+    @property
+    def blank(self):
+        return self.lines == ["\n"]
 
-        # The offset inside the lime where the line delimiter ends.
-        self.end_delim = start
+    @property
+    def text(self):
+        if self._text is None:
+            self._text = "".join(self.lines)
 
-        # The text of the line.
-        self.text = ''
+        return self._text
 
-        # The full text, including any comments or delimiters.
-        self.full_text = ''
+    @property
+    def full_text(self):
+        if self._full_text is not None:
+            return self._full_text
+
+        def join():
+            for i, line in enumerate(self.lines):
+                comment = self.comments[i]
+                if comment:
+                    yield f"{line[:-1]}{comment}\n"
+                else:
+                    yield line
+
+        self._full_text = "".join(join())
+        return self._full_text
+
+    def with_added_code(self, code: str) -> str:
+        """
+        Return a full text with a code added to the end before comment.
+        """
+
+        def join():
+            last_i = len(self.lines) - 1
+            for i, line in enumerate(self.lines):
+                if i == last_i:
+                    line = f"{line[:-1]}{code}\n"
+
+                comment = self.comments[i]
+                if comment:
+                    yield f"{line[:-1]}{comment}\n"
+                else:
+                    yield line
+
+        return "".join(join())
 
 
-    def __repr__(self):
-        return "<Line {}:{} {!r}>".format(self.filename, self.number, self.text)
+
+# A map from elided filename to dict of line number to Line objects
+# representing that file.
+lines = dict[str, dict[int, Line]]()
 
 
-def ensure_loaded(filename):
+def ensure_loaded(filename: str, reload: bool = False ) -> dict[int, Line] | None:
     """
-    Ensures that the given filename and linenumber are loaded. Doesn't do
-    anything if the filename can't be loaded.
+    Ensures that the given filename is loaded.
+    Doesn't do anything if the filename can't be loaded or is empty.
+    If `reload` is true, the file will be reloaded even if it's already
+    loaded.
     """
 
-    if not (filename.endswith(".rpy") or filename.endswith(".rpyc")):
-        return
+    filename = renpy.parser.unelide_filename(filename)
+    path = Path(filename)
 
-    if filename in files:
-        return
+    if not (path.exists() and (
+        path.name.endswith("_ren.py") or
+        path.name.endswith(".rpy") or
+        path.name.endswith(".rpym")
+    )):
+        return None
 
-    files.add(filename)
+    filename = renpy.parser.elide_filename(path.as_posix())
 
-    fn = renpy.lexer.unelide_filename(filename)
-    renpy.lexer.list_logical_lines(fn, add_lines=True)
+    if reload:
+        lines.pop(filename, None)
+    elif filename in lines:
+        return lines[filename]
+
+    lines[filename] = dict()
+
+    def linesfunc():
+        tok = renpy.tokenizer.from_file(filename)
+
+        yield from tok.physical_lines()
+
+        while True:
+            yield ""
+
+    file_lines = linesfunc()
+    current_line = next(file_lines)
+    last_lineno = 1
+    start_lineno = 1
+    seen_token = False
+
+    tokens = renpy.tokenizer.from_file(filename).tokens()
+    text_lines: list[str] = []
+    comments: list[str] = []
+    for token in tokens:
+        # Fast-path for comment-only and empty lines.
+        while not seen_token:
+            comment = ""
+            if token.kind == "comment":
+                comment = token.string
+                start = token.physical_location.start_col_offset
+                end = token.physical_location.end_col_offset
+                current_line = f"{current_line[:start]}{current_line[end:]}"
+                # Next token is always NL.
+                token = next(tokens)
+
+            if token.kind == "nl":
+                lines[filename][last_lineno] = Line(
+                    filename,
+                    last_lineno,
+                    (current_line, ),
+                    (comment, ))
+
+                current_line = next(file_lines)
+                last_lineno += 1
+                token = next(tokens)
+                continue
+
+            # New logical line just started.
+            seen_token = True
+            start_lineno = last_lineno
+            break
+
+        # Consume lines between the last line and the start of the token.
+        # It happens only for multiline strings, so no comments here.
+        while last_lineno < token.physical_location.start_lineno:
+            text_lines.append(current_line)
+            comments.append("")
+            current_line = next(file_lines)
+            last_lineno += 1
+
+        if token.kind == "comment":
+            comment = token.string
+            start = token.physical_location.start_col_offset
+            end = token.physical_location.end_col_offset
+            current_line = f"{current_line[:start]}{current_line[end:]}"
+
+            # Should be NL or NEWLINE.
+            token = next(tokens)
+        else:
+            comment = ""
+
+        if token.kind == "nl":
+            text_lines.append(current_line)
+            comments.append(comment)
+            current_line = next(file_lines)
+            last_lineno += 1
+
+        elif token.kind == "newline":
+            text_lines.append(current_line)
+            comments.append(comment)
+
+            lines[filename][start_lineno] = Line(
+                filename,
+                start_lineno,
+                tuple(text_lines),
+                tuple(comments))
+
+            text_lines.clear()
+            comments.clear()
+            current_line = next(file_lines)
+            last_lineno += 1
+            seen_token = False
+
+    if not lines[filename]:
+        del lines[filename]
+        return None
+
+    return lines[filename]
 
 
-def get_line_text(filename, linenumber):
+def get_line_text(filename: str, linenumber: int):
     """
     Gets the text of the line with `filename` and `linenumber`, or the None if
     the line does not exist.
     """
 
-    filename = filename.replace("\\", "/")
+    lines = ensure_loaded(filename)
+    if lines is None:
+        return None
 
-    ensure_loaded(filename)
-
-    if (filename, linenumber) in lines:
-        return lines[filename, linenumber].text
-    else:
+    try:
+        return lines[linenumber].text
+    except KeyError:
         return None
 
 
-def adjust_line_locations(filename, linenumber, char_offset, line_offset):
+def get_full_text(filename: str, linenumber: int):
     """
-    Adjusts the locations in the line data structure.
-
-    `filename`, `linenumber`
-        The filename and first line number to adjust.
-
-    `char_offset`
-        The number of characters in the file to offset the code by,.
-
-    `line_offset`
-        The number of line in the file to offset the code by.
+    Returns the full text of `linenumber` from `filename`, including
+    any comment or delimiter characters that exist.
     """
 
-    filename = filename.replace("\\", "/")
+    lines = ensure_loaded(filename)
+    if lines is None:
+        return None
 
-    ensure_loaded(filename)
-
-    global lines
-
-    new_lines = { }
-
-    for key, line in lines.items():
-
-        (fn, ln) = key
-
-        if (fn == filename) and (linenumber <= ln):
-            ln += line_offset
-            line.number += line_offset
-            line.start += char_offset
-            line.end += char_offset
-            line.end_delim += char_offset
-
-        new_lines[fn, ln] = line
-
-    lines = new_lines
+    try:
+        return lines[linenumber].full_text
+    except KeyError:
+        return None
 
 
-def insert_line_before(code, filename, linenumber):
+def insert_line_before(code: str, filename: str, linenumber: int):
     """
     Adds `code` immediately before `filename` and `linenumber`. Those must
     correspond to an existing line, and the code is inserted with the same
     indentation as that line.
     """
 
-    filename = filename.replace("\\", "/")
-
     if renpy.config.clear_lines:
         raise Exception("config.clear_lines must be False for script editing to work.")
 
-    ensure_loaded(filename)
+    lines = ensure_loaded(filename)
+    if lines is None:
+        return
 
-    old_line = lines[filename, linenumber]
+    old_line = lines[linenumber]
 
-    m = re.match(r' *', old_line.text)
-    indent = m.group(0)
-
-    if not code:
+    if code:
+        indent = re.match(r' *', old_line.text).group(0)
+    else:
         indent = ''
 
-    if old_line.text.endswith("\r\n") or not old_line.text.endswith("\n"):
-        line_ending = "\r\n"
-    else:
-        line_ending = "\n"
+    line_ending = "\n" if code.endswith("\n") else "\n"
+    code = f"{indent}{code}{line_ending}"
 
-    raw_code = indent + code
-    code = indent + code + line_ending
+    path = Path(renpy.parser.unelide_filename(old_line.filename))
+    with path.open("w", encoding="utf-8") as f, renpy.loader.auto_lock:
+        f.write("\ufeff")
 
-    new_line = Line(old_line.filename, old_line.number, old_line.start)
-    new_line.text = raw_code
-    new_line.full_text = code
-    new_line.end = new_line.start + len(raw_code)
-    new_line.end_delim = new_line.start + len(code)
+        for lineno, line in lines.items():
+            if lineno == linenumber:
+                f.write(code)
 
-    with codecs.open(old_line.filename, "r", "utf-8") as f:
-        data = f.read()
-
-    data = data[:old_line.start] + code + data[old_line.start:]
-
-    adjust_line_locations(filename, linenumber, len(code), code.count("\n"))
-
-    with renpy.loader.auto_lock:
-
-        with codecs.open(old_line.filename, "w", "utf-8") as f:
-            f.write(data)
+            f.write(line.full_text)
 
         renpy.loader.add_auto(old_line.filename, force=True)
 
-    lines[filename, linenumber] = new_line
+    ensure_loaded(old_line.filename, reload=True)
 
 
-def remove_line(filename, linenumber):
+def remove_line(filename: str, linenumber: int):
     """
     Removes `linenumber` from `filename`. The line must exist and correspond
     to a logical line.
     """
 
-    filename = filename.replace("\\", "/")
-
     if renpy.config.clear_lines:
         raise Exception("config.clear_lines must be False for script editing to work.")
 
-    ensure_loaded(filename)
+    lines = ensure_loaded(filename)
+    if lines is None:
+        return
 
-    line = lines[filename, linenumber]
+    old_line = lines[linenumber]
 
-    with codecs.open(line.filename, "r", "utf-8") as f:
-        data = f.read()
+    path = Path(renpy.parser.unelide_filename(old_line.filename))
+    with path.open("w", encoding="utf-8") as f, renpy.loader.auto_lock:
+        f.write("\ufeff")
 
-    code = data[line.start:line.end_delim]
-    data = data[:line.start] + data[line.end_delim:]
+        for lineno, line in lines.items():
+            if lineno == linenumber:
+                continue
 
-    del lines[filename, linenumber]
-    adjust_line_locations(filename, linenumber, -len(code), -code.count("\n"))
+            f.write(line.full_text)
 
-    with renpy.loader.auto_lock:
+        renpy.loader.add_auto(old_line.filename, force=True)
 
-        with codecs.open(line.filename, "w", "utf-8") as f:
-            f.write(data)
-
-        renpy.loader.add_auto(line.filename, force=True)
+    ensure_loaded(old_line.filename, reload=True)
 
 
-def get_full_text(filename, linenumber):
-    """
-    Returns the full text of `linenumber` from `filename`, including
-    any comment or delimiter characters that exist.
-    """
-
-    filename = filename.replace("\\", "/")
-
-    ensure_loaded(filename)
-
-    if (filename, linenumber) not in lines:
-        return None
-
-    return lines[filename, linenumber].full_text
-
-
-def nodes_on_line(filename, linenumber):
+def nodes_on_line(filename: str, linenumber: int) -> list[renpy.ast.Node]:
     """
     Returns a list of nodes that are found on the given line.
     """
 
-    ensure_loaded(filename)
+    if (lines := ensure_loaded(filename)) is None:
+        return [ ]
+
+    filename = next(iter(lines.values())).filename
 
     rv = [ ]
 
@@ -263,12 +349,15 @@ def nodes_on_line(filename, linenumber):
     return rv
 
 
-def nodes_on_line_at_or_after(filename, linenumber):
+def nodes_on_line_at_or_after(filename: str, linenumber: int) -> list[renpy.ast.Node]:
     """
     Returns a list of nodes that are found at or after the given line.
     """
 
-    ensure_loaded(filename)
+    if (lines := ensure_loaded(filename)) is None:
+        return [ ]
+
+    filename = next(iter(lines.values())).filename
 
     lines = [ i.linenumber
               for i in renpy.game.script.all_stmts
@@ -282,15 +371,15 @@ def nodes_on_line_at_or_after(filename, linenumber):
     return nodes_on_line(filename, min(lines))
 
 
-def first_and_last_nodes(nodes):
+def first_and_last_nodes(nodes: list[renpy.ast.Node]):
     """
     Finds the first and last nodes in `nodes`, a list of nodes. This assumes
     that all the nodes are "simple", with no control flow, and that all of
     the relevant nodes are in `nodes`.
     """
 
-    firsts = [ ]
-    lasts = [ ]
+    firsts: list[renpy.ast.Node] = [ ]
+    lasts: list[renpy.ast.Node] = [ ]
 
     for i in nodes:
         for j in nodes:
@@ -315,7 +404,7 @@ def first_and_last_nodes(nodes):
     return firsts[0], lasts[0]
 
 
-def adjust_ast_linenumbers(filename, linenumber, offset):
+def adjust_ast_linenumbers(filename: str, linenumber: int, offset: int):
     """
     This adjusts the line numbers in the ast.
 
@@ -334,7 +423,7 @@ def adjust_ast_linenumbers(filename, linenumber, offset):
             i.linenumber += offset
 
 
-def add_to_ast_before(code, filename, linenumber):
+def add_to_ast_before(code: str, filename: str, linenumber: int):
     """
     Adds `code`, which must be a textual line of Ren'Py code,
     before the given filename and line number.
@@ -345,7 +434,8 @@ def add_to_ast_before(code, filename, linenumber):
 
     adjust_ast_linenumbers(old.filename, linenumber, 1)
 
-    block, _init = renpy.game.script.load_string(old.filename, code, linenumber=linenumber)
+    block, _init = renpy.game.script.load_string(
+        old.filename, code, linenumber=linenumber)
 
     # Remove the return statement at the end of the block.
     ret_stmt = block.pop()
@@ -365,7 +455,7 @@ def add_to_ast_before(code, filename, linenumber):
     renpy.game.log.replace_node(old, block[0])
 
 
-def can_add_before(filename, linenumber):
+def can_add_before(filename: str, linenumber: int):
     """
     Returns True if it's possible to add a line before the given filename
     and linenumber, and False if it's not possible.
@@ -380,7 +470,7 @@ def can_add_before(filename, linenumber):
         return False
 
 
-def remove_from_ast(filename, linenumber):
+def remove_from_ast(filename: str, linenumber: int):
     """
     Removes from the AST all statements that happen to be at `filename`
     and `linenumber`, then adjusts the line numbers appropriately.

renpy/statements.py

@@ -21,11 +21,6 @@
 
 # This module contains code to support user-defined statements.
 
-from __future__ import division, absolute_import, with_statement, print_function, unicode_literals
-from renpy.compat import PY2, basestring, bchr, bord, chr, open, pystr, range, round, str, tobytes, unicode # *
-
-
-
 import renpy
 
 # The statement registry. It's a map from tuples giving the prefixes of
@@ -281,7 +276,6 @@ def register(
             l.subparses = [ ]
 
             text = l.text
-            subblock = l.subblock
 
             code_block = None
             atl = None
@@ -296,23 +290,23 @@ def register(
                 l.expect_block(" ".join(name) + " statement")
                 code_block = renpy.parser.parse_block(l.subblock_lexer())
             elif block == "script-possible":
-                if l.subblock:
+                if l.has_block():
                     code_block = renpy.parser.parse_block(l.subblock_lexer())
             elif block == "atl":
                 l.expect_block(" ".join(name) + " statement")
                 atl = renpy.atl.parse_atl(l.subblock_lexer())
             elif block == "atl-possible":
-                if l.subblock:
+                if l.has_block():
                     atl = renpy.atl.parse_atl(l.subblock_lexer())
 
-            start_line = l.line
+            start_line = l.number
 
             parsed = name, parse(l)
 
-            if l.line == start_line:
+            if l.number == start_line:
                 l.advance()
 
-            rv = renpy.ast.UserStatement(loc, text, subblock, parsed)
+            rv = renpy.ast.UserStatement(loc, text, [], parsed)
             rv.translatable = translatable
             rv.translation_relevant = bool(translation_strings)
             rv.code_block = code_block

renpy/translation/scanstrings.py

@@ -128,9 +128,10 @@ def scan_strings(filename):
     for line, s in renpy.game.script.translator.additional_strings[filename]: # @UndefinedVariable
         rv.append(String(filename, line, s, False))
 
-    for _filename, lineno, text in renpy.lexer.list_logical_lines(filename):
+    tok = renpy.tokenizer.from_file(filename)
+    for line in tok.logical_lines():
 
-        for m in re.finditer(STRING_RE, text):
+        for m in re.finditer(STRING_RE, line):
 
             s = m.group(1)
             s = s.replace('\\\n', "")
@@ -142,8 +143,10 @@ def scan_strings(filename):
                 if m.group(0).startswith("_p"):
                     s = renpy.minstore._p(s)
 
-                if s:
-                    rv.append(String(filename, lineno, s, False))
+                if not s:
+                    continue
+
+                rv.append(String(filename, line.physical_location.start_lineno, s, False))
 
     return rv
 

scripts/generate_pyi.py

@@ -68,10 +68,8 @@ def python_signature(o):
 
     s = s.replace("-> void", "")
 
-    lines = renpy.parser.list_logical_lines('<test>', s, 1, add_lines=True)
-    nested = renpy.parser.group_logical_lines(lines)
-
-    l = renpy.parser.Lexer(nested)
+    tok = renpy.tokenizer.from_string(s, "<generate_pyi>")
+    l = renpy.lexer.Lexer(list(tok.logical_lines()))
     l.advance()
 
     l.word()