Compare commits

...

5 Commits

Author SHA1 Message Date
Andy_kl 0f1a4b40f7 Rewrite fix_locations.
1. It makes `ast.increment_lineno` in the same pass.
2. It correctly sets `end_lineno/end_col_offset`.
3. It can strip `end_...`s for generated AST.
2024-11-02 19:59:57 +04:00
Andy_kl e5ec1ac42f Add monkeypatch of linecache.updatecache.
This replaces `renpy.lexer.get_line_text` but also
allows other python modules to read renpy files,
namely `traceback` module.
2024-11-02 19:27:52 +04:00
Andy_kl 6dfc282fa6 Do not accept ast in py_compile.
It was used in screen lang v1 and no longer used.
2024-10-29 14:33:42 +04:00
Andy_kl 834465af3c Render multi-caret parse errors correctly. 2024-10-28 22:31:11 +04:00
Andy_kl 290eeac6d8 Modernize ParserError and uses. 2024-10-28 19:38:44 +04:00
8 changed files with 319 additions and 155 deletions
+1
View File
@@ -253,6 +253,7 @@ name_blacklist = {
"renpy.exports.sdl_dll",
"renpy.sl2.slast.serial",
"renpy.gl2.gl2draw.default_position",
"renpy.lexer._python_updatecache",
}
class Backup(_object):
+81 -30
View File
@@ -509,50 +509,101 @@ init python:
return "\n".join(rv)
def __format_parse_errors(s):
def __format_parse_errors(errors: list[str]):
"""
Takes list of ParseError messages and transforms it
to string of sane length to be used as child of viewport.
"""
import re
rv = ""
rv = []
rv_len = 0
for error in errors:
lines = error.split("\n")
lines = s.split("\n")
lines = __limit_lines(lines)
len_lines = len(lines)
# Do not count first line in total to include those as much as
# possible.
rv.append(re.sub(
r'(File "(.*)", line (\d+))',
r'{a=edit:\3:\2}\1{/a}',
lines.pop(0)))
ln = 0
print(rv)
print(repr(lines))
while ln < len_lines:
line = lines[ln]
ln += 1
error_rv = []
error_len = 0
for line in lines:
# In case we have single insanely lengthy single error.
error_len += len(line)
if error_len > 65536:
error_rv.clear()
break
if ln < len_lines and lines[ln].endswith("^"):
highlight = len(lines[ln]) - 1
ln += 1
else:
highlight = -1
if set(line.strip()) == {"^"}:
idx = line.find("^")
count = line.count("^", idx)
pos = 0
prev_line = error_rv[-1]
for c in line:
if pos == highlight:
rv += u"{color=#c00}\u2192{/color}"
highlight = -1
line = prev_line[:idx]
pos += 1
if count == 1:
if len(prev_line) <= idx + count:
mark = "\u2190"
else:
mark = "\u2192"
count = 0
else:
mark = prev_line[idx:idx + count]
line += "{b}{color=#a00}"
line += mark
line += "{/color}{/b}"
line += prev_line[idx + count:]
if c == "{":
rv += "{{"
else:
rv += c
error_rv[-1] = line
continue
if highlight > 0:
rv += u"{color=#c00}\u2190{/color}"
error_rv.append(line)
rv += "\n"
rv_len += error_len
if rv_len > 1048576:
break
rv.extend(error_rv)
rv.append("")
rv = re.sub(r'(File "(.*)", line (\d+))', r'{a=edit:\3:\2}\1{/a}', rv)
# while ln < len_lines:
# line = lines[ln]
# ln += 1
return rv
# if ln < len_lines and "^" in lines[ln]:
# highlight = len(lines[ln]) - 1
# ln += 1
# else:
# highlight = -1
# pos = 0
# for c in line:
# if pos == highlight:
# rv += u"{color=#c00}\u2192{/color}"
# highlight = -1
# pos += 1
# if c == "{":
# rv += "{{"
# else:
# rv += c
# if highlight > 0:
# rv += u"{color=#c00}\u2190{/color}"
# rv += "\n"
return "\n".join(rv)
class _EditFile(Action):
def __init__(self, filename, line=1):
@@ -778,7 +829,7 @@ screen _exception:
key "console" action __EnterConsole()
# The screen that is used for error handling.
screen _parse_errors:
screen _parse_errors(errors, error_fn, reload_action):
modal True
layer config.interface_layer
zorder 1090
+5 -3
View File
@@ -162,7 +162,7 @@ def report_exception(short, full, traceback_fn):
raise
def report_parse_errors(errors, error_fn):
def report_parse_errors(errors: list[str], error_fn: str) -> bool:
"""
Reports an exception to the user. Returns True if the exception should
be raised by the normal reporting mechanisms. Otherwise, should raise
@@ -174,7 +174,7 @@ def report_parse_errors(errors, error_fn):
error_dump()
if renpy.game.args.command != "run": # @UndefinedVariable
if renpy.game.args.command != "run": # @UndefinedVariable
return True
if "RENPY_SIMPLE_EXCEPTIONS" in os.environ:
@@ -199,7 +199,7 @@ def report_parse_errors(errors, error_fn):
reload_action=reload_action,
errors=errors,
error_fn=error_fn,
)
)
except renpy.game.CONTROL_EXCEPTIONS:
raise
@@ -208,3 +208,5 @@ def report_parse_errors(errors, error_fn):
renpy.display.log.write("While handling exception:")
renpy.display.log.exception()
raise
return False
+138 -74
View File
@@ -23,114 +23,141 @@
from __future__ import division, absolute_import, with_statement, print_function, unicode_literals # type: ignore
from renpy.compat import PY2, basestring, bchr, bord, chr, open, pystr, range, round, str, tobytes, unicode # *
from typing import Any
import codecs
import re
import sys
import os
import time
import linecache
import contextlib
import renpy
from renpy.lexersupport import match_logical_word
# The filename that's in the line text cache.
line_text_filename = ""
# The content of the line text cache.
line_text_cache = [ ]
_python_updatecache = linecache.updatecache
def get_line_text(filename, lineno):
def _updatecache(filename: str, module_globals: dict[str, Any] | None = None) -> list[str]:
"""
Gets the text of a line, in a best-effort way, for debugging purposes. May
return just a newline, if the line doesn't exist.
Monkeypatch internal linecache.updatecache to work around RenPy python code
being in different files even in the same namespace. Assume filename is one
of '_ren.py', '.rpy' or '.rpym'.
This is needed because a lot of Python modules (namely traceback) assume
that the compiled python code is always comes from the python file and uses
linecache to get the source code, which tries to use PEP302 loader, which
for RenPy python code points to renpy.minstore.
"""
global line_text_filename
global line_text_cache
import linecache
full_filename = renpy.exports.unelide_filename(filename)
if full_filename != line_text_filename:
line_text_filename = full_filename
if filename.endswith(("_ren.py", ".rpy", ".rpym")):
# Assume filename also relative to basedir or renpy_base.
full_fn = renpy.lexer.unelide_filename(filename)
# If we can't find absolute path that way, assume we are
# in build, so we can't show source.
if not (os.path.isabs(full_fn) and os.path.exists(full_fn)):
linecache.cache.pop(filename, None)
return []
try:
with open(full_filename, "rb") as f:
with open(full_fn, "rb") as f:
data = f.read().decode("utf-8", "python_strict")
if full_filename.endswith("_ren.py"):
if full_fn.endswith("_ren.py"):
data = ren_py_to_rpy(data, None)
data += "\n\n"
line_text_cache = data.split("\n")
lines = data.split("\n")
if lines and lines[0].startswith("\ufeff"):
lines[0] = lines[0][1:]
lines = [line.removesuffix("\r") + "\n" for line in lines]
stat = os.stat(full_fn)
linecache.cache[filename] = stat.st_size, stat.st_mtime, lines, full_fn
return lines
except Exception:
line_text_cache = [ ]
return []
if lineno <= len(line_text_cache):
return line_text_cache[lineno - 1] + "\n"
else:
return "\n"
return _python_updatecache(filename, module_globals)
class ParseError(Exception):
linecache.updatecache = _updatecache
def __init__(self, filename, number, msg, line=None, pos=None, first=False):
message = u"File \"%s\", line %d: %s" % (unicode_filename(filename), number, msg)
if line:
if isinstance(line, list):
line = "".join(line)
class ParseError(SyntaxError):
"""
Special exception type for syntax errors in Ren'Py.
This exception includes syntax errors of Python code, converted to
appropriate report style, and Ren'Py own syntax errors in user script.
"""
lines = line.split('\n')
_message: str | None = None
if len(lines) > 1:
open_string = None
i = 0
def __init__(
self,
message: str,
filename: str,
lineno: int,
offset: int | None = None,
text: str | None = None,
end_lineno: int | None = None,
end_offset: int | None = None,
):
super().__init__(message, (
unicode_filename(filename),
lineno, offset,
text,
end_lineno, end_offset))
while i < len(lines[0]):
c = lines[0][i]
@property
def message(self) -> str:
"""
Fully formatted message of the error close to the result of
`traceback.print_exception_only`.
"""
if self._message is None:
message = f'File "{self.filename}", line {self.lineno}: {self.msg}'
if self.text is not None:
# Neither Python nor this class does not support multiline syntax error code.
# Just strip the first line of provided code.
text = self.text.split("\n")[0]
if c == "\\":
i += 1
elif c == open_string:
open_string = None
elif open_string:
pass
elif c == '`' or c == '\'' or c == '"':
open_string = c
# Remove ending escape chars, so we can render it.
text = text.rstrip()
i += 1
# And also replace any escape chars at the start with an indent.
message += f'\n {text.lstrip()}'
if open_string:
message += "\n(Perhaps you left out a %s at the end of the first line.)" % open_string
if self.offset is not None:
offset = self.offset
for l in lines:
message += "\n " + l
if pos is not None:
if pos <= len(l):
message += "\n " + " " * pos + "^"
pos = None
# Fallback to single caret for cases end_offset is before offset.
if self.end_offset is None or self.end_offset <= offset:
end_offset = offset + 1
else:
pos -= len(l)
end_offset = self.end_offset
if first:
break
left_spaces = len(text) - len(text.lstrip())
offset -= left_spaces
end_offset -= left_spaces
self.message = message
if offset >= 1:
caret_space = ' ' * (offset - 1)
carets = '^' * (end_offset - offset)
message += f"\n {caret_space}{carets}"
Exception.__init__(self, message)
for note in getattr(self, "__notes__", ()):
message += f"\n{note}"
def __unicode__(self):
return self.message
self._message = message
return self._message
def defer(self, queue):
renpy.parser.deferred_parse_errors[queue].append(self.message)
@@ -356,7 +383,8 @@ def list_logical_lines(filename, filedata=None, linenumber=1, add_lines=False):
c = data[pos]
if c == u'\t':
raise ParseError(filename, number, "Tab characters are not allowed in Ren'Py scripts.")
raise ParseError("Tab characters are not allowed in Ren'Py scripts.",
filename, number)
if c == u'\n' and not parendepth:
@@ -496,10 +524,19 @@ def list_logical_lines(filename, filedata=None, linenumber=1, add_lines=False):
pos = end
if (pos - startpos) > 65536:
raise ParseError(filename, start_number, "Overly long logical line. (Check strings and parenthesis.)", line=line, first=True)
err = ParseError(
"Overly long logical line.",
filename, start_number,
text="".join(line))
err.add_note("Check strings and parenthesis.")
raise err
if line:
raise ParseError(filename, start_number, "is not terminated with a newline. (Check strings and parenthesis.)", line=line, first=True)
err = ParseError("is not terminated with a newline.",
filename, start_number,
text="".join(line))
err.add_note("Check strings and parenthesis.")
raise err
return rv
@@ -547,7 +584,9 @@ def gll_core(lines, i, min_depth):
depth = line_depth
if depth != line_depth:
raise ParseError(filename, number, "Indentation mismatch.")
raise ParseError(
"Indentation mismatch.",
filename, number, text=text)
# Advance to the next line.
i += 1
@@ -573,7 +612,9 @@ def group_logical_lines(lines):
filename, number, text = lines[0]
if depth_split(text)[0] != 0:
raise ParseError(filename, number, "Unexpected indentation at start of file.")
raise ParseError(
"Unexpected indentation at start of file.",
filename, number, text=text)
return gll_core(lines, 0, 0)[0]
@@ -813,16 +854,35 @@ class Lexer(object):
except ParseError as e:
renpy.parser.parse_errors.append(e.message)
def error(self, msg):
def error(self, msg, start_pos: int | None = None, *, note: str | None = None):
"""
Convenience function for reporting a parse error at the current
location.
If `start_pos` is passed, it should be index of character in line where
bad token starts, otherwise error will render single caret at current position.
If `note` is passed, it should be a string which is added to error message after
error line, and is often an appropriate place to add suggestion of fix.
"""
if (self.line == -1) and self.block:
self.filename, self.number, self.text, self.subblock = self.block[0]
raise ParseError(self.filename, self.number, msg, self.text, self.pos)
if start_pos is None:
lineno = self.number
offset = self.pos + 1
end_lineno = end_offset = None
else:
lineno = end_lineno = self.number
offset = start_pos + 1
end_offset = self.pos + 1
raise ParseError(
msg, self.filename,
lineno, offset,
self.text,
end_lineno, end_offset)
def deferred_error(self, queue, msg):
"""
@@ -836,7 +896,10 @@ class Lexer(object):
if (self.line == -1) and self.block:
self.filename, self.number, self.text, self.subblock = self.block[0]
ParseError(self.filename, self.number, msg, self.text, self.pos).defer(queue)
ParseError(
msg, self.filename,
self.number, self.pos + 1,
self.text).defer(queue)
def eol(self):
"""
@@ -1582,7 +1645,7 @@ class Lexer(object):
return sp
def ren_py_to_rpy(text, filename):
def ren_py_to_rpy(text: str, filename: str | None) -> str:
"""
Transforms an _ren.py file into the equivalent .rpy file. This should retain line numbers.
@@ -1660,8 +1723,9 @@ def ren_py_to_rpy(text, filename):
raise Exception('In {!r}, there are no """renpy blocks, so every line is ignored.'.format(filename))
if state == RENPY:
raise Exception('In {!r}, there is a """renpy block at line {} that is not terminated by """.'.format(filename,
open_linenumber))
raise Exception(
'In {!r}, there is a """renpy block at line {} that is not terminated by """.'.format(
filename, open_linenumber))
rv = "\n".join(result)
+6 -3
View File
@@ -43,7 +43,6 @@ from renpy.lexer import (
munge_filename,
elide_filename,
unelide_filename,
get_line_text,
SubParse,
)
@@ -1763,8 +1762,10 @@ def report_parse_errors():
full_text = ""
f, error_fn = renpy.error.open_error_file("errors.txt", "w")
screen_parse_errors = []
with f:
f.write("\ufeff") # BOM
f.write("\ufeff") # BOM
print("I'm sorry, but errors were detected in your script. Please correct the", file=f)
print("errors listed below, and try again.", file=f)
@@ -1781,6 +1782,8 @@ def report_parse_errors():
print("", file=f)
print(i, file=f)
screen_parse_errors.append(i)
try:
print("")
print(i)
@@ -1791,7 +1794,7 @@ def report_parse_errors():
print("Ren'Py Version:", renpy.version, file=f)
print(str(time.ctime()), file=f)
renpy.display.error.report_parse_errors(full_text, error_fn)
renpy.display.error.report_parse_errors(screen_parse_errors, error_fn)
try:
if renpy.game.args.command == "run" or renpy.game.args.errors_in_editor: # type: ignore
+74 -28
View File
@@ -26,7 +26,7 @@
from __future__ import division, absolute_import, with_statement, print_function, unicode_literals
from renpy.compat import PY2, basestring, bchr, bord, chr, open, pystr, range, round, str, tobytes, unicode # *
from typing import Optional, Any
from typing import Literal, Optional, Any, cast, overload
import contextlib
# Import the python ast module, not ours.
@@ -876,35 +876,86 @@ py_compile_cache = { }
old_py_compile_cache = { }
def fix_locations(node, lineno, col_offset):
def fix_locations(
node: ast.AST,
lineno_increment: int = 0,
col_offset: int = 0,
strip_ends: bool = False,
_parent_offset=(0, 0),
_first=True,
):
"""
Assigns locations to the given node, and all of its children, adding
any missing line numbers and column offsets.
This function does `ast.increment_lineno`, `ast.fix_missing_locations`
and `[end_]col_offset` increments in one pass.
`lineno_increment`
Amount of lines to increment `lineno` by.
`col_offset`
0-indexed offset of source code. This should be used in case
source code have extra indent relative to compiled source.
`strip_ends`
If `True`, removes `end_lineno` and `end_col_offset`
so traceback does not render incorrect carets for given nodes tree.
"""
start = max(
(lineno, col_offset),
(getattr(node, "lineno", None) or 1, getattr(node, "col_offset", None) or 0)
)
if _first:
if not isinstance(node, (ast.Module, ast.Expression)):
raise ValueError(f"'fix_locations' called with unsupported top level node: {node}")
lineno, col_offset = start
# TypeIgnore is a special case where lineno is not an attribute
# but rather a field of the node itself.
if isinstance(node, ast.TypeIgnore):
node.lineno = getattr(node, 'lineno', 0) + lineno_increment
return
node.lineno = lineno
node.col_offset = col_offset
parent_lineno, parent_col_offset = _parent_offset
ends = [ start, (getattr(node, "end_lineno", None) or 1, getattr(node, "end_col_offset", None) or 0) ]
# lineno and col_offset must exist on every node.
# If there are none, we assume ast creator knows what they
# are doing and we can just assign to parent values.
if 'lineno' in node._attributes:
node = cast("ast.expr", node)
node.lineno = getattr(node, 'lineno', parent_lineno)
parent_lineno = node.lineno
node.lineno += lineno_increment
if 'col_offset' in node._attributes:
node = cast("ast.expr", node)
node.col_offset = getattr(node, 'col_offset', parent_col_offset)
parent_col_offset = node.col_offset
node.col_offset += col_offset
for child in ast.iter_child_nodes(node):
fix_locations(child, lineno, col_offset)
ends.append((child.end_lineno, child.end_col_offset))
fix_locations(
child,
lineno_increment,
col_offset,
strip_ends,
(parent_lineno, parent_col_offset),
False)
end = max(ends)
# end_lineno and end_col_offset are optional, and we remove them if
# strip_ends is True (for generated code that doesn't have source).
if 'end_lineno' in node._attributes:
node = cast("ast.expr", node)
if strip_ends:
node.end_lineno = None
else:
end_lineno = getattr(node, 'end_lineno', None)
node.end_lineno = (end_lineno or parent_lineno) + lineno_increment
node.end_lineno = end[0]
node.end_col_offset = end[1]
if 'end_col_offset' in node._attributes:
node = cast("ast.expr", node)
if strip_ends:
node.end_col_offset = None
else:
end_col_offset = getattr(node, 'end_col_offset', None)
node.end_col_offset = (end_col_offset or parent_col_offset) + col_offset
def quote_eval(s):
def quote_eval(s: str):
"""
Quotes a string for `eval`. This is necessary when it's in certain places,
like as part of an argument string. We need to stick a single backslash
@@ -999,16 +1050,15 @@ compile_filename = ""
def py_compile(source, mode, filename='<none>', lineno=1, ast_node=False, cache=True, py=None):
"""
Compiles the given source code using the supplied codegenerator.
Compiles the given source code using the supplied code generator.
Lists, List Comprehensions, and Dictionaries are wrapped when
appropriate.
`source`
The source code, as a either a string, pyexpr, or ast module
node.
The source code, as a either a string or PyExpr.
`mode`
One of "exec" or "eval".
One of "exec", "hide" or "eval".
`filename`
The filename the source comes from. If a pyexpr is given, the
@@ -1029,9 +1079,6 @@ def py_compile(source, mode, filename='<none>', lineno=1, ast_node=False, cache=
if ast_node:
cache = False
if isinstance(source, ast.Module):
return compile(source, filename, mode)
if isinstance(source, renpy.ast.PyExpr):
filename = source.filename
lineno = source.linenumber
@@ -1108,8 +1155,7 @@ def py_compile(source, mode, filename='<none>', lineno=1, ast_node=False, cache=
if mode == "hide":
wrap_hide(tree)
fix_locations(tree, 1, 0)
ast.increment_lineno(tree, lineno - 1)
fix_locations(tree, line_offset, 0)
line_offset = 0
@@ -1122,7 +1168,7 @@ def py_compile(source, mode, filename='<none>', lineno=1, ast_node=False, cache=
except SyntaxError as orig_e:
try:
tree = renpy.compat.fixes.fix_ast(tree)
fix_locations(tree, 1, 0)
fix_locations(tree)
with save_warnings():
rv = compile(tree, filename, py_mode, flags, 1)
except Exception:
+10 -13
View File
@@ -560,12 +560,13 @@ class Script(object):
if renpy.config.allow_duplicate_labels:
return
import linecache
self.duplicate_labels.append(
u'The label {} is defined twice, at File "{}", line {}:\n{}and File "{}", line {}:\n{}'.format(
bad_name, old_node.filename, old_node.linenumber,
renpy.lexer.get_line_text(old_node.filename, old_node.linenumber),
linecache.getline(old_node.filename, old_node.linenumber),
bad_node.filename, bad_node.linenumber,
renpy.lexer.get_line_text(bad_node.filename, bad_node.linenumber),
linecache.getline(bad_node.filename, bad_node.linenumber),
))
self.update_bytecode()
@@ -1015,18 +1016,14 @@ class Script(object):
code = renpy.python.py_compile_eval_bytecode(i.source, filename=i.location[0], lineno=i.location[1], py=i.py)
except SyntaxError as e:
text = e.text
if text is None:
text = ''
assert e.filename is not None
assert e.lineno is not None
pem = renpy.parser.ParseError(
filename=e.filename,
number=e.lineno,
msg=e.msg,
line=text,
pos=e.offset)
e.msg,
e.filename,
e.lineno, e.offset,
e.text,
e.end_lineno, e.end_offset)
renpy.parser.parse_errors.append(pem.message)
+4 -4
View File
@@ -198,10 +198,10 @@ class ScriptTranslator(object):
if n.identifier in self.default_translates:
old_node = self.default_translates[n.identifier]
renpy.lexer.ParseError(n.filename, n.linenumber, "Line with id %s appears twice. The other line is %s:%d" % (
n.identifier,
old_node.filename, old_node.linenumber)
).defer("duplicate_id")
renpy.lexer.ParseError(
f"Line with id {n.identifier} appears twice. "
f"The other line is {old_node.filename}:{old_node.linenumber}",
n.filename, n.linenumber,).defer("duplicate_id")
self.default_translates[n.identifier] = n
self.file_translates[filename].append((label, n))