pig2014/renpy
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Track column numbers during parsing.

Browse Source 
How these are handled varies for PyCode and PyExpr.

PyCodes are, for now, parsed with all the indentation present in the
file. This is then passed to the parser, which pads the PyCode out
with an if True statement so it compiles.

PyExprs have the column number determined by the parser, and it's
stored in the object, to be passed to the compiler.
This commit is contained in:
Tom Rothamel
2025-02-16 21:27:54 -05:00
parent e1b8de8512
commit e89441a77c
3 changed files with 114 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
renpy/astsupport.pyx
+51 -3
View File
@@ -73,11 +73,12 @@ cdef class PyExpr(str):
cdef public str filename
cdef public unsigned int hashcode
cdef public int linenumber
cdef public unsigned int linenumber
cdef public unsigned short column
cdef public unsigned char py
def __reduce__(self):
return (PyExpr, (str(self), self.filename, self.linenumber, self.py, self.hashcode))
return (PyExpr, (str(self), self.filename, self.linenumber, self.py, self.hashcode, self.column))
@staticmethod
def checkpoint() -> Any:
@@ -116,15 +117,20 @@ cdef object PyExpr_new(type cls, PyObject *args, PyObject *kwargs):
cdef tuple cargs = <tuple> args
if len(cargs) == 5:
if len(cargs) == 6:
s, filename, linenumber, py, hashcode, column = cargs
elif len(cargs) == 5:
s, filename, linenumber, py, hashcode = cargs
column = 0
elif len(cargs) == 4:
s, filename, linenumber, py = cargs
hashcode = None
column = 0
elif len(cargs) == 3:
s, filename, linenumber = cargs
py = 2
hashcode = None
column = 0
else:
raise Exception("PyExpr.__new__ called with invalid arguments.", str(<object> args))
@@ -136,6 +142,7 @@ cdef object PyExpr_new(type cls, PyObject *args, PyObject *kwargs):
rv.filename = filename
rv.linenumber = linenumber
rv.column = column
rv.py = py
if hashcode is not None:
@@ -152,3 +159,44 @@ cdef object PyExpr_new(type cls, PyObject *args, PyObject *kwargs):
return rv
PyExprType.tp_new = <newfunc> PyExpr_new
def make_pyexpr(s, str filename, int linenumber, int column, str text, int pos):
"""
Used by lexer to make a pyexpr, rapidly adjusting line number and column.
`s`
The string that is the expression.
`filename`
The name of the file the expression is in.
`linenumber`
The line number the logical line starts at.
`column`
The column the logical line starts at.
`text`
The text of the line.
`pos`
The position in the text where the expression starts.
"""
cdef Py_UCS4 c
cdef int i = 0
for c in text:
if i >= pos:
break
i += 1
if c == 10: # NL
linenumber += 1
column = 0
else:
column += 1
return PyExpr(s, filename, linenumber, 3, hash32(s), column)
renpy/lexer.py
+49 -27
View File
@@ -20,7 +20,7 @@
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
from typing import Callable
from typing import Callable, NamedTuple
import re
import sys
@@ -31,6 +31,7 @@ import functools
import renpy
from renpy.lexersupport import match_logical_word
from renpy.astsupport import make_pyexpr
class ParseError(SyntaxError):
@@ -485,9 +486,9 @@ def list_logical_lines(filename, filedata=None, linenumber=1, add_lines=False):
return rv
def depth_split(l):
def split_indent(l):
"""
Returns the length of the line's prefix, and the rest of the line.
Returns the length of the line's indentation, and the rest of the line.
"""
depth = 0
@@ -506,6 +507,14 @@ def depth_split(l):
# i, min_depth -> block, new_i
class GroupedLine(NamedTuple):
# The filename the line is from.
filename: str
number: int
indent: int
text: str
block: list
def gll_core(lines, i, min_depth):
"""
Recursively groups lines into blocks.
@@ -520,16 +529,16 @@ def gll_core(lines, i, min_depth):
filename, number, text = lines[i]
line_depth, rest = depth_split(text)
indent, rest = split_indent(text)
# This catches a block exit.
if line_depth < min_depth:
if indent < min_depth:
break
if depth is None:
depth = line_depth
depth = indent
if depth != line_depth:
if depth != indent:
raise ParseError(
"Indentation mismatch.",
filename, number, text=text)
@@ -540,17 +549,16 @@ def gll_core(lines, i, min_depth):
# Try parsing a block associated with this line.
block, i = gll_core(lines, i, depth + 1)
rv.append((filename, number, rest, block))
rv.append(GroupedLine(filename, number, indent, rest, block))
return rv, i
def group_logical_lines(lines):
"""
This takes as input the list of logical line triples output from
list_logical_lines, and breaks the lines into blocks. Each block
is represented as a list of (filename, line number, line text,
block) triples, where block is a block list (which may be empty if
is represented as a list of (filename, line number, starting column, line text,
block) tuples, where block is a block list (which may be empty if
no block is associated with this line.)
"""
@@ -558,7 +566,7 @@ def group_logical_lines(lines):
filename, number, text = lines[0]
if depth_split(text)[0] != 0:
if split_indent(text)[0] != 0:
raise ParseError(
"Unexpected indentation at start of file.",
filename, number, text=text)
@@ -653,8 +661,16 @@ class Lexer(object):
sub-lexers to lex sub-blocks.
"""
block: list[GroupedLine]
def __init__(self, block, init=False, init_offset=0, global_label=None, monologue_delimiter="\n\n", subparses=None):
# Older version of Lexer had block being a list of tuples. Those lists can be found in UserStatements,
# and so need to be upgraded.
if block and len(block[0]) == 4:
self.block = [ GroupedLine(filename, line, 0, text, subblock) for filename, line, text, subblock in block ]
# Are we underneath an init block?
self.init = init
@@ -677,10 +693,14 @@ class Lexer(object):
self.word_cache_newpos = -1
self.word_cache = ""
# The column text starts at.
self.column = 0
self.monologue_delimiter = monologue_delimiter
self.subparses = subparses
def _unmunge_string(self, s: str) -> str:
prefix = munge_filename(self.filename)
return s.replace(prefix, "__")
@@ -703,7 +723,7 @@ class Lexer(object):
self.eob = True
return False
self.filename, self.number, self.text, self.subblock = self.block[self.line]
self.filename, self.number, self.column, self.text, self.subblock = self.block[self.line]
self.pos = 0
self.word_cache_pos = -1
@@ -718,7 +738,7 @@ class Lexer(object):
self.line -= 1
self.eob = False
self.filename, self.number, self.text, self.subblock = self.block[self.line]
self.filename, self.number, self.column, self.text, self.subblock = self.block[self.line]
self.pos = len(self.text)
self.word_cache_pos = -1
@@ -750,8 +770,6 @@ class Lexer(object):
Advances the current position beyond any contiguous whitespace.
"""
# print self.text[self.pos].encode('unicode_escape')
self.match_regexp(r"(\s+|\\\n)+")
def match(self, regexp):
@@ -812,7 +830,7 @@ class Lexer(object):
"""
if (self.line == -1) and self.block:
self.filename, self.number, self.text, self.subblock = self.block[0]
self.filename, self.number, self.column, self.text, self.subblock = self.block[0]
raise ParseError(
msg,
@@ -831,7 +849,7 @@ class Lexer(object):
"""
if (self.line == -1) and self.block:
self.filename, self.number, self.text, self.subblock = self.block[0]
self.filename, self.number, self.column, self.text, self.subblock = self.block[0]
ParseError(
msg, self.filename,
@@ -1238,9 +1256,10 @@ class Lexer(object):
if not expr:
return s
pos = self.pos - len(s)
s = self._unmunge_string(s)
return renpy.ast.PyExpr(s, self.filename, self.number)
return make_pyexpr(s, self.filename, self.number, self.column, self.text, pos)
def delimited_python(self, delim, expr=True):
"""
@@ -1417,7 +1436,7 @@ class Lexer(object):
passed to revert to back the lexer up.
"""
return self.line, self.filename, self.number, self.text, self.subblock, self.pos, renpy.ast.PyExpr.checkpoint()
return self.line, self.filename, self.number, self.text, self.subblock, self.pos, self.column, renpy.ast.PyExpr.checkpoint()
def revert(self, state):
"""
@@ -1425,7 +1444,7 @@ class Lexer(object):
by a previous checkpoint operation on this lexer.
"""
self.line, self.filename, self.number, self.text, self.subblock, self.pos, pyexpr_checkpoint = state
self.line, self.filename, self.number, self.text, self.subblock, self.pos, self.column, pyexpr_checkpoint = state
renpy.ast.PyExpr.revert(pyexpr_checkpoint)
@@ -1489,19 +1508,22 @@ class Lexer(object):
self.pos = len(self.text)
return self.text[pos:].strip()
def _process_python_block(self, block, indent, rv, line_holder):
for _fn, ln, text, subblock in block:
def _process_python_block(self, block, rv, line_holder):
for _fn, ln, indent, text, subblock in block:
prefix = " " * indent
while line_holder.line < ln:
rv.append(indent + '\n')
rv.append(prefix + '\n')
line_holder.line += 1
linetext = indent + text + '\n'
linetext = prefix + text + '\n'
rv.append(linetext)
line_holder.line += linetext.count('\n')
self._process_python_block(subblock, indent + ' ', rv, line_holder)
self._process_python_block(subblock, rv, line_holder)
def python_block(self):
"""
@@ -1515,7 +1537,7 @@ class Lexer(object):
line_holder = LineNumberHolder()
line_holder.line = self.number
self._process_python_block(self.subblock, '', rv, line_holder)
self._process_python_block(self.subblock, rv, line_holder)
return self._unmunge_string(''.join(rv))
def arguments(self):
renpy/python.py
+14 -2
View File
@@ -1112,6 +1112,14 @@ def py_compile(source, mode, filename='<none>', lineno=1, ast_node=False, cache=
if py is None:
py = 3
# This determines if the lines are indented. If so, we adjust the
# ast to match.
indented = source and (source[0] == " ")
if indented:
lineno -= 1
source = "if True:\n" + source
flags = file_compiler_flags.get(filename, 0)
if cache:
@@ -1130,6 +1138,7 @@ def py_compile(source, mode, filename='<none>', lineno=1, ast_node=False, cache=
return rv
bytecode = renpy.game.script.bytecode_oldcache.get(key, None)
if bytecode is not None:
try:
@@ -1192,8 +1201,11 @@ def py_compile(source, mode, filename='<none>', lineno=1, ast_node=False, cache=
if not handled:
raise
tree = wrap_node.visit(tree)
# If the body is indented, it's wrapped in an "if True:" statement, which needs to be eliminated.
if indented:
tree.body = tree.body[0].body
tree = wrap_node.visit(tree)
tree = MungeNodes(filename).visit(tree)
if mode == "hide":
@@ -1223,7 +1235,7 @@ def py_compile(source, mode, filename='<none>', lineno=1, ast_node=False, cache=
if not handled:
raise
if cache:
py_compile_cache[key] = rv