Implement 'add image' screen language statement.

README.rst

@@ -71,14 +71,14 @@ To return to this virtualenv later, run::
 
 After activating the virtualenv, install additional dependencies::
 
-    pip install -U "setuptools cython<3.0.0 future six typing pefile requests ecdsa"
+    pip install -U cython future six typing pefile requests ecdsa
 
 Then, install pygame_sdl2 by running the following commands::
 
     git clone https://www.github.com/renpy/pygame_sdl2
     pushd pygame_sdl2
     python setup.py install
-    python install_headers.py $VIRTUAL_ENV
+    python setup.py install_headers
     popd
 
 Next, set RENPY_DEPS_INSTALL To a \:-separated (\;-separated on Windows)

distribute.py

@@ -252,7 +252,7 @@ def main():
             "-q",
             "egg_info",
             "--tag-build",
-            "+renpy" + args.version.replace("+", "-"),
+            "+renpy" + args.version,
             "sdist",
             "-d",
             os.path.abspath(destination)

launcher/game/change_icon.py

@@ -76,14 +76,24 @@ class BinFile(object):
         self.addr = addr
 
     def tostring(self):
-        return self.a.tobytes()
+        if PY2:
+            return self.a.tostring() # type: ignore
+        else:
+            return self.a.tobytes()
 
     def substring(self, start, len): # @ReservedAssignment
-        return self.a[start:start + len].tobytes()
+        if PY2:
+            return self.a[start:start + len].tostring() # type: ignore
+        else:
+            return self.a[start:start + len].tobytes()
 
     def __init__(self, data):
         self.a = array.array('B')
-        self.a.frombytes(data)
+
+        if PY2:
+            self.a.fromstring(data) # type: ignore
+        else:
+            self.a.frombytes(data)
 
 ##############################################################################
 # These functions parse data out of the file. In these functions, offset is

launcher/game/distribute.rpy

@@ -432,7 +432,10 @@ fix_dlc("renios", "renios")
             for f in sorted(self, key=lambda a : a.name):
                 f.hash(sha, distributor)
 
-            return sha.hexdigest()
+            if PY2:
+                return sha.hexdigest().decode("utf-8")
+            else:
+                return sha.hexdigest()
 
         def split_by_prefix(self, prefix):
             """
@@ -658,6 +661,7 @@ fix_dlc("renios", "renios")
             # Build the mac app and windows exes.
             self.add_mac_files()
             self.add_windows_files()
+            self.add_main_py()
 
             # Add the main.py.
             self.add_main_py()
@@ -1071,8 +1075,11 @@ fix_dlc("renios", "renios")
 
             rv = self.temp_filename("Info.plist")
 
-            with open(rv, "wb") as f:
-                plistlib.dump(plist, f)
+            if PY2:
+                plistlib.writePlist(plist, rv)
+            else:
+                with open(rv, "wb") as f:
+                    plistlib.dump(plist, f)
 
             return rv
 
@@ -1219,8 +1226,19 @@ fix_dlc("renios", "renios")
                 if os.path.exists(tmp):
                     self.add_file(fl, dst, tmp)
 
-            write_exe("lib/py3-windows-x86_64/renpy.exe", self.exe, self.exe, windows)
-            write_exe("lib/py3-windows-x86_64/pythonw.exe", "lib/py3-windows-x86_64/pythonw.exe", "pythonw-64.exe", windows)
+            if PY2:
+
+                if self.build["include_i686"]:
+                    write_exe("lib/py2-windows-i686/renpy.exe", self.exe32, self.exe32, windows_i686)
+                    write_exe("lib/py2-windows-i686/pythonw.exe", "lib/py2-windows-i686/pythonw.exe", "pythonw-32.exe", windows_i686)
+
+                write_exe("lib/py2-windows-x86_64/renpy.exe", self.exe, self.exe, windows)
+                write_exe("lib/py2-windows-x86_64/pythonw.exe", "lib/py2-windows-x86_64/pythonw.exe", "pythonw-64.exe", windows)
+
+            else:
+
+                write_exe("lib/py3-windows-x86_64/renpy.exe", self.exe, self.exe, windows)
+                write_exe("lib/py3-windows-x86_64/pythonw.exe", "lib/py3-windows-x86_64/pythonw.exe", "pythonw-64.exe", windows)
 
 
         def add_main_py(self):
@@ -1513,12 +1531,11 @@ fix_dlc("renios", "renios")
 
             update = { variant : { "version" : self.update_versions[variant], "base_name" : self.base_name, "files" : update_files, "directories" : update_directories, "xbit" : update_xbit } }
 
-            update_fn = self.temp_filename(filename + ".update.json")
-
+            update_fn = os.path.join(self.destination, filename + ".update.json")
 
             if self.include_update and not format.startswith("app-"):
 
-                with open(update_fn, "w") as f:
+                with open(update_fn, "wb" if PY2 else "w") as f:
                     json.dump(update, f, indent=2)
 
                 if (not dlc) or (format == "update"):
@@ -1548,11 +1565,14 @@ fix_dlc("renios", "renios")
                 in this thread or a background thread.
                 """
 
-                final_update_fn = os.path.join(self.destination, filename + ".update.json")
-
-                if self.build_update or dlc:
+                if self.include_update and not self.build_update and not dlc:
                     if os.path.exists(update_fn):
-                        shutil.copy(update_fn, final_update_fn)
+                        os.unlink(update_fn)
+
+                if not directory:
+                    file_hash = hash_file(path)
+                else:
+                    file_hash = ""
 
             if format == "tar.bz2" or format == "bare-tar.bz2":
                 pkg = TarPackage(path, "w:bz2")

launcher/game/front_page.rpy

@@ -158,8 +158,12 @@ screen front_page_project:
 
                 frame style "l_indent":
                     has vbox
-                    for button_name, path in p.data["renpy_launcher"]["open_directory"].items():
-                        textbutton button_name action OpenDirectory(os.path.join(p.path, path), absolute=True)
+
+                    textbutton "game" action OpenDirectory(os.path.join(p.path, "game"), absolute=True)
+                    textbutton "base" action OpenDirectory(os.path.join(p.path, "."), absolute=True)
+                    textbutton "images" action OpenDirectory(os.path.join(p.path, "game/images"), absolute=True)
+                    textbutton "audio" action OpenDirectory(os.path.join(p.path, "game/audio"), absolute=True)
+                    textbutton "gui" action OpenDirectory(os.path.join(p.path, "game/gui"), absolute=True)
 
             vbox:
                 if persistent.show_edit_funcs:
@@ -169,8 +173,10 @@ screen front_page_project:
                     frame style "l_indent":
                         has vbox
 
-                        for button_name, path in p.data["renpy_launcher"]["edit_file"].items():
-                            textbutton button_name action editor.Edit(path, check=True)
+                        textbutton "script.rpy" action editor.Edit("game/script.rpy", check=True)
+                        textbutton "options.rpy" action editor.Edit("game/options.rpy", check=True)
+                        textbutton "gui.rpy" action editor.Edit("game/gui.rpy", check=True)
+                        textbutton "screens.rpy" action editor.Edit("game/screens.rpy", check=True)
 
                         if editor.CanEditProject():
                             textbutton _("Open project") action editor.EditProject()

launcher/game/gui7/code.py

@@ -172,7 +172,7 @@ class CodeGenerator(object):
 
         for l in self.lines:
 
-            m = re.match(r'^(\s*)define (.*?) =', l)
+            m = re.match('^(\s*)define (.*?) =', l)
 
             if m:
                 indent = m.group(1)

launcher/game/install.rpy

@@ -81,7 +81,10 @@ init python:
 
 label install_live2d:
     python hide:
-        _prefix = r"lib/py3-"
+        if PY2:
+            _prefix = r"lib/py2-"
+        else:
+            _prefix = r"lib/py3-"
 
         patterns = [
             (r".*/Core/dll/linux/x86_64/(libLive2DCubismCore.so)", _prefix + r"linux-x86_64/\1"),
@@ -94,6 +97,11 @@ label install_live2d:
             (r".*/Core/dll/android/(x86_64/libLive2DCubismCore.so)", r"rapt/prototype/renpyandroid/src/main/jniLibs/\1"),
         ]
 
+        if PY2:
+           patterns.extend([
+                (r".*/Core/dll/windows/x86/(Live2DCubismCore.dll)", _prefix + r"windows-i686/\1"),
+           ])
+
         install_from_zip("Live2D Cubism SDK for Native", "CubismSdkForNative-[45]-*.zip", patterns)
 
     jump front_page

launcher/game/installer.py

@@ -169,7 +169,7 @@ def _check_hash(filename, hashj):
 download_file = ""
 download_url = ""
 
-def download(url, filename, hash=None, headers=None, requests_kwargs={}):
+def download(url, filename, hash=None):
     """
     Downloads `url` to `filename`, a tempfile.
     """
@@ -188,13 +188,9 @@ def download(url, filename, hash=None, headers=None, requests_kwargs={}):
 
     progress_time = time.time()
 
-    all_headers = {"User-Agent": 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.0.0 Safari/537.36 renpy/8'}
-    if headers:
-        all_headers.update(headers)
-
     try:
 
-        response = requests.get(url, stream=True, proxies=renpy.exports.proxies, timeout=15, headers=all_headers, **requests_kwargs)
+        response = requests.get(url, stream=True, proxies=renpy.exports.proxies, timeout=15)
         response.raise_for_status()
 
         total_size = int(response.headers.get('content-length', 1))

launcher/game/ios.rpy

@@ -164,6 +164,9 @@ init python:
 
         print("Eliminating __pycache__...")
 
+        if PY2:
+            return
+
         import pathlib
         import sys
 

launcher/game/new_project.rpy

@@ -34,6 +34,9 @@ label new_project:
     if persistent.projects_directory is None:
         $ interface.error(_("The projects directory could not be set. Giving up."))
 
+    if PY2:
+        $ interface.info(_("Warning : you are using Ren'Py 7. It is recommended to start new projects using Ren'Py 8 instead."))
+
     python:
         new_project_language = __("{#language name and font}")
         gui_kind = "new_gui_project"

launcher/game/options.rpy

@@ -301,8 +301,6 @@ init python:
     build.classify_renpy("**.new", None)
     build.classify_renpy("**.bak", None)
 
-    build.classify_renpy("**.keystore", None)
-
     build.classify_renpy("**/log.txt", None)
     build.classify_renpy("**/traceback.txt", None)
     build.classify_renpy("**/errors.txt", None)
@@ -322,7 +320,7 @@ init python:
         """
 
         if py is True:
-            py = 'pycache'
+            py = 'pyo' if PY2 else 'pycache'
 
         if py == 'pycache':
             build.classify_renpy(pattern + "/**__pycache__/", binary)
@@ -357,7 +355,6 @@ init python:
 
         build.classify_renpy(pattern + "/**", source)
 
-    build.classify_renpy("renpy/gl/", None)
 
     build.classify_renpy("renpy.py", "binary")
     source_and_binary("renpy")
@@ -414,11 +411,18 @@ init python:
     build.classify_renpy("lib/**/*steam_api*", "steam")
     build.classify_renpy("lib/**/*Live2D*", None)
 
-    build.classify_renpy("lib/py3-linux-armv7l/**", None)
-    build.classify_renpy("lib/py3-linux-aarch64/**", "linux_arm")
-    source_and_binary("lib/py3-**", "binary", "binary")
-    source_and_binary("lib/python3**", "binary", "binary", py='pyc')
-    build.classify_renpy("renpy3.sh", "binary")
+    if PY2:
+        build.classify_renpy("lib/py2-linux-armv7l/**", "linux_arm")
+        build.classify_renpy("lib/py2-linux-aarch64/**", "linux_arm")
+        source_and_binary("lib/py2-**", "binary", "binary")
+        source_and_binary("lib/python2**", "binary", "binary")
+        build.classify_renpy("renpy2.sh", "binary")
+    else:
+        build.classify_renpy("lib/py3-linux-armv7l/**", "linux_arm")
+        build.classify_renpy("lib/py3-linux-aarch64/**", "linux_arm")
+        source_and_binary("lib/py3-**", "binary", "binary")
+        source_and_binary("lib/python3**", "binary", "binary", py='pyc')
+        build.classify_renpy("renpy3.sh", "binary")
 
     build.classify_renpy("lib/", "binary")
 

launcher/game/package_formats.rpy

@@ -34,21 +34,106 @@ init python in distribute:
     from zipfile import crc32
 
     # Since the long type doesn't exist on py3, define it here
-    long = int
+    if not PY2:
+        long = int
 
     zlib.Z_DEFAULT_COMPRESSION = 5
 
-    class ZipFile(zipfile.ZipFile):
+    if PY2:
 
-        def write_with_info(self, zinfo, filename):
+        class ZipFile(zipfile.ZipFile):
 
-            if zinfo.filename.endswith("/"):
-                data = b''
-            else:
-                with open(filename, "rb") as f:
-                    data = f.read()
+            def write_with_info(self, zinfo, filename):
+                """Put the bytes from filename into the archive under the name
+                arcname."""
+                if not self.fp:
+                    raise RuntimeError(
+                        "Attempt to write to ZIP archive that was already closed")
 
-            self.writestr(zinfo, data)
+                st = os.stat(filename)
+                isdir = stat.S_ISDIR(st.st_mode)
+
+                zinfo.file_size = st.st_size
+                zinfo.flag_bits = 0x00
+                zinfo.header_offset = self.fp.tell()    # Start of header bytes
+
+                self._writecheck(zinfo)
+                self._didModify = True
+
+                if isdir:
+                    zinfo.file_size = 0
+                    zinfo.compress_size = 0
+                    zinfo.CRC = 0
+                    self.filelist.append(zinfo)
+                    self.NameToInfo[zinfo.filename] = zinfo
+                    self.fp.write(zinfo.FileHeader())
+                    return
+
+                with open(filename, "rb") as fp:
+                    # Must overwrite CRC and sizes with correct data later
+                    zinfo.CRC = CRC = 0
+                    zinfo.compress_size = compress_size = 0
+                    file_size = 0
+
+                    zip64 = self._allowZip64 and \
+                            zinfo.file_size * 1.05 > zipfile.ZIP64_LIMIT
+
+                    self.fp.write(zinfo.FileHeader(zip64))
+
+                    if zinfo.compress_type == zipfile.ZIP_DEFLATED:
+                        cmpr = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION,
+                            zlib.DEFLATED, -15)
+                    else:
+                        cmpr = None
+                    while 1:
+                        buf = fp.read(1024 * 1024)
+                        if not buf:
+                            break
+                        file_size = file_size + len(buf)
+                        CRC = crc32(buf, CRC) & 0xffffffff
+                        if cmpr:
+                            buf = cmpr.compress(buf)
+                            compress_size = compress_size + len(buf)
+                        self.fp.write(buf)
+                if cmpr:
+                    buf = cmpr.flush()
+                    compress_size = compress_size + len(buf)
+                    self.fp.write(buf)
+                    zinfo.compress_size = compress_size
+                else:
+                    zinfo.compress_size = file_size
+                zinfo.CRC = CRC
+                zinfo.file_size = file_size
+
+                if not zip64 and self._allowZip64:
+                    if file_size > zipfile.ZIP64_LIMIT:
+                        raise RuntimeError('File size has increased during compressing')
+                    if compress_size > zipfile.ZIP64_LIMIT:
+                        raise RuntimeError('Compressed size larger than uncompressed size')
+
+                # Seek backwards and write CRC and file sizes
+                position = self.fp.tell()       # Preserve current position in file
+                self.fp.seek(zinfo.header_offset, 0)
+
+                self.fp.write(zinfo.FileHeader(zip64))
+
+                self.fp.seek(position, 0)
+                self.filelist.append(zinfo)
+                self.NameToInfo[zinfo.filename] = zinfo
+
+    else:
+
+        class ZipFile(zipfile.ZipFile):
+
+            def write_with_info(self, zinfo, filename):
+
+                if zinfo.filename.endswith("/"):
+                    data = b''
+                else:
+                    with open(filename, "rb") as f:
+                        data = f.read()
+
+                self.writestr(zinfo, data)
 
 
     class ZipPackage(object):
@@ -384,5 +469,3 @@ init python in distribute:
         for i in parallel_threads:
             if i.done:
                 i.done()
-
-        parallel_threads.clear()

launcher/game/project.rpy

@@ -123,25 +123,6 @@ init python in project:
         def update_data(self):
             data = self.data
 
-            data.setdefault("renpy_launcher",
-            {
-                "open_directory":
-                {
-                    "game": "game",
-                    "base": ".",
-                    "images": "game/images",
-                    "audio": "game/audio",
-                    "gui": "game/gui"
-                },
-                "edit_file":
-                {
-                    "script.rpy": "game/script.rpy",
-                    "options.rpy": "game/options.rpy",
-                    "gui.rpy": "game/gui.rpy",
-                    "screens.rpy": "game/screens.rpy"
-                }
-            })
-
             data.setdefault("build_update", False)
             data.setdefault("packages", [ "pc", "mac" ])
             data.setdefault("add_from", True)
@@ -371,6 +352,12 @@ init python in project:
 
                     line = line[:1024]
 
+                    if PY2:
+                        try:
+                            line = line.decode("utf-8")
+                        except Exception:
+                            continue
+
                     m = re.search(r"#\s*TODO(\s*:\s*|\s+)(.*)", line, re.I)
 
                     if m is None:

launcher/game/web.rpy

@@ -440,12 +440,13 @@ init python:
         # Find the presplash and copy it over.
         presplash = None
 
-        for fn in [ "web-presplash.png", "web-presplash.jpg", "web-presplash.webp" ]:
-            fullfn = os.path.join(project.current.path, fn)
+        if not PY2:
+            for fn in [ "web-presplash.png", "web-presplash.jpg", "web-presplash.webp" ]:
+                fullfn = os.path.join(project.current.path, fn)
 
-            if os.path.exists(fullfn):
-                presplash = fn
-                break
+                if os.path.exists(fullfn):
+                    presplash = fn
+                    break
 
         if presplash:
             os.unlink(os.path.join(destination, "web-presplash.jpg"))
@@ -455,16 +456,20 @@ init python:
         with io.open(os.path.join(WEB_PATH, "index.html"), encoding='utf-8') as f:
             html = f.read()
 
-        html = html.replace("Ren'Py Web Game", display_name)
+        if PY2:
+            html = html.replace("%%TITLE%%", display_name)
+        else:
+            html = html.replace("Ren'Py Web Game", display_name)
 
-        if presplash:
-            html = html.replace("web-presplash.jpg", presplash)
+            if presplash:
+                html = html.replace("web-presplash.jpg", presplash)
 
         with io.open(os.path.join(destination, "index.html"), "w", encoding='utf-8') as f:
             f.write(html)
 
-        generate_web_icons(p, destination)
-        prepare_pwa_files(p, destination)
+        if not PY2:
+            generate_web_icons(p, destination)
+            prepare_pwa_files(p, destination)
 
         # Zip up the game.
 

launcher/game/webserver.py

@@ -17,8 +17,6 @@ root = ""
 
 import renpy
 
-class BannedException(Exception):
-    pass
 
 class WebHandler(http.server.BaseHTTPRequestHandler):
 
@@ -61,13 +59,7 @@ class WebHandler(http.server.BaseHTTPRequestHandler):
         None, in which case the caller has nothing further to do.
 
         """
-
-        try:
-            path = self.translate_path(self.path)
-        except BannedException:
-            self.send_error(403, "File not found")
-            return None
-
+        path = self.translate_path(self.path)
         f = None
         if os.path.isdir(path):
             parts = urllib.parse.urlsplit(self.path)
@@ -200,9 +192,6 @@ class WebHandler(http.server.BaseHTTPRequestHandler):
         path = root
 
         for word in words:
-            if word.startswith("."):
-                raise BannedException("Invalid path.")
-
             if os.path.dirname(word) or word in (os.curdir, os.pardir):
                 # Ignore components that are not a simple file/directory name
                 continue

module/Setup

@@ -6,6 +6,7 @@ renpy.lexersupport gen/renpy.lexersupport.c
 renpy.pydict gen/renpy.pydict.c
 renpy.style gen/renpy.style.c
 renpy.encryption gen/renpy.encryption.c
+renpy.compat.dictviews gen/renpy.compat.dictviews.c
 renpy.styledata.styleclass gen/renpy.styledata.styleclass.c
 renpy.styledata.stylesets gen/renpy.styledata.stylesets.c
 renpy.styledata.style_selected_hover_functions gen/renpy.styledata.style_selected_hover_functions.c

module/ffmedia.c

@@ -71,11 +71,7 @@ static int rwops_read(void *opaque, uint8_t *buf, int buf_size) {
 
 }
 
-#if (LIBAVFORMAT_VERSION_MAJOR < 61)
 static int rwops_write(void *opaque, uint8_t *buf, int buf_size) {
-#else
-static int rwops_write(void *opaque, const uint8_t *buf, int buf_size) {
-#endif
     printf("Writing to an SDL_rwops is a really bad idea.\n");
     return -1;
 }
@@ -694,14 +690,9 @@ static void decode_audio(MediaState *ms) {
 			}
 
             converted_frame->sample_rate = audio_sample_rate;
-#if (LIBAVUTIL_VERSION_MAJOR < 59)
             converted_frame->channel_layout = AV_CH_LAYOUT_STEREO;
-#else
-            converted_frame->ch_layout = (AVChannelLayout) AV_CHANNEL_LAYOUT_STEREO;
-#endif
             converted_frame->format = AV_SAMPLE_FMT_S16;
 
-#if (LIBAVUTIL_VERSION_MAJOR < 59)
 			if (!ms->audio_decode_frame->channel_layout) {
 				ms->audio_decode_frame->channel_layout = av_get_default_channel_layout(ms->audio_decode_frame->channels);
 
@@ -720,26 +711,6 @@ static void decode_audio(MediaState *ms) {
 				    swr_set_matrix(ms->swr, stereo_matrix, 1);
 				}
 			}
-#else
-			if (ms->audio_decode_frame->ch_layout.order == AV_CHANNEL_ORDER_UNSPEC) {
-				av_channel_layout_default(&ms->audio_decode_frame->ch_layout, ms->audio_decode_frame->ch_layout.nb_channels);
-
-				if (audio_equal_mono && (ms->audio_decode_frame->ch_layout.nb_channels == 1)) {
-				    swr_alloc_set_opts2(
-                        &ms->swr,
-                        &converted_frame->ch_layout,
-                        converted_frame->format,
-                        converted_frame->sample_rate,
-                        &ms->audio_decode_frame->ch_layout,
-                        ms->audio_decode_frame->format,
-                        ms->audio_decode_frame->sample_rate,
-                        0,
-                        NULL);
-
-				    swr_set_matrix(ms->swr, stereo_matrix, 1);
-				}
-			}
-#endif
 
 			if(swr_convert_frame(ms->swr, converted_frame, ms->audio_decode_frame)) {
 				av_frame_free(&converted_frame);
@@ -1188,11 +1159,7 @@ static int decode_thread(void *arg) {
 
 	// Compute the number of samples we need to play back.
 	if (ms->audio_duration < 0) {
-#if (LIBAVFORMAT_VERSION_MAJOR < 62)
 		if (av_fmt_ctx_get_duration_estimation_method(ctx) != AVFMT_DURATION_FROM_BITRATE) {
-#else
-		if (ctx->duration_estimation_method != AVFMT_DURATION_FROM_BITRATE) {
-#endif
 
 			long long duration = ((long long) ctx->duration) * audio_sample_rate;
 			ms->audio_duration = (unsigned int) (duration /  AV_TIME_BASE);
@@ -1352,11 +1319,7 @@ static int decode_sync_start(void *arg) {
 
 	// Compute the number of samples we need to play back.
 	if (ms->audio_duration < 0) {
-#if (LIBAVFORMAT_VERSION_MAJOR < 62)
 		if (av_fmt_ctx_get_duration_estimation_method(ctx) != AVFMT_DURATION_FROM_BITRATE) {
-#else
-		if (ctx->duration_estimation_method != AVFMT_DURATION_FROM_BITRATE) {
-#endif
 
 			long long duration = ((long long) ctx->duration) * audio_sample_rate;
 			ms->audio_duration = (unsigned int) (duration /  AV_TIME_BASE);

module/renpysound_core.c

@@ -1202,7 +1202,7 @@ void RPS_set_secondary_volume(int channel, float vol2, float delay) {
 /**
  * Replaces audio filters with the given PyObject.
  */
-void RPS_replace_audio_filter(int channel, PyObject *new_filter, int primary) {
+void RPS_replace_audio_filter(int channel, PyObject *new_filter) {
 
     struct Channel *c;
 
@@ -1214,13 +1214,10 @@ void RPS_replace_audio_filter(int channel, PyObject *new_filter, int primary) {
 
     LOCK_AUDIO();
 
-    if (primary) {
-
-        if (c->playing_audio_filter) {
-            Py_DECREF(c->playing_audio_filter);
-            Py_INCREF(new_filter);
-            c->playing_audio_filter = new_filter;
-        }
+    if (c->playing_audio_filter) {
+        Py_DECREF(c->playing_audio_filter);
+        Py_INCREF(new_filter);
+        c->playing_audio_filter = new_filter;
     }
 
     if (c->queued_audio_filter) {

module/renpysound_core.h

@@ -43,7 +43,7 @@ void RPS_set_volume(int channel, float volume);
 float RPS_get_volume(int channel);
 void RPS_set_pan(int channel, float pan, float delay);
 void RPS_set_secondary_volume(int channel, float vol2, float delay);
-void RPS_replace_audio_filter(int channel, PyObject *audio_filter, int primary);
+void RPS_replace_audio_filter(int channel, PyObject *audio_filter);
 
 int RPS_video_ready(int channel);
 PyObject *RPS_read_video(int channel);

module/setup.py

@@ -154,6 +154,10 @@ cython("renpy.style")
 
 cython("renpy.encryption")
 
+# renpy.compat
+if PY2:
+    cython("renpy.compat.dictviews")
+
 # renpy.styledata
 cython("renpy.styledata.styleclass")
 cython("renpy.styledata.stylesets")
@@ -191,10 +195,12 @@ cython(
     [ "ftsupport.c", "ttgsubtable.c" ],
     libs=sdl + [ 'freetype', 'z', 'm' ])
 
-cython(
-    "renpy.text.hbfont",
-    [ "ftsupport.c" ],
-    libs=sdl + [ 'harfbuzz', 'freetype', 'z', 'm' ])
+if not (PY2 and emscripten):
+
+    cython(
+        "renpy.text.hbfont",
+        [ "ftsupport.c" ],
+        libs=sdl + [ 'harfbuzz', 'freetype', 'z', 'm' ])
 
 generate_all_cython()
 find_unnecessary_gen()

module/setuplib.py

@@ -28,12 +28,11 @@ import os
 import sys
 import re
 import threading
-import warnings
-
-import setuptools
-
-warnings.simplefilter("ignore", category=setuptools.SetuptoolsDeprecationWarning)
 
+if sys.version_info.major == 2:
+    import distutils.core as setuptools
+else:
+    import setuptools
 
 # This flag determines if we are compiling for Android or not.
 android = "RENPY_ANDROID" in os.environ

renpy/__init__.py

@@ -253,7 +253,6 @@ name_blacklist = {
     "renpy.exports.sdl_dll",
     "renpy.sl2.slast.serial",
     "renpy.gl2.gl2draw.default_position",
-    "renpy.lexer._python_updatecache",
     }
 
 class Backup(_object):
@@ -405,9 +404,12 @@ def import_all():
     # Adds in the Ren'Py loader.
     import renpy.loader
 
-    import renpy.pyanalysis
-    sys.modules["renpy.py3analysis"] = renpy.pyanalysis
+    if not PY2:
+        import renpy.py3analysis
+    else:
+        import renpy.py2analysis
 
+    import renpy.pyanalysis
     import renpy.parameter
 
     import renpy.ast
@@ -718,6 +720,8 @@ if 1 == 0:
     from . import performance
     from . import persistent
     from . import preferences
+    from . import py2analysis
+    from . import py3analysis
     from . import pyanalysis
     from . import pydict
     from . import python

renpy/ast.py

@@ -157,7 +157,10 @@ class PyCode(object):
         if isinstance(source, PyExpr):
             loc = (source.filename, source.linenumber, source)
 
-        self.py = 3
+        if PY2:
+            self.py = 2
+        else:
+            self.py = 3
 
         # The source code.
         self.source = source

renpy/atl.py

@@ -422,7 +422,7 @@ class ATLTransformBase(renpy.object.Object):
             rv_parameters = []
             for name, value in self.context.context.items():
                 if name not in self.parameters.parameters:
-                    rv_parameters.append(ValuedParameter(name, ValuedParameter.KEYWORD_ONLY, default=value))
+                    rv_parameters.append(ValuedParameter(name, ValuedParameter.KEYWORD_ONLY, value))
             if rv_parameters:
                 rv_parameters = list(self.parameters.parameters.values()) + rv_parameters
                 rv_parameters.sort(key=lambda p: p.kind)
@@ -657,10 +657,10 @@ class ATLTransformBase(renpy.object.Object):
             #     continue
 
             if passed: # turn into elif when possible
-                param = ValuedParameter(name, param.KEYWORD_ONLY, default=scope[name])
+                param = ValuedParameter(name, param.KEYWORD_ONLY, scope[name])
 
             elif param.has_default:
-                param = ValuedParameter(name, pkind, default=scope[name])
+                param = ValuedParameter(name, pkind, scope[name])
 
             else:
                 ## not passed and no default value
@@ -1146,16 +1146,6 @@ compatible_pairs = [
 # values of the variables here.
 
 
-def check_spline_types(value):
-    if isinstance(value, (position, int, float)):
-        return True
-
-    if isinstance(value, tuple):
-        return all(check_spline_types(i) for i in value)
-
-    return False
-
-
 class RawMultipurpose(RawStatement):
 
     warp_function = None
@@ -1218,6 +1208,15 @@ class RawMultipurpose(RawStatement):
 
         compiling(self.loc)
 
+        def check_spline_types(value):
+            if isinstance(value, (position, int, float)):
+                return True
+
+            if isinstance(value, tuple):
+                return all(check_spline_types(i) for i in value)
+
+            return False
+
         # Figure out what kind of statement we have. If there's no
         # interpolator, and no properties, than we have either a
         # call, or a child statement.

renpy/audio/audio.py

@@ -230,7 +230,6 @@ class Channel(object):
 
     # The audio filter to use.
     audio_filter = None
-    raw_audio_filter = None
 
     def __init__(self, name, default_loop, stop_on_mute, tight, file_prefix, file_suffix, buffer_queue, movie, framedrop, synchro_start):
 
@@ -691,7 +690,6 @@ class Channel(object):
 
             old_raw_audio_filter = self.context.raw_audio_filter
             self.context.raw_audio_filter = audio_filter
-            self.raw_audio_filter = audio_filter
 
             if old_raw_audio_filter is None and audio_filter is None:
                 new_audio_filter = None
@@ -703,14 +701,11 @@ class Channel(object):
 
             self.context.audio_filter = new_audio_filter
 
-            for q in self.queue:
-                q.audio_filter = new_audio_filter
-
             if replace:
-                renpysound.replace_audio_filter(self.number, new_audio_filter, 1)
-            else:
-                renpysound.replace_audio_filter(self.number, new_audio_filter, 0)
+                for q in self.queue:
+                    q.audio_filter = new_audio_filter
 
+                renpysound.replace_audio_filter(self.number, new_audio_filter)
 
     def enqueue(self, filenames, loop=True, synchro_start=None, fadein=0, tight=None, loop_only=False, relative_volume=1.0):
 
@@ -720,9 +715,8 @@ class Channel(object):
         with lock:
 
             for filename in filenames:
-                if renpy.exports.is_seen_allowed():
-                    filename, _, _ = self.split_filename(filename, False)
-                    renpy.game.persistent._seen_audio[str(filename)] = True # type: ignore
+                filename, _, _ = self.split_filename(filename, False)
+                renpy.game.persistent._seen_audio[str(filename)] = True # type: ignore
 
             if not loop_only:
 
@@ -911,20 +905,17 @@ def register_channel(name,
         Ren'Py will display frames late rather than dropping them.
 
     `synchro_start`
-        Does this channel participate in synchro start? Synchro start determines if
+        Does this channel particpate in synchro start? Synchro start determines if
         the channel will start playing at the same time as other channels. If None,
-        this defaults to `loop` if `movie` is False, and False otherwise.
+        this defaults to `loop`.
     """
 
+    if synchro_start is None:
+        synchro_start = loop
+
     if name == "movie":
         movie = True
 
-    if synchro_start is None:
-        if movie:
-            synchro_start = False
-        else:
-            synchro_start = loop
-
     if not force and not renpy.game.context().init_phase and (" " not in name):
         raise Exception("Can't register channel outside of init phase.")
 
@@ -1269,7 +1260,6 @@ def interact():
 
                 ctx = c.context
 
-
                 # If we're in the same music change, then do nothing with the
                 # music.
                 if c.last_changed == ctx.last_changed:
@@ -1278,14 +1268,11 @@ def interact():
                 filenames = ctx.last_filenames
                 tight = ctx.last_tight
 
-                if ctx.raw_audio_filter != c.raw_audio_filter:
-                    c.set_audio_filter(ctx.raw_audio_filter, True)
-
                 if c.loop != filenames:
                     c.fadeout(max(renpy.config.context_fadeout_music, renpy.config.fadeout_audio))
 
                 if filenames:
-                    c.enqueue(filenames, loop=True, synchro_start=c.default_synchro_start, tight=tight, fadein=renpy.config.context_fadein_music, relative_volume=ctx.last_relative_volume)
+                    c.enqueue(filenames, loop=True, synchro_start=True, tight=tight, fadein=renpy.config.context_fadein_music, relative_volume=ctx.last_relative_volume)
 
                 c.last_changed = ctx.last_changed
 

renpy/audio/music.py

@@ -136,6 +136,7 @@ def play(filenames, channel="music", loop=None, fadeout=None, synchro_start=None
 
             t = get_serial()
             ctx.last_changed = t
+            c.last_changed = t
 
             if loop:
                 ctx.last_filenames = filenames
@@ -233,6 +234,7 @@ def queue(filenames, channel="music", loop=None, clear_queue=True, fadein=0, tig
 
             t = get_serial()
             ctx.last_changed = t
+            c.last_changed = t
 
             if loop:
                 ctx.last_filenames = filenames
@@ -305,6 +307,7 @@ def stop(channel="music", fadeout=None):
 
             t = get_serial()
             ctx.last_changed = t
+            c.last_changed = t
             ctx.last_filenames = [ ]
             ctx.last_tight = False
 
@@ -659,19 +662,11 @@ def set_audio_filter(channel, audio_filter, replace=False, duration=0.016):
         The duration to change from the current to the new filter, in seconds.
         This prevents a popping sound when changing filters.
     """
-
-    replace = replace or renpy.game.after_rollback
-
     if audio_filter is not None:
         audio_filter = renpy.audio.filter.to_audio_filter(audio_filter)
 
     try:
         c = renpy.audio.audio.get_channel(channel)
-        ctx = c.copy_context()
-
-        t = get_serial()
-        ctx.last_changed = t
-
         c.set_audio_filter(audio_filter, replace=replace, duration=duration)
     except Exception:
         if renpy.config.debug_sound:

renpy/audio/renpysound.pyx

@@ -74,7 +74,7 @@ cdef extern from "renpysound_core.h":
     float RPS_get_volume(int channel)
     void RPS_set_pan(int channel, float pan, float delay)
     void RPS_set_secondary_volume(int channel, float vol2, float delay)
-    void RPS_replace_audio_filter(int channel, object audio_filter, int primary)
+    void RPS_replace_audio_filter(int channel, object audio_filter)
 
     void RPS_advance_time()
     int RPS_video_ready(int channel)
@@ -343,19 +343,15 @@ def set_secondary_volume(channel, volume, delay):
     check_error()
 
 
-def replace_audio_filter(channel, audio_filter, playing):
+def replace_audio_filter(channel, audio_filter):
     """
     Replaces the audio filter for `channel` with `audio_filter`.
-
-    `playing`
-        If true, the filter is applied to the currently playing file and queued file. If false,
-        the filter is only applied to the queued file.
     """
 
     if audio_filter is not None:
         audio_filter.prepare(get_sample_rate())
 
-    RPS_replace_audio_filter(channel, audio_filter, playing)
+    RPS_replace_audio_filter(channel, audio_filter)
 
 
 def deallocate_audio_filter(audio_filter):

renpy/audio/webaudio.py

@@ -352,14 +352,14 @@ def set_secondary_volume(channel, volume, delay):
 
 
 @proxy_with_channel
-def replace_audio_filter(channel, audio_filter, primary):
+def replace_audio_filter(channel, audio_filter):
     """
     Replaces the audio filter for `channel` with `audio_filter`.
     """
 
     afid = load_audio_filter(audio_filter)
 
-    call("replace_audio_filter", channel, afid, primary)
+    call("replace_audio_filter", channel, afid)
 
 
 def audio_filter_constructor(f):

renpy/bootstrap.py

@@ -37,6 +37,12 @@ FSENCODING = sys.getfilesystemencoding() or "utf-8"
 old_stdout = sys.stdout
 old_stderr = sys.stderr
 
+if PY2:
+    sys_executable = sys.executable
+    reload(sys) # type: ignore
+    sys.setdefaultencoding("utf-8") # type: ignore
+    sys.executable = sys_executable
+
 def _setdefaultencoding(name):
     """
     This is install in sys to prevent games from trying to change the default
@@ -371,7 +377,10 @@ You may be using a system install of python. Please run {0}.sh,
                     if hasattr(sys, "renpy_executable"):
                         subprocess.Popen([sys.renpy_executable] + sys.argv[1:]) # type: ignore
                     else:
-                        subprocess.Popen([sys.executable] + sys.argv)
+                        if PY2:
+                            subprocess.Popen([sys.executable, "-EO"] + sys.argv)
+                        else:
+                            subprocess.Popen([sys.executable] + sys.argv)
 
             except renpy.game.ParseErrorException:
                 pass

renpy/character.py

@@ -1295,6 +1295,9 @@ class ADVCharacter(object):
 
         rv = renpy.substitutions.substitute(who)[0]
 
+        if PY2:
+            rv = rv.encode("utf-8")
+
         return rv
 
     def __format__(self, spec):

renpy/common/00action_data.rpy

@@ -111,7 +111,7 @@ init -1600 python hide:
             try:
                 return self.dict[self.key]
             except LookupError as e:
-                raise Exception("The {!r} {} does not exist.".format(key, self.kind)) from e
+                raise Exception("The {!r} {} does not exist.".format(key, self.kind)) # from e # PY3 only
 
     class ScreenVariable(Accessor):
         """

renpy/common/00action_other.rpy

@@ -544,8 +544,12 @@ init -1500 python:
             if type(self) is not type(other):
                 return False
 
-            if self.callable != other.callable:
-                return False
+            if PY2:
+                if self.callable is not other.callable:
+                    return False
+            else:
+                if self.callable != other.callable:
+                    return False
 
             if self.args != other.args:
                 return False

renpy/common/00barvalues.rpy

@@ -215,7 +215,7 @@ init -1500 python:
             try:
                 self.dict[self.key] = value
             except LookupError as e:
-                raise Exception("The {!r} {} does not exist".format(self.key, self.kind)) from e
+                raise Exception("The {!r} {} does not exist".format(self.key, self.kind)) # from e # PY3 only
 
     @renpy.pure
     class FieldValue(__GenericValue):

renpy/common/00build.rpy

@@ -63,13 +63,23 @@ init -1500 python in build:
 
     renpy_sh = "renpy.sh"
 
-    renpy_patterns = pattern_list([
-        ("renpy/**__pycache__/**.{}.pyc".format(sys.implementation.cache_tag), "all"),
-        ("renpy/**__pycache__", "all"),
-    ])
+    if PY2:
+        renpy_patterns = pattern_list([
+            ("renpy/**.pyo", "all"),
+            ("renpy/**__pycache__", None),
+        ])
 
-    if os.path.exists(os.path.join(config.renpy_base, "renpy3.sh")):
-        renpy_sh = "renpy3.sh"
+        if os.path.exists(os.path.join(config.renpy_base, "renpy2.sh")):
+            renpy_sh = "renpy2.sh"
+
+    else:
+        renpy_patterns = pattern_list([
+            ("renpy/**__pycache__/**.{}.pyc".format(sys.implementation.cache_tag), "all"),
+            ("renpy/**__pycache__", "all"),
+        ])
+
+        if os.path.exists(os.path.join(config.renpy_base, "renpy3.sh")):
+            renpy_sh = "renpy3.sh"
 
 
     # Patterns that are used to classify Ren'Py.
@@ -113,7 +123,7 @@ init -1500 python in build:
         ( "lib/*/pythonw.exe", None),
 
         # Ignore the wrong Python.
-        ( "lib/py2-*/", None),
+        ( "lib/py3-*/" if PY2 else "lib/py2-*/", None),
 
         # Windows patterns.
         ( "lib/py*-windows-i686/**", "windows_i686"),
@@ -129,7 +139,7 @@ init -1500 python in build:
         ( "lib/py*-mac-*/**", "mac"),
 
         # Old Python library.
-        ( "lib/python2.*/**", None),
+        ( "lib/python3.*/**" if PY2 else "lib/python2.*/**", None),
 
         # Shared patterns.
         ( "lib/**", "windows linux mac android ios"),
@@ -153,6 +163,7 @@ init -1500 python in build:
         ("*.dll", None),
         ("*.manifest", None),
         ("*.keystore", None),
+        ( "**.rpe", None),
         ( "**.rpe.py", None),
 
         ("update.pem", None),

renpy/common/00console.rpy

@@ -156,8 +156,12 @@ init -1500 python in _console:
                 s = s[:i] + self._ellipsis + s[len(s) - i:]
             return s
 
-        repr_bytes = _repr_bytes
-        repr_str = _repr_string
+        if PY2:
+            repr_str = _repr_bytes
+            repr_unicode = _repr_string
+        else:
+            repr_bytes = _repr_bytes
+            repr_str = _repr_string
 
         def repr_tuple(self, x, level):
             if not x: return "()"
@@ -208,7 +212,7 @@ init -1500 python in _console:
 
             if level <= 0: return "{...}"
 
-            iter_keys = self._to_shorted_list(x, self.maxdict, sort=False)
+            iter_keys = self._to_shorted_list(x, self.maxdict, sort=PY2)
             iter_x = self._make_pretty_items(x, iter_keys, '{', '}')
             return self._repr_iterable(iter_x, level, '{', '}')
 
@@ -223,7 +227,7 @@ init -1500 python in _console:
 
             if level <= 0: return left + "...})"
 
-            iter_keys = self._to_shorted_list(x, self.maxdict, sort=False)
+            iter_keys = self._to_shorted_list(x, self.maxdict, sort=PY2)
             iter_x = self._make_pretty_items(x, iter_keys, left, '})')
             return self._repr_iterable(iter_x, level, left, '})')
 

renpy/common/00defaults.rpy

@@ -135,6 +135,15 @@ init 1500 python hide:
 
     config.emphasize_audio_volume = _vol(config.emphasize_audio_volume)
 
+    if not persistent._linearized_volumes:
+        for k, v in _preferences.volumes.items():
+            _preferences.volumes[k] = v ** 2
+
+        for k, v in persistent._character_volume.items():
+            persistent._character_volume[k] = v ** 2
+
+        persistent._linearized_volumes = True
+
     _apply_default_preferences()
 
     error = _preferences.check()

renpy/common/00inputvalues.rpy

@@ -273,8 +273,8 @@ init -1510 python:
         def get_text(self):
             try:
                 return super(LocalVariableInputValue, self).get_text()
-            except LookupError as e:
-                raise Exception("The {!r} local variable does not exist.".format(self.key)) from e
+            except LookupError:
+                raise Exception("The {!r} local variable does not exist.".format(self.key)) # from e # PY3 only
 
 init -1510 python hide:
     if config.generating_documentation:

renpy/common/00steam.rpy

@@ -982,8 +982,8 @@ init -1499 python in achievement:
             if not config.enable_steam:
                 return
 
-            if "RENPY_NO_STEAM" in os.environ:
-                return
+            # if "RENPY_NO_STEAM" in os.environ:
+            #     return
 
             dll = ctypes.cdll[dll_path]
 

renpy/common/00style.rpy

@@ -173,7 +173,7 @@ init -1800:
         adjust_spacing True
         emoji_font "TwemojiCOLRv0.ttf"
         prefer_emoji True
-        shaper "harfbuzz"
+        shaper ("harfbuzz" if (not PY2) else "freetype")
 
         # Window properties
         background None

renpy/common/00sync.rpy

@@ -79,6 +79,9 @@ init 1100 python:
     else:
         config.has_sync = None
 
+    if renpy.emscripten and PY2:
+        config.has_sync = None
+
 init -1100 python in _sync:
 
     # Do not participate in saves.
@@ -146,7 +149,10 @@ init -1100 python in _sync:
         for _ in range(10000):
             hashed = hashlib.sha256(hashed).digest()
 
-        return hashed.hex()
+        if PY2:
+            return hashed.encode("hex")
+        else:
+            return hashed.hex()
 
 
     def key_and_hash(sync_id):
@@ -169,7 +175,10 @@ init -1100 python in _sync:
         for _ in range(10000):
             hashed = hashlib.sha256(hashed).digest()
 
-        return key, hashed.hex()
+        if PY2:
+            return key, hashed.encode("hex")
+        else:
+            return key, hashed.hex()
 
     def verbose_error(e):
         renpy.display.log.write("Sync error:")
@@ -251,6 +260,8 @@ init -1100 python in _sync:
 
             sd = renpy.config.save_directory
             if sd:
+                if PY2:
+                    sd = sd.encode("utf-8")
                 zf.writestr("save_directory", sd)
 
             persistent = location.path("persistent")[1]
@@ -377,7 +388,11 @@ init -1100 python in _sync:
 
                 zi = zf.getinfo(fn)
 
-                timestamp = datetime.datetime(*zi.date_time).timestamp()
+                if PY2:
+                    epoch = datetime.datetime.utcfromtimestamp(0)
+                    timestamp = (datetime.datetime(*zi.date_time) - epoch).total_seconds()
+                else:
+                    timestamp = datetime.datetime(*zi.date_time).timestamp()
 
                 data = zf.read(fn)
 

renpy/common/_audio.js

@@ -308,7 +308,6 @@ let video_start = (c) => {
 };
 
 let video_pause = (c) => {
-    const p = c.playing;
     if (p.started === null) {
         return;
     }
@@ -714,11 +713,11 @@ renpyAudio.set_pan = (channel, pan, delay) => {
     linearRampToValue(control, control.value, pan, delay);
 };
 
-renpyAudio.replace_audio_filter = (channel, afid, primary) => {
+renpyAudio.replace_audio_filter = (channel, afid) => {
     let c = get_channel(channel);
     let filter = renpyAudio.getFilter(afid);
 
-    if (c.playing && primary) {
+    if (c.playing) {
         renpyAudio.disconnectFilter(c.playing.filter, c.playing.source, c.destination);
         c.playing.filter = filter;
         renpyAudio.connectFilter(filter, c.playing.source, c.destination);

renpy/common/_developer/developer.rpym

@@ -117,20 +117,8 @@ screen _image_attributes():
 
                 for l in renpy.display.scenelists.ordered_layers:
 
-                    python:
-
-                        hidden = renpy.get_hidden_tags(l)
-                        showing = renpy.get_showing_tags(l, sort=True)
-                        transforms = renpy.layer_has_transforms(l)
-                        transform_list = [ ]
-
-                        if transforms.at_list:
-                            transform_list.append("show layer")
-                        if transforms.camera:
-                            transform_list.append("camera")
-                        if transforms.config_layer_transforms:
-                            transform_list.append("config.layer_transforms")
-
+                    $ hidden = renpy.get_hidden_tags(l)
+                    $ showing = renpy.get_showing_tags(l, sort=True)
 
                     if not hidden and not showing:
                         continue
@@ -140,9 +128,6 @@ screen _image_attributes():
 
                     text _("Layer [l]:")
 
-                    if transform_list:
-                        text _("    (transforms: [', '.join(transform_list)])")
-
                     for name in hidden:
                         $ attributes = " ".join(renpy.get_attributes(name, layer=l))
                         text _("    [name] [attributes] (hidden)")
@@ -514,7 +499,10 @@ label _filename_list:
 
                 for fn in files:
                     fn = os.path.join(dirname, fn)
+                    if PY2:
+                        fn = fn.encode("utf-8", "replace")
                     print(fn, file=f)
+                    print(fn)
 
         renpy.launch_editor([ FILES_TXT ], transient=1)
 

renpy/common/_errorhandling.rpym

@@ -96,7 +96,7 @@ init label _errorhandling:
         adjust_spacing True
         emoji_font "TwemojiCOLRv0.ttf"
         prefer_emoji True
-        shaper "harfbuzz"
+        shaper ("harfbuzz" if (not PY2) else "freetype")
 
         # Window properties
         background None
@@ -509,101 +509,50 @@ init python:
 
         return "\n".join(rv)
 
-    def __format_parse_errors(errors: list[str]):
-        """
-        Takes list of ParseError messages and transforms it
-        to string of sane length to be used as child of viewport.
-        """
-
+    def __format_parse_errors(s):
         import re
 
-        rv = []
-        rv_len = 0
-        for error in errors:
-            lines = error.split("\n")
+        rv = ""
 
-            # Do not count first line in total to include those as much as
-            # possible.
-            rv.append(re.sub(
-                r'(File "(.*)", line (\d+))',
-                r'{a=edit:\3:\2}\1{/a}',
-                lines.pop(0)))
+        lines = s.split("\n")
+        lines = __limit_lines(lines)
+        len_lines = len(lines)
 
-            print(rv)
-            print(repr(lines))
+        ln = 0
 
-            error_rv = []
-            error_len = 0
-            for line in lines:
-                # In case we have single insanely lengthy single error.
-                error_len += len(line)
-                if error_len > 65536:
-                    error_rv.clear()
-                    break
+        while ln < len_lines:
+            line = lines[ln]
+            ln += 1
 
-                if set(line.strip()) == {"^"}:
-                    idx = line.find("^")
-                    count = line.count("^", idx)
+            if ln < len_lines and lines[ln].endswith("^"):
+                highlight = len(lines[ln]) - 1
+                ln += 1
+            else:
+                highlight = -1
 
-                    prev_line = error_rv[-1]
+            pos = 0
 
-                    line = prev_line[:idx]
+            for c in line:
+                if pos == highlight:
+                    rv += u"{color=#c00}\u2192{/color}"
+                    highlight = -1
 
-                    if count == 1:
-                        if len(prev_line) <= idx + count:
-                            mark = "\u2190"
-                        else:
-                            mark = "\u2192"
-                            count = 0
-                    else:
-                        mark = prev_line[idx:idx + count]
-                    line += "{b}{color=#a00}"
-                    line += mark
-                    line += "{/color}{/b}"
-                    line += prev_line[idx + count:]
+                pos += 1
 
-                    error_rv[-1] = line
-                    continue
+                if c == "{":
+                    rv += "{{"
+                else:
+                    rv += c
 
-                error_rv.append(line)
+            if highlight > 0:
+                rv += u"{color=#c00}\u2190{/color}"
 
-            rv_len += error_len
-            if rv_len > 1048576:
-                break
+            rv += "\n"
 
-            rv.extend(error_rv)
-            rv.append("")
 
-        # while ln < len_lines:
-        #     line = lines[ln]
-        #     ln += 1
+        rv = re.sub(r'(File "(.*)", line (\d+))', r'{a=edit:\3:\2}\1{/a}', rv)
 
-        #     if ln < len_lines and "^" in lines[ln]:
-        #         highlight = len(lines[ln]) - 1
-        #         ln += 1
-        #     else:
-        #         highlight = -1
-
-        #     pos = 0
-
-        #     for c in line:
-        #         if pos == highlight:
-        #             rv += u"{color=#c00}\u2192{/color}"
-        #             highlight = -1
-
-        #         pos += 1
-
-        #         if c == "{":
-        #             rv += "{{"
-        #         else:
-        #             rv += c
-
-        #     if highlight > 0:
-        #         rv += u"{color=#c00}\u2190{/color}"
-
-        #     rv += "\n"
-
-        return "\n".join(rv)
+        return rv
 
     class _EditFile(Action):
         def __init__(self, filename, line=1):
@@ -829,7 +778,7 @@ screen _exception:
     key "console" action __EnterConsole()
 
 # The screen that is used for error handling.
-screen _parse_errors(errors, error_fn, reload_action):
+screen _parse_errors:
     modal True
     layer config.interface_layer
     zorder 1090

renpy/compat/__init__.py

@@ -58,6 +58,8 @@ Right now, it does the following things:
 
 from __future__ import division, absolute_import, with_statement, print_function, unicode_literals
 
+import future.standard_library
+import future.utils
 import builtins
 
 import io
@@ -66,15 +68,25 @@ import operator
 
 python_open = open
 
+################################################################################
+# Alias the Python 3 standard library.
+
+future.standard_library.install_aliases()
+
 ################################################################################
 # Determine if this is Python2.
 
-PY2 = False
+PY2 = future.utils.PY2
 
 ################################################################################
 # Make open mimic Python 3.
 
-open = builtins.open
+if PY2:
+    open = io.open
+    import re
+    re.Pattern = re._pattern_type # type:ignore
+else:
+    open = builtins.open
 
 
 def compat_open(*args, **kwargs):
@@ -85,57 +97,121 @@ def compat_open(*args, **kwargs):
 
 
 ################################################################################
-# Codecs.
+# Make strict use surrogateescape error handling on PY2.
 
 import codecs
 
 strict_error = codecs.lookup_error("strict")
 codecs.register_error("python_strict", strict_error)
 
+if PY2:
+    surrogateescape_error = codecs.lookup_error("surrogateescape")
+    codecs.register_error("strict", surrogateescape_error)
+
 import renpy
 renpy.update_path()
 
 ################################################################################
 # String (text and binary) types and functions.
 
-basestring = (str, bytes)
+basestring = future.utils.string_types
 pystr = str
-unicode = str
-str = builtins.str
+unicode = future.utils.text_type
 
-def bord(s): # type: (bytes) -> int
-    return s[0]
+# This tries to help pylance get the types right.
+str = builtins.str; globals()["str"] = future.utils.text_type
 
-def bchr(i): # type: (int) -> bytes
-    return bytes([i])
 
-def tobytes(s):
-    if isinstance(s, bytes):
-        return s
-    else:
-        if isinstance(s, str):
-            return s.encode('latin-1')
-        else:
-            return bytes(s)
+bord = future.utils.bord
+
+if PY2:
+    bchr = chr # type: ignore
+else:
+    def bchr(i): # type: (int) -> bytes
+        return bytes([i])
+tobytes = future.utils.tobytes
+
+from future.builtins import chr
+
+################################################################################
+# Dictionary views.
+
+# The try block solves a chicken-and-egg problem when dictviews is not
+# compiled yet, as part of the Ren'Py build process.
+
+def add_attribute(obj, name, value):
+    pass
+
+if PY2:
+    try:
+        from renpy.compat.dictviews import add_attribute
+    except ImportError:
+        print("Could not import renpy.compat.dictviews.", file=sys.stderr)
 
-chr = builtins.chr
 
 ################################################################################
 # Range.
 
-range = builtins.range
+if PY2:
+    range = xrange # type: ignore
+else:
+    range = builtins.range
 
 ################################################################################
 # Round.
 
 round = builtins.round
 
+################################################################################
+# Allow TextIOWrapper to take utf8-bytes.
+
+if PY2:
+
+    import types
+
+    # io.TextIOWrapper._write = io.TextIOWrapper.write
+
+    def text_write(self, s):
+        if isinstance(s, bytes):
+            s = s.decode("utf-8", "surrogateescape")
+        return self._write(s)
+
+    add_attribute(io.TextIOWrapper, "_write", io.TextIOWrapper.write)
+    add_attribute(io.TextIOWrapper, "write", types.MethodType(text_write, None, io.TextIOWrapper)) # type: ignore
+
+################################################################################
+# Chance the default for subprocess.Popen.
+if PY2:
+    import subprocess
+    if hasattr(subprocess, 'Popen'):  # Web2 does not have subprocess.Popen
+        class Popen(subprocess.Popen):
+            def __init__(self, *args, **kwargs):
+                if ("stdout" not in kwargs) and ("stderr" not in kwargs) and ("stdin" not in kwargs):
+                    kwargs.setdefault("close_fds", True)
+                super(Popen, self).__init__(*args, **kwargs)
+
+        subprocess.Popen = Popen
+
+################################################################################
+# Intern
+
+if PY2:
+    intern_cache = {}
+
+    def intern(s):
+        return intern_cache.setdefault(s, s)
+
+    sys.intern = intern
+
 ################################################################################
 # Export functions.
 
 __all__ = [ "PY2", "open", "basestring", "str", "pystr", "range",
             "round", "bord", "bchr", "tobytes", "chr", "unicode", ]
 
+if PY2:
+    __all__ = [ bytes(i) for i in __all__ ] # type: ignore
+
 
 # Generated by scripts/relative_imports.py, do not edit below this line.
 if 1 == 0:

renpy/compat/dictviews.pyx

@@ -0,0 +1,124 @@
+# Copyright 2004-2024 Tom Rothamel <pytom@bishoujo.us>
+#
+# Permission is hereby granted, free of charge, to any person
+# obtaining a copy of this software and associated documentation files
+# (the "Software"), to deal in the Software without restriction,
+# including without limitation the rights to use, copy, modify, merge,
+# publish, distribute, sublicense, and/or sell copies of the Software,
+# and to permit persons to whom the Software is furnished to do so,
+# subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+from cpython.object cimport PyObject
+import types
+
+# Import the Python internals. Here be dragons, but at least Python 2.7 isn't
+# going to be changing.
+
+cdef extern from "Python.h":
+    ctypedef struct PyTypeObject
+    cdef void PyType_Modified(PyTypeObject *)
+
+    ctypedef struct PyCodeObject:
+        int co_flags
+        PyObject *co_filename
+
+cdef extern from "frameobject.h":
+
+    ctypedef struct PyFrameObject:
+        PyFrameObject *f_back
+        PyCodeObject *f_code
+
+cdef extern from "Python.h":
+
+    ctypedef struct PyThreadState:
+        PyFrameObject *frame
+
+    PyThreadState *PyThreadState_Get()
+
+    enum:
+        CO_FUTURE_DIVISION
+        CO_FUTURE_WITH_STATEMENT
+
+
+###############################################################################
+# Add attributes to built in types.
+
+cdef class MappingProxy:
+    """
+    This is something that Python's built-in mapping proxy can be converted
+    to, in order to gain access to the internal dictionary so we can make
+    changes to it.
+    """
+
+    cdef dict dict
+
+
+def add_attribute(cls, name, value):
+    """
+    This adds an attribute to a Python built-in type.
+    """
+
+    cdef MappingProxy mp = <MappingProxy> <PyObject *> (cls.__dict__)
+    mp.dict[name] = value
+
+    PyType_Modified(<PyTypeObject *> cls)
+
+
+################################################################################
+# dict
+#
+# For the .items(), .keys(), and .values() methods, check to see if the calling
+# code was compiled with "from __future__ import division", or the equivalent.
+# If it was, then invoke Python 3 semantics for these methods, which means
+# returning a view into a dict. Otherwise, go with Python 2.
+#
+# Why division? It's the most Python 3-specific of the flags, at least for
+# Ren'Py purposes.
+
+add_attribute(dict, "_items", dict.items)
+add_attribute(dict, "_keys", dict.keys)
+add_attribute(dict, "_values", dict.values)
+
+cdef bint use_view():
+    """
+    Returns true if the methods should use view semantics, or false if
+    they should use legacy/list semantics.
+    """
+
+    return ((PyThreadState_Get().frame.f_code.co_flags) & (CO_FUTURE_DIVISION | CO_FUTURE_WITH_STATEMENT)) == (CO_FUTURE_DIVISION | CO_FUTURE_WITH_STATEMENT)
+
+
+def items(self):
+    if use_view():
+        return self.viewitems()
+    else:
+        return self._items()
+
+def keys(self):
+    if use_view():
+        return self.viewkeys()
+    else:
+        return self._keys()
+
+def values(self):
+    if use_view():
+        return self.viewvalues()
+    else:
+        return self._values()
+
+
+add_attribute(dict, "items", types.MethodType(items, None, dict))
+add_attribute(dict, "keys", types.MethodType(keys, None, dict))
+add_attribute(dict, "values", types.MethodType(values, None, dict))
+

renpy/compat/pickle.py

@@ -27,230 +27,260 @@ import renpy
 import pickle
 import io
 
-PROTOCOL = pickle.HIGHEST_PROTOCOL
+# Protocol 2 can be loaded on Python 2 and Python 3.
+PROTOCOL = 2
 
+if PY2:
 
-import functools
-import datetime
-import ast
-
-def make_datetime(cls, *args, **kwargs):
-    """
-    Makes a datetime.date, datetime.time, or datetime.datetime object
-    from a surrogateescaped str. This is used when unpickling a datetime
-    object that was first created in Python 2.
-    """
-
-    if (len(args) == 1) and isinstance(args[0], str):
-        data = args[0].encode("utf-8", "surrogateescape")
-        return cls.__new__(cls, data.decode("latin-1"))
-
-    return cls.__new__(cls, *args, **kwargs)
-
-class Unpickler(pickle.Unpickler):
-    date = functools.partial(make_datetime, datetime.date)
-    time = functools.partial(make_datetime, datetime.time)
-    datetime = functools.partial(make_datetime, datetime.datetime)
-
-    def find_class(self, module, name):
-        if module == "datetime":
-            if name == "date":
-                return self.date
-            elif name == "time":
-                return self.time
-            elif name == "datetime":
-                return self.datetime
-
-        if module == "_ast" and name in REWRITE_NODES:
-            return REWRITE_NODES[name]
-
-        return super().find_class(module, name)
-
-def load(f):
-    up = Unpickler(f, fix_imports=True, encoding="utf-8", errors="surrogateescape")
-    return up.load()
-
-def loads(s):
-    return load(io.BytesIO(s))
-
-def dump(o, f, highest=False):
-    pickle.dump(o, f, pickle.HIGHEST_PROTOCOL if highest else PROTOCOL)
-
-def dumps(o, highest=False):
-    return pickle.dumps(o, pickle.HIGHEST_PROTOCOL if highest else PROTOCOL)
-
-# The python AST module changed significantly between python 2 and 3. Old-style
-# screenlang support records raw python ast nodes into the rpyc data, making these
-# impossible to load normally. This dict contains mappings of nodes that need to be
-# modified to wrapper classes with a custom __setstate__ implementation that will
-# cause the pickle machinery to do the right thing.
-# There are some things that cannot be handled by this
-# (as the type of the node to be emitted is not fixed at that point), so those
-# are handled by a NodeTransformer in the ast.Module replacement.
-# Note: this isn't a complete python 2 -> python 3 ast conversion, we just convert
-# what is needed to still support old-style screens (Ren'py 6.17 and below)
-# mapping of "classname": WrapperClass
-REWRITE_NODES = {}
-
-# NodeTransformer that runs after the ast has been instantiated in the ast.Module
-# handler, and allows us to fix some more difficult issues. Currently only
-# handles converting ast.Name nodes that contain True/False/None into the appropriate
-# ast.Constant nodes.
-class AstFixupTransformer(ast.NodeTransformer):
-    def visit_Name(self, node):
-        # in python 2 True, False, None are keywords, and get parsed as such.
-        if node.id == "True":
-            alt_node = ast.Constant(True)
-
-        elif node.id == "False":
-            alt_node = ast.Constant(False)
-
-        elif node.id == "None":
-            alt_node = ast.Constant(None)
+    import cPickle # type: ignore
 
+    def load(f): # type: ignore
+        if renpy.config.use_cpickle:
+            return cPickle.load(f)
         else:
-            return node
+            return pickle.load(f)
 
-        alt_node.lineno = node.lineno
-        alt_node.col_offset = node.col_offset
-        return alt_node
+    def loads(s): # type: ignore
+        if renpy.config.use_cpickle:
+            return cPickle.loads(s)
+        else:
+            return pickle.loads(s)
 
-# wrapper classes. They all have __setstate__ defined to handle converting from the
-# py2 class, and __reduce__ implemented to convert to the underlying py3 class
-# if it gets repickled. Note that py3 ast classes will not hit the REWRITE_NODES check
-# as py3 classes report to be from "ast" instead of "_ast"
-class CallWrapper(ast.Call):
-    def __reduce__(self):
-        _, args, attrs = super().__reduce__()
-        return ast.Call, args, attrs
+    def dump(o, f, highest=False):
+        if renpy.config.use_cpickle:
+            cPickle.dump(o, f, PROTOCOL)
+        else:
+            pickle.dump(o, f,PROTOCOL)
 
-    def __setstate__(self, state):
-        # source info
-        self.lineno = state["lineno"]
-        self.col_offset = state["col_offset"]
+    def dumps(o, highest=False): # type: ignore
+        if renpy.config.use_cpickle:
+            return cPickle.dumps(o, PROTOCOL)
+        else:
+            return pickle.dumps(o, PROTOCOL)
 
-        # contents
-        self.func = state["func"]
-        self.args = state["args"]
-        self.keywords = state["keywords"]
+else:
 
-        # these gained some extra info
-        for keyword in self.keywords:
-            keyword.lineno = self.lineno
-            keyword.col_offset = self.col_offset
+    import functools
+    import datetime
+    import ast
 
-        # these are no longer an extra field, they're just part of args and keywords
-        # as you can now supply multiple of them.
-        if state["starargs"]:
-            node = ast.Starred(value=state["starargs"], ctx=ast.Load())
-            node.lineno = self.lineno
-            node.col_offset = self.col_offset
-            self.args.append(node)
+    def make_datetime(cls, *args, **kwargs):
+        """
+        Makes a datetime.date, datetime.time, or datetime.datetime object
+        from a surrogateescaped str. This is used when unpickling a datetime
+        object that was first created in Python 2.
+        """
 
-        if state["kwargs"]:
-            node = ast.keyword(None, state["kwargs"])
-            node.lineno = self.lineno
-            node.col_offset = self.col_offset
-            self.keywords.append(node)
+        if (len(args) == 1) and isinstance(args[0], str):
+            data = args[0].encode("utf-8", "surrogateescape")
+            return cls.__new__(cls, data.decode("latin-1"))
 
-REWRITE_NODES["Call"] = CallWrapper
+        return cls.__new__(cls, *args, **kwargs)
 
-class NumWrapper(ast.Constant):
-    def __reduce__(self):
-        _, args, attrs = super().__reduce__()
-        return ast.Constant, args, attrs
+    class Unpickler(pickle.Unpickler):
+        date = functools.partial(make_datetime, datetime.date)
+        time = functools.partial(make_datetime, datetime.time)
+        datetime = functools.partial(make_datetime, datetime.datetime)
 
-    def __setstate__(self, state):
-        # source info
-        self.lineno = state["lineno"]
-        self.col_offset = state["col_offset"]
+        def find_class(self, module, name):
+            if module == "datetime":
+                if name == "date":
+                    return self.date
+                elif name == "time":
+                    return self.time
+                elif name == "datetime":
+                    return self.datetime
 
-        # contents
-        self.value = state["n"]
+            if module == "_ast" and name in REWRITE_NODES:
+                return REWRITE_NODES[name]
 
-REWRITE_NODES["Num"] = NumWrapper
+            return super().find_class(module, name)
 
-class StrWrapper(ast.Constant):
-    def __reduce__(self):
-        _, args, attrs = super().__reduce__()
-        return ast.Constant, args, attrs
+    def load(f):
+        up = Unpickler(f, fix_imports=True, encoding="utf-8", errors="surrogateescape")
+        return up.load()
 
-    def __setstate__(self, state):
-        # source info
-        self.lineno = state["lineno"]
-        self.col_offset = state["col_offset"]
+    def loads(s):
+        return load(io.BytesIO(s))
 
-        # contents
-        self.value = state["s"]
+    def dump(o, f, highest=False):
+        pickle.dump(o, f, pickle.HIGHEST_PROTOCOL if highest else PROTOCOL)
 
-REWRITE_NODES["Str"] = StrWrapper
+    def dumps(o, highest=False):
+        return pickle.dumps(o, pickle.HIGHEST_PROTOCOL if highest else PROTOCOL)
 
-class ModuleWrapper(ast.Module):
-    def __reduce__(self):
-        _, args, attrs = super().__reduce__()
-        return ast.Module, args, attrs
+    # The python AST module changed significantly between python 2 and 3. Old-style
+    # screenlang support records raw python ast nodes into the rpyc data, making these
+    # impossible to load normally. This dict contains mappings of nodes that need to be
+    # modified to wrapper classes with a custom __setstate__ implementation that will
+    # cause the pickle machinery to do the right thing.
+    # There are some things that cannot be handled by this
+    # (as the type of the node to be emitted is not fixed at that point), so those
+    # are handled by a NodeTransformer in the ast.Module replacement.
+    # Note: this isn't a complete python 2 -> python 3 ast conversion, we just convert
+    # what is needed to still support old-style screens (Ren'py 6.17 and below)
+    # mapping of "classname": WrapperClass
+    REWRITE_NODES = {}
 
-    def __setstate__(self, state):
-        # contents
-        self.body = state["body"]
-        self.type_ignores = []
+    # NodeTransformer that runs after the ast has been instantiated in the ast.Module
+    # handler, and allows us to fix some more difficult issues. Currently only
+    # handles converting ast.Name nodes that contain True/False/None into the appropriate
+    # ast.Constant nodes.
+    class AstFixupTransformer(ast.NodeTransformer):
+        def visit_Name(self, node):
+            # in python 2 True, False, None are keywords, and get parsed as such.
+            if node.id == "True":
+                alt_node = ast.Constant(True)
 
-        # this is the root node, so now is a good moment do do some transforms we couldn't
-        # do earlier because we weren't sure of the node type to be created.
+            elif node.id == "False":
+                alt_node = ast.Constant(False)
 
-        transformer = AstFixupTransformer()
-        transformer.visit(self)
+            elif node.id == "None":
+                alt_node = ast.Constant(None)
 
-REWRITE_NODES["Module"] = ModuleWrapper
+            else:
+                return node
 
-class ReprWrapper(ast.Call):
-    def __reduce__(self):
-        _, args, attrs = super().__reduce__()
-        return ast.Call, args, attrs
+            alt_node.lineno = node.lineno
+            alt_node.col_offset = node.col_offset
+            return alt_node
 
-    def __setstate__(self, state):
-        # we need to transform `thing` into repr(thing)
-        # source info
-        self.lineno = state["lineno"]
-        self.col_offset = state["col_offset"]
+    # wrapper classes. They all have __setstate__ defined to handle converting from the
+    # py2 class, and __reduce__ implemented to convert to the underlying py3 class
+    # if it gets repickled. Note that py3 ast classes will not hit the REWRITE_NODES check
+    # as py3 classes report to be from "ast" instead of "_ast"
+    class CallWrapper(ast.Call):
+        def __reduce__(self):
+            _, args, attrs = super().__reduce__()
+            return ast.Call, args, attrs
 
-        # contents
-        self.func = ast.Name("repr", ast.Load(), lineno=self.lineno, col_offset=self.col_offset)
-        self.args = [state["value"]]
-        self.keywords = []
+        def __setstate__(self, state):
+            # source info
+            self.lineno = state["lineno"]
+            self.col_offset = state["col_offset"]
 
-REWRITE_NODES["Repr"] = ReprWrapper
+            # contents
+            self.func = state["func"]
+            self.args = state["args"]
+            self.keywords = state["keywords"]
 
-class ArgumentsWrapper(ast.arguments):
-    def __reduce__(self):
-        _, args, attrs = super().__reduce__()
-        return ast.arguments, args, attrs
+            # these gained some extra info
+            for keyword in self.keywords:
+                keyword.lineno = self.lineno
+                keyword.col_offset = self.col_offset
 
-    def __setstate__(self, state):
-        # source info: this node doesn't get source info
+            # these are no longer an extra field, they're just part of args and keywords
+            # as you can now supply multiple of them.
+            if state["starargs"]:
+                node = ast.Starred(value=state["starargs"], ctx=ast.Load())
+                node.lineno = self.lineno
+                node.col_offset = self.col_offset
+                self.args.append(node)
 
-        def make_arg(name):
-            # python 2 just uses bare ast.Name nodes as arguments
-            # technically it also can support more complex tuple destructuring
-            # expressions in here, but python 3 just doesn't support that,
-            # and there's really no good way of exactly handling that crazyness.
-            assert isinstance(name, ast.Name)
-            return ast.arg(name.id, lineno=name.lineno, col_offset=name.col_offset)
+            if state["kwargs"]:
+                node = ast.keyword(None, state["kwargs"])
+                node.lineno = self.lineno
+                node.col_offset = self.col_offset
+                self.keywords.append(node)
 
-        # contents. source doesn't record lineno/col_offset for vararg/kwarg
-        self.posonlyargs = []
-        self.args = [make_arg(i) for i in state["args"]]
-        self.vararg = ast.arg(state["vararg"], lineno=1, col_offset=0)
-        self.kwonlyargs = []
-        self.kw_defaults = []
-        self.kwarg = ast.arg(state["kwarg"], lineno=1, col_offset=0)
-        self.defaults =  state["defaults"]
+    REWRITE_NODES["Call"] = CallWrapper
 
-REWRITE_NODES["arguments"] = ArgumentsWrapper
+    class NumWrapper(ast.Constant):
+        def __reduce__(self):
+            _, args, attrs = super().__reduce__()
+            return ast.Constant, args, attrs
 
-class ParamWrapper(ast.Load):
-    def __reduce__(self):
-        _, args, attrs = super().__reduce__()
-        return ast.Load, args, attrs
+        def __setstate__(self, state):
+            # source info
+            self.lineno = state["lineno"]
+            self.col_offset = state["col_offset"]
 
-REWRITE_NODES["Param"] = ParamWrapper
+            # contents
+            self.value = state["n"]
+
+    REWRITE_NODES["Num"] = NumWrapper
+
+    class StrWrapper(ast.Constant):
+        def __reduce__(self):
+            _, args, attrs = super().__reduce__()
+            return ast.Constant, args, attrs
+
+        def __setstate__(self, state):
+            # source info
+            self.lineno = state["lineno"]
+            self.col_offset = state["col_offset"]
+
+            # contents
+            self.value = state["s"]
+
+    REWRITE_NODES["Str"] = StrWrapper
+
+    class ModuleWrapper(ast.Module):
+        def __reduce__(self):
+            _, args, attrs = super().__reduce__()
+            return ast.Module, args, attrs
+
+        def __setstate__(self, state):
+            # contents
+            self.body = state["body"]
+            self.type_ignores = []
+
+            # this is the root node, so now is a good moment do do some transforms we couldn't
+            # do earlier because we weren't sure of the node type to be created.
+
+            transformer = AstFixupTransformer()
+            transformer.visit(self)
+
+    REWRITE_NODES["Module"] = ModuleWrapper
+
+    class ReprWrapper(ast.Call):
+        def __reduce__(self):
+            _, args, attrs = super().__reduce__()
+            return ast.Call, args, attrs
+
+        def __setstate__(self, state):
+            # we need to transform `thing` into repr(thing)
+            # source info
+            self.lineno = state["lineno"]
+            self.col_offset = state["col_offset"]
+
+            # contents
+            self.func = ast.Name("repr", ast.Load(), lineno=self.lineno, col_offset=self.col_offset)
+            self.args = [state["value"]]
+            self.keywords = []
+
+    REWRITE_NODES["Repr"] = ReprWrapper
+
+    class ArgumentsWrapper(ast.arguments):
+        def __reduce__(self):
+            _, args, attrs = super().__reduce__()
+            return ast.arguments, args, attrs
+
+        def __setstate__(self, state):
+            # source info: this node doesn't get source info
+
+            def make_arg(name):
+                # python 2 just uses bare ast.Name nodes as arguments
+                # technically it also can support more complex tuple destructuring
+                # expressions in here, but python 3 just doesn't support that,
+                # and there's really no good way of exactly handling that crazyness.
+                assert isinstance(name, ast.Name)
+                return ast.arg(name.id, lineno=name.lineno, col_offset=name.col_offset)
+
+            # contents. source doesn't record lineno/col_offset for vararg/kwarg
+            self.posonlyargs = []
+            self.args = [make_arg(i) for i in state["args"]]
+            self.vararg = ast.arg(state["vararg"], lineno=1, col_offset=0)
+            self.kwonlyargs = []
+            self.kw_defaults = []
+            self.kwarg = ast.arg(state["kwarg"], lineno=1, col_offset=0)
+            self.defaults =  state["defaults"]
+
+    REWRITE_NODES["arguments"] = ArgumentsWrapper
+
+    class ParamWrapper(ast.Load):
+        def __reduce__(self):
+            _, args, attrs = super().__reduce__()
+            return ast.Load, args, attrs
+
+    REWRITE_NODES["Param"] = ParamWrapper

renpy/config.py

@@ -1503,8 +1503,6 @@ interface_layer = "screens"
 # Should Transform crop be limited to the width and height of the image being cropped?
 limit_transform_crop = False
 
-# Marking labels, images and audio in replays as seen is not allowed.
-no_replay_seen = False
 
 
 del os

renpy/debug.py

@@ -36,7 +36,10 @@ import builtins
 import io
 import time
 
-real_open = builtins.open
+if PY2:
+    real_open = io.open
+else:
+    real_open = builtins.open
 
 report = True
 

renpy/display/behavior.py

@@ -895,8 +895,6 @@ KEY_EVENTS = (
 
 class Button(renpy.display.layout.Window):
 
-    _store_transform_event = True
-
     keymap = { }
     action = None
     alternate = None
@@ -2190,8 +2188,6 @@ class Bar(renpy.display.displayable.Displayable):
     to clicks on that value.
     """
 
-    _store_transform_event = True
-
     @property
     def _draggable(self):
         return self.focusable
@@ -2460,10 +2456,8 @@ class Bar(renpy.display.displayable.Displayable):
 
         vertical = self.style.bar_vertical
         invert = self.style.bar_invert ^ vertical
-
         if invert:
             value = range - value
-            old_inverted_value = value
 
         grabbed = (renpy.display.focus.get_grab() is self)
         just_grabbed = False
@@ -2473,7 +2467,7 @@ class Bar(renpy.display.displayable.Displayable):
         if not grabbed and map_event(ev, "bar_activate"):
             renpy.display.tts.speak(renpy.minstore.__("activate"))
             renpy.display.focus.set_grab(self)
-            self.set_style_prefix("hover_", True)
+            self.set_style_prefix("selected_hover_", True)
             just_grabbed = True
             grabbed = True
             ignore_event = True
@@ -2533,16 +2527,12 @@ class Bar(renpy.display.displayable.Displayable):
                 value = range
 
         if invert:
-            if value == old_inverted_value: # type: ignore
-                value = old_value
-            else:
-                value = range - value
-
+            value = range - value
 
         if grabbed and not just_grabbed and map_event(ev, "bar_deactivate"):
             renpy.display.tts.speak(renpy.minstore.__("deactivate"))
-            renpy.display.focus.set_grab(None)
             self.set_style_prefix("hover_", True)
+            renpy.display.focus.set_grab(None)
 
             # Invoke rounding adjustment on bar release
             value = self.adjustment.round_value(value, release=True)
@@ -2570,10 +2560,6 @@ class Bar(renpy.display.displayable.Displayable):
 
     def set_style_prefix(self, prefix, root):
         if root:
-
-            if renpy.display.focus.get_grab() is self:
-                prefix = "selected_" + prefix
-
             super(Bar, self).set_style_prefix(prefix, root)
 
     def _tts(self):
@@ -3002,7 +2988,7 @@ class WebInput(renpy.display.displayable.Displayable):
 
     @staticmethod
     def post_find_focusable():
-        if not renpy.emscripten:
+        if PY2 or not renpy.emscripten:
             return
 
         if WebInput.active is None:

renpy/display/core.py

@@ -994,14 +994,6 @@ class Interface(object):
         if renpy.android:
             self.check_android_start()
 
-        gc.collect()
-
-        if gc.garbage:
-            del gc.garbage[:]
-
-        # Kill off the presplash.
-        renpy.display.presplash.end()
-
         # Initialize audio.
         pygame.display.hint("SDL_APP_NAME", (renpy.config.name or "Ren'Py Game").encode("utf-8"))
         pygame.display.hint("SDL_AUDIO_DEVICE_APP_NAME", (renpy.config.name or "Ren'Py Game").encode("utf-8"))
@@ -1019,8 +1011,16 @@ class Interface(object):
 
         renpy.display.emulator.init_emulator()
 
+        gc.collect()
+
+        if gc.garbage:
+            del gc.garbage[:]
+
         renpy.display.render.render_ready()
 
+        # Kill off the presplash.
+        renpy.display.presplash.end()
+
         # If we are on the web browser, start preloading the browser cache.
         if renpy.emscripten and renpy.game.preferences.web_cache_preload:
             emscripten.run_script("loadCache()")
@@ -2827,7 +2827,7 @@ class Interface(object):
                         renpy.game.preferences.fullscreen = False
 
                     if renpy.game.preferences.fullscreen != self.fullscreen:
-                        if renpy.emscripten:
+                        if (not PY2) and renpy.emscripten:
                             if renpy.game.preferences.fullscreen:
                                 emscripten.run_script("setFullscreen(true);")
                             else:
@@ -2928,14 +2928,6 @@ class Interface(object):
                     renpy.loadsave.did_autosave = False
                     renpy.exports.run(renpy.config.autosave_callback)
 
-                # End an obsolete ongoing transition.
-                if (not trans_pause) and self.ongoing_transition.get(None, None) and not self.get_ongoing_transition(None):
-                    self.transition.pop(None, None)
-                    self.ongoing_transition.pop(None, None)
-                    self.transition_time.pop(None, None)
-                    self.transition_from.pop(None, None)
-                    self.restart_interaction = True
-
                 # See if we want to restart the interaction entirely.
                 if self.restart_interaction and not self.display_reset:
                     return True, None
@@ -3243,11 +3235,6 @@ class Interface(object):
                     x = -1
                     y = -1
 
-                self.event_time = end_time = get_time()
-
-                if ev.type in input_events:
-                    self.input_event_time = self.event_time
-
                 # This can set the event to None, to ignore it.
                 ev = renpy.display.controller.event(ev)
                 if not ev:
@@ -3256,6 +3243,11 @@ class Interface(object):
                 # Handle skipping.
                 renpy.display.behavior.skipping(ev)
 
+                self.event_time = end_time = get_time()
+
+                if ev.type in input_events:
+                    self.input_event_time = self.event_time
+
                 try:
 
                     if self.touch:

renpy/display/displayable.py

@@ -505,27 +505,18 @@ class Displayable(renpy.object.Object):
 
         return pos
 
-    _store_transform_event = False
-
     def set_transform_event(self, event):
         """
         Sets the transform event of this displayable to event.
-
-        transform_event_responder needs to be set on displayables that respond to transform events.
-
-        _store_transform_event should be set on displayables that store a generated transform event,
-        like Button or Bar.
         """
 
-        if self.transform_event_responder or self._store_transform_event:
+        if event == self.transform_event:
+            return
 
-            if event == self.transform_event:
-                return
+        self.transform_event = event
 
-            self.transform_event = event
-
-            if self.transform_event_responder:
-                renpy.display.render.redraw(self, 0)
+        if self.transform_event_responder:
+            renpy.display.render.redraw(self, 0)
 
     def _handles_event(self, event):
         """

renpy/display/dragdrop.py

@@ -94,9 +94,6 @@ class Drag(renpy.display.displayable.Displayable, renpy.revertable.RevertableObj
     has been computed, the layout properties are ignored in favor of the
     position stored inside the Drag.
 
-    Transforms should not be applied to a Drag directly. Instead, apply
-    the transform to the child of the Drag.
-
     `d`
         If present, the child of this Drag. Drags use the child style
         in preference to this, if it's not None.
@@ -739,12 +736,8 @@ class Drag(renpy.display.displayable.Displayable, renpy.revertable.RevertableObj
 
     def event(self, ev, x, y, st):
 
-        rv = self.child.event(ev, x, y, st)
-        if rv is not None:
-            return rv
-
         if not self.is_focused():
-            return None
+            return self.child.event(ev, x, y, st)
 
         # Mouse, in parent-relative coordinates.
         par_x = int(self.last_x + x)

renpy/display/error.py

@@ -162,7 +162,7 @@ def report_exception(short, full, traceback_fn):
         raise
 
 
-def report_parse_errors(errors: list[str], error_fn: str) -> bool:
+def report_parse_errors(errors, error_fn):
     """
     Reports an exception to the user. Returns True if the exception should
     be raised by the normal reporting mechanisms. Otherwise, should raise
@@ -174,7 +174,7 @@ def report_parse_errors(errors: list[str], error_fn: str) -> bool:
 
     error_dump()
 
-    if renpy.game.args.command != "run":  # @UndefinedVariable
+    if renpy.game.args.command != "run": # @UndefinedVariable
         return True
 
     if "RENPY_SIMPLE_EXCEPTIONS" in os.environ:
@@ -199,7 +199,7 @@ def report_parse_errors(errors: list[str], error_fn: str) -> bool:
             reload_action=reload_action,
             errors=errors,
             error_fn=error_fn,
-        )
+            )
 
     except renpy.game.CONTROL_EXCEPTIONS:
         raise
@@ -208,5 +208,3 @@ def report_parse_errors(errors: list[str], error_fn: str) -> bool:
         renpy.display.log.write("While handling exception:")
         renpy.display.log.exception()
         raise
-
-    return False

renpy/display/focus.py

@@ -468,7 +468,6 @@ def before_interact(roots):
         set_grab(None)
         set_focused(max_default_focus, None, max_default_screen)
         old_max_default = max_default
-        explicit = True
 
     # Try to find the current focus.
     if current is not None:
@@ -779,7 +778,7 @@ def focus_nearest(from_x0, from_y0, from_x1, from_y1,
         focus_extreme(xmul, ymul, wmul, hmul)
         return
 
-    from_focus_x, from_focus_y, from_focus_w, from_focus_h = from_rect = from_focus.inset_rect()
+    from_focus_x, from_focus_y, from_focus_w, from_focus_h = from_focus.inset_rect()
 
     fx0 = from_focus_x + from_focus_w * from_x0
     fy0 = from_focus_y + from_focus_h * from_y0
@@ -807,11 +806,11 @@ def focus_nearest(from_x0, from_y0, from_x1, from_y1,
         if f.x is False:
             continue
 
-        f_x, f_y, f_w, f_h = to_rect = f.inset_rect()
-
-        if not condition(from_rect, to_rect):
+        if not condition(from_focus, f):
             continue
 
+        f_x, f_y, f_w, f_h = f.inset_rect()
+
         tx0 = f_x + f_w * to_x0
         ty0 = f_y + f_h * to_y0
         tx1 = f_x + f_w * to_x1
@@ -900,31 +899,30 @@ def key_handler(ev):
 
     else:
 
-
         if map_event(ev, 'focus_right'):
             return focus_nearest(0.9, 0.1, 0.9, 0.9,
                                  0.1, 0.1, 0.1, 0.9,
                                  verti_line_dist,
-                                 lambda old, new : old[0] + old[2] <= new[0],
+                                 lambda old, new : old.x + old.w <= new.x,
                                  -1, 0, 0, 0)
 
         if map_event(ev, 'focus_left'):
             return focus_nearest(0.1, 0.1, 0.1, 0.9,
                                  0.9, 0.1, 0.9, 0.9,
                                  verti_line_dist,
-                                 lambda old, new : new[0] + new[2] <= old[0],
+                                 lambda old, new : new.x + new.w <= old.x,
                                  1, 0, 1, 0)
 
         if map_event(ev, 'focus_up'):
             return focus_nearest(0.1, 0.1, 0.9, 0.1,
                                  0.1, 0.9, 0.9, 0.9,
                                  horiz_line_dist,
-                                 lambda old, new : new[1] + new[3] <= old[1],
+                                 lambda old, new : new.y + new.h <= old.y,
                                  0, 1, 0, 1)
 
         if map_event(ev, 'focus_down'):
             return focus_nearest(0.1, 0.9, 0.9, 0.9,
                                  0.1, 0.1, 0.9, 0.1,
                                  horiz_line_dist,
-                                 lambda old, new : old[1] + old[3] <= new[1],
+                                 lambda old, new : old.y + old.h <= new.y,
                                  0, -1, 0, 0)

renpy/display/image.py

@@ -407,14 +407,10 @@ class ImageReference(renpy.display.displayable.Displayable):
 
         if not name:
             error("Image '%s' not found." % ' '.join(self.name))
-            if renpy.game.lint:
-                renpy.lint.report("References image '%s', which does not exist.", ' '.join(self.name))
             return False
 
         if name and (self._args.name == name):
             error("Image '{}' refers to itself.".format(' '.join(name)))
-            if renpy.game.lint:
-                renpy.lint.report("Image '%s' refers to itself.", ' '.join(name))
             return False
 
         args += self._args.args

renpy/display/imagelike.py

@@ -425,7 +425,6 @@ class Frame(renpy.display.displayable.Displayable):
             return
 
         rv = Render(dw, dh)
-        rv.add_property("pixel_perfect", False)
 
         self.draw_pattern(draw, left, top, right, bottom)
 

renpy/display/layout.py

@@ -31,7 +31,11 @@ import renpy
 from renpy.display.render import render, Render
 
 
-compute_raw = renpy.display.core.absolute.compute_raw
+if PY2:
+    def compute_raw(value, room):
+        return renpy.display.core.absolute.compute_raw(value, room)
+else:
+    compute_raw = renpy.display.core.absolute.compute_raw
 
 
 def xyminimums(style, width, height):
@@ -128,6 +132,9 @@ class Container(renpy.display.displayable.Displayable):
         super(Container, self).__init__(**properties)
 
     def set_transform_event(self, event):
+        """
+        Sets the transform event of this displayable to event.
+        """
 
         super(Container, self).set_transform_event(event)
 
@@ -1120,9 +1127,6 @@ class MultiBox(Container):
                     first_line = False
                     next_padding = 0
 
-                    surf = render(d, rw, height - y, cst, cat)
-                    sw, sh = surf.get_size()
-
                 line.append((d, (next_padding, 0), x, y, surf))
                 line_height = max(line_height, sh)
                 x += sw + next_padding
@@ -1178,9 +1182,6 @@ class MultiBox(Container):
                     first_line = False
                     next_padding = 0
 
-                    surf = render(d, width - x, rh, cst, cat)
-                    sw, sh = surf.get_size()
-
                 line.append((d, (0, next_padding), x, y, surf))
                 line_width = max(line_width, sw)
                 y += sh + next_padding

renpy/display/presplash.py

@@ -206,8 +206,6 @@ def end():
     global progress_bar
     progress_bar = None
 
-    pygame_sdl2.display.quit()
-
 
 def sleep():
     """
@@ -248,12 +246,14 @@ def progress(kind, done, total):
     if done == total:
         return
 
-    if progress_kind != kind:
-        print()
-        print(kind)
-        progress_kind = kind
-        sys.stdout.flush()
+    if not PY2:
 
-    emscripten.run_script(r"""progress(%d, %d);""" % (done, total))
+        if progress_kind != kind:
+            print()
+            print(kind)
+            progress_kind = kind
+            sys.stdout.flush()
+
+        emscripten.run_script(r"""progress(%d, %d);""" % (done, total))
 
     emscripten.sleep(0)

renpy/display/render.pyx

@@ -1662,17 +1662,6 @@ cdef class Render:
         else:
             self.properties[name] = value
 
-    def get_property(self, name, default):
-
-        if self.properties is None:
-            return default
-
-        if name[:3] == "gl_":
-            name = name[3:]
-
-        return self.properties.get(name, default)
-
-
 class Canvas(object):
 
     def __init__(self, surf): #@DuplicatedSignature

renpy/display/scenelists.py

@@ -979,41 +979,6 @@ class SceneLists(renpy.object.Object):
         sl.sort(key=lambda sle : sle.zorder)
 
 
-class _HasTransforms:
-    """
-    This is returned from layer_has_transforms.
-    """
-
-    at_list : bool
-    camera: bool
-    config_layer_transforms: bool
-
-def layer_has_transforms(layer):
-    """
-    :doc: undocumented
-
-    Used to determine if a layer has transforms associated with it. Returns
-    an object with the following attributes:
-
-    at_list
-        True unless the at_list is empty.
-
-    camera
-        True unless the camera list is empty.
-
-    config_layer_transforms
-        True unless the config.layer_transforms list is empty.
-    """
-
-    rv = _HasTransforms()
-
-    rv.at_list = bool(scene_lists().layer_at_list[layer][1])
-    rv.camera = bool(scene_lists().camera_list[layer][1])
-    rv.config_layer_transforms = bool(renpy.config.layer_transforms.get(layer, [ ]))
-
-    return rv
-
-
 def scene_lists(index=-1):
     """
     Returns either the current scenelists object, or the one for the

renpy/display/transform.py

@@ -606,14 +606,6 @@ class TransformState(renpy.object.Object):
     xycenter = property(get_pos, set_xycenter)
 
 
-def simplify_position(v):
-    if isinstance(v, tuple):
-        return tuple(simplify_position(i) for i in v)
-    elif isinstance(v, position):
-        return v.simplify()
-    else:
-        return v
-
 class Proxy(object):
     """
     This class proxies a field from the transform to its state.
@@ -624,6 +616,14 @@ class Proxy(object):
 
     def __get__(self, instance, owner):
 
+        def simplify_position(v):
+            if isinstance(v, tuple):
+                return tuple(simplify_position(i) for i in v)
+            elif isinstance(v, position):
+                return v.simplify()
+            else:
+                return v
+
         return simplify_position(getattr(instance.state, self.name))
 
     def __set__(self, instance, value):

renpy/display/video.py

@@ -592,9 +592,6 @@ class Movie(renpy.display.displayable.Displayable):
         else:
             old_play = old._play
 
-        if (self._play is None) and (old_play is None):
-            return
-
         if (self._play != old_play) or renpy.config.replay_movie_sprites:
             if self._play:
 
@@ -644,39 +641,29 @@ def playing():
     return
 
 
-# A map from a channel to the movie playing on it in the last
-# interaction. Used to restart looping movies.
-last_channel_movie = { }
-
 def update_playing():
     """
     Calls play/stop on Movie displayables.
     """
 
-    global last_channel_movie
-
     old_channel_movie = renpy.game.context().movie
 
     for c, m in channel_movie.items():
 
         old = old_channel_movie.get(c, None)
-        last = last_channel_movie.get(c, None)
 
         if (c in reset_channels) and renpy.config.replay_movie_sprites:
             m.play(old)
         elif old is not m:
             m.play(old)
-        elif m.loop and last is not m:
-            m.play(last)
 
-    for c, m in last_channel_movie.items():
+    for c, m in old_channel_movie.items():
         if c not in channel_movie:
             m.stop()
 
-    renpy.game.context().movie = last_channel_movie = dict(channel_movie)
+    renpy.game.context().movie = dict(channel_movie)
     reset_channels.clear()
 
-
 def frequent():
     """
     Called to update the video playback. Returns true if a video refresh is

renpy/display/viewport.py

@@ -336,14 +336,12 @@ class Viewport(renpy.display.layout.Container):
         else:
             inside = True
 
-        # True if the player can drag the viewport.
+        # True if the player can drag the viewoport.
         draggable = self.draggable and (self.xadjustment.range or self.yadjustment.range)
 
         grab = renpy.display.focus.get_grab()
 
-        if (grab is not None) and getattr(grab, '_draggable', False) and (grab is not self):
-            self.drag_position = None
-        elif draggable:
+        if draggable:
             if grab is None and renpy.display.behavior.map_event(ev, 'viewport_drag_end'):
                 self.drag_position = None
         else:
@@ -351,22 +349,24 @@ class Viewport(renpy.display.layout.Container):
 
         if inside and draggable and (self.drag_position is not None) and (grab is not self):
 
-            if ev.type == pygame.MOUSEMOTION:
+            focused = renpy.display.focus.get_focused()
 
-                oldx, oldy = self.drag_position
+            if (focused is None) or (focused is self) or not focused._draggable:
 
-                grabbed = getattr(grab, "_draggable", False) and grab.is_focused()
+                if ev.type == pygame.MOUSEMOTION:
 
-                if math.hypot(oldx - x, oldy - y) >= renpy.config.viewport_drag_radius and not grabbed:
-                    rv = renpy.display.focus.force_focus(self)
-                    renpy.display.focus.set_grab(self)
-                    self.drag_position = (x, y)
-                    self.drag_position_time = st
-                    self.drag_speed = (0.0, 0.0)
-                    grab = self
+                    oldx, oldy = self.drag_position
 
-                    if rv is not None:
-                        return rv
+                    if math.hypot(oldx - x, oldy - y) >= renpy.config.viewport_drag_radius:
+                        rv = renpy.display.focus.force_focus(self)
+                        renpy.display.focus.set_grab(self)
+                        self.drag_position = (x, y)
+                        self.drag_position_time = st
+                        self.drag_speed = (0.0, 0.0)
+                        grab = self
+
+                        if rv is not None:
+                            return rv
 
         if renpy.display.focus.get_grab() == self:
 
@@ -538,6 +538,19 @@ class Viewport(renpy.display.layout.Container):
                 else:
                     raise renpy.display.core.IgnoreEvent()
 
+        if inside and draggable:
+
+            focused = renpy.display.focus.get_focused()
+
+            if (focused is self) or (focused is None) or (not focused._draggable):
+                if renpy.display.behavior.map_event(ev, 'viewport_drag_start'):
+                    self.drag_position = (x, y)
+                    self.drag_position_time = st
+                    self.drag_speed = (0.0, 0.0)
+
+                    self.xadjustment.end_animation(instantly=True)
+                    self.yadjustment.end_animation(instantly=True)
+
         if inside and self.edge_size and ev.type in [ pygame.MOUSEMOTION, pygame.MOUSEBUTTONDOWN, pygame.MOUSEBUTTONUP ]:
 
             def speed(n, zero, one):
@@ -571,29 +584,11 @@ class Viewport(renpy.display.layout.Container):
             else:
                 self.edge_last_st = None
 
-        ignore_event = False
-
-        if inside and draggable:
-
-            if renpy.display.behavior.map_event(ev, 'viewport_drag_start'):
-
-                self.drag_position = (x, y)
-                self.drag_position_time = st
-                self.drag_speed = (0.0, 0.0)
-
-                self.xadjustment.end_animation(instantly=True)
-                self.yadjustment.end_animation(instantly=True)
-
-                ignore_event = True
-
         rv = super(Viewport, self).event(ev, x, y, st)
 
         if rv is not None:
             return rv
 
-        if ignore_event:
-            raise renpy.display.core.IgnoreEvent()
-
         return None
 
     def set_xoffset(self, offset):

renpy/dump.py

@@ -249,8 +249,12 @@ def dump(error):
     if filename != "-":
         new = filename + ".new"
 
-        with open(new, "w") as f:
-            json.dump(result, f)
+        if PY2:
+            with open(new, "wb") as f:
+                json.dump(result, f) # type: ignore
+        else:
+            with open(new, "w") as f:
+                json.dump(result, f)
 
         if os.path.exists(filename):
             os.unlink(filename)

renpy/easy.py

@@ -34,8 +34,10 @@ import renpy
 Color = renpy.color.Color
 color = renpy.color.Color
 
-
-from collections.abc import Iterable
+if PY2:
+    from collections import Iterable # type: ignore
+else:
+    from collections.abc import Iterable
 
 
 def lookup_displayable_prefix(d):

renpy/execution.py

@@ -654,9 +654,8 @@ class Context(renpy.object.Object):
                 renpy.store._kwargs = e.kwargs
 
             if self.seen:
-                if renpy.exports.is_seen_allowed():
-                    renpy.game.persistent._seen_ever[self.current] = True # type: ignore
-                    renpy.game.seen_session[self.current] = True
+                renpy.game.persistent._seen_ever[self.current] = True # type: ignore
+                renpy.game.seen_session[self.current] = True
 
             renpy.plog(2, "    end {} ({}:{})", type_node_name, this_node.filename, this_node.linenumber)
 

renpy/exports/__init__.py

@@ -40,7 +40,7 @@ import pygame_sdl2
 try:
     import emscripten
 except ImportError:
-    emscripten = None
+    pass
 
 import renpy.audio.sound as sound
 import renpy.audio.music as music
@@ -113,10 +113,6 @@ from renpy.display.predict import (
     screen as predict_screen,
 )
 
-from renpy.display.scenelists import (
-    layer_has_transforms,
-)
-
 from renpy.display.screen import (
     ScreenProfile as profile_screen,
     current_screen,
@@ -488,7 +484,6 @@ from renpy.exports.persistentexports import (
     mark_image_unseen,
     save_persistent,
     is_seen,
-    is_seen_allowed,
 )
 
 from renpy.exports.platformexports import (

renpy/exports/displayexports.py

@@ -492,8 +492,7 @@ def show(name, at_list=[ ], layer=None, what=None, zorder=None, tag=None, behind
         img._unique()
 
     # Update the list of images we have ever seen.
-    if renpy.exports.is_seen_allowed():
-        renpy.game.persistent._seen_images[tuple(str(i) for i in name)] = True
+    renpy.game.persistent._seen_images[tuple(str(i) for i in name)] = True
 
     if tag and munge_name:
         name = (tag,) + name[1:]
@@ -1279,6 +1278,9 @@ def get_refresh_rate(precision=5):
         to 1 disables this.
     """
 
+    if PY2:
+        precision = float(precision)
+
     info = renpy.display.get_info()
     rv = info.refresh_rate # type: ignore
     rv = round(rv / precision) * precision

renpy/exports/fetchexports.py

@@ -34,11 +34,18 @@ except ImportError:
 
 import renpy
 
-from urllib.parse import urlencode as _urlencode
+if PY2:
+    from urllib import urlencode as _urlencode # type: ignore
+else:
+    from urllib.parse import urlencode as _urlencode
 
 try:
-    import urllib.request
-    proxies = urllib.request.getproxies()
+    if PY2:
+        import urllib
+        proxies = urllib.getproxies() # type: ignore
+    else:
+        import urllib.request
+        proxies = urllib.request.getproxies()
 except Exception as e:
     proxies = {}
 

renpy/exports/inputexports.py

@@ -122,7 +122,7 @@ def input(prompt, default='', allow=None, exclude='{}', length=None, with_none=N
 
     fixed = renpy.exports.in_fixed_rollback()
 
-    if renpy.emscripten and renpy.config.web_input and not fixed:
+    if (not PY2) and renpy.emscripten and renpy.config.web_input and not fixed:
         return web_input(prompt, default, allow, exclude, length, bool(mask))
 
     renpy.exports.mode('input')

renpy/exports/loaderexports.py

@@ -164,7 +164,7 @@ def fsencode(s, force=False): # type: (str, bool) -> str
     Converts s from unicode to the filesystem encoding.
     """
 
-    if not force:
+    if (not PY2) and (not force):
         return s
 
     if not isinstance(s, str):

renpy/exports/persistentexports.py

@@ -161,13 +161,3 @@ def is_seen(ever=True):
     """
 
     return renpy.game.context().seen_current(ever)
-
-
-def is_seen_allowed():
-    """
-    :doc: other
-
-    Returns False only if in a replay and no seen events allowed by setting :var:`config.no_replay_seen` to True.
-    """
-
-    return not (renpy.store._in_replay and renpy.config.no_replay_seen)

renpy/gl2/gl2draw.pyx

@@ -1356,9 +1356,6 @@ cdef class GL2DrawingContext:
 
         sx, sy = transform.transform(0, 0)
 
-        sx = round(sx, 5)
-        sy = round(sy, 5)
-
         sx = sx * halfwidth + halfwidth
         sy = sy * halfheight + halfheight
 

renpy/gl2/gl2mesh2.pyx

@@ -253,7 +253,6 @@ cdef class Mesh2(Mesh):
         double left, double top, double right, double bottom,
         double left_time, double right_time,
         double ascent, double descent,
-        double xoffset, double yoffset
         ):
         """
         Adds a glyph to a mesh created by `text_mesh`.
@@ -283,10 +282,6 @@ cdef class Mesh2(Mesh):
         cdef int stride = self.layout.stride
         cdef int attribute = self.points * stride
 
-        left -= xoffset
-        right -= xoffset
-        cx -= xoffset
-
         self.point[point + 0].x = left
         self.point[point + 0].y = bottom
         self.attribute[attribute + 0] = tex_left

renpy/gl2/gl2shader.pyx

@@ -118,7 +118,7 @@ cdef class UniformSampler2D(Uniform):
             self.cleanup = True
 
         if "anisotropic" in properties:
-            if not properties.get("anisotropic", True) and renpy.display.draw.texture_loader.max_anisotropy > 1.0:
+            if not properties.get("anisotropic", True) and renpy.display.draw.loader.max_anisotropy > 1.0:
                 glTexParameterf(GL_TEXTURE_2D, TEXTURE_MAX_ANISOTROPY_EXT, 1.0)
                 self.cleanup = True
 
@@ -141,8 +141,8 @@ cdef class UniformSampler2D(Uniform):
                 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE)
 
             if "anisotropic" in properties:
-                if not properties.get("anisotropic", True) and renpy.display.draw.texture_loader.max_anisotropy > 1.0:
-                    glTexParameterf(GL_TEXTURE_2D, TEXTURE_MAX_ANISOTROPY_EXT, renpy.display.draw.texture_loader.max_anisotropy)
+                if not properties.get("anisotropic", True) and renpy.display.draw.loader.max_anisotropy > 1.0:
+                    glTexParameterf(GL_TEXTURE_2D, TEXTURE_MAX_ANISOTROPY_EXT, renpy.display.draw.loader.max_anisotropy)
 
         self.last_data = None
 

renpy/gl2/live2d.py

@@ -63,7 +63,8 @@ def onetime_init():
     if os.path.exists(fn):
         dll = fn
 
-    dll = dll.encode("utf-8")
+    if not PY2:
+        dll = dll.encode("utf-8")
 
     if not renpy.gl2.live2dmodel.load(dll): # type: ignore
         raise Exception("Could not load Live2D. {} was not found.".format(dll))
@@ -859,11 +860,7 @@ class Live2D(renpy.display.displayable.Displayable):
         state.old_expressions = [ (name, shown, hidden) for (name, shown, hidden) in state.old_expressions if (now - hidden) < common.all_expressions[name].fadeout ]
 
         # Determine the list of expressions that are being shown by this displayable.
-        if self.used_nonexclusive is None:
-            expressions = [ ]
-        else:
-            expressions = list(self.used_nonexclusive) # type: ignore
-
+        expressions = list(self.used_nonexclusive) # type: ignore
         if self.expression:
             expressions.append(self.expression)
 

renpy/lexer.py

@@ -23,141 +23,114 @@
 from __future__ import division, absolute_import, with_statement, print_function, unicode_literals # type: ignore
 from renpy.compat import PY2, basestring, bchr, bord, chr, open, pystr, range, round, str, tobytes, unicode # *
 
-from typing import Any
 
 import codecs
 import re
 import sys
 import os
 import time
-import linecache
 import contextlib
 
 import renpy
 
 from renpy.lexersupport import match_logical_word
 
+# The filename that's in the line text cache.
+line_text_filename = ""
 
-_python_updatecache = linecache.updatecache
+# The content of the line text cache.
+line_text_cache = [ ]
 
 
-def _updatecache(filename: str, module_globals: dict[str, Any] | None = None) -> list[str]:
+def get_line_text(filename, lineno):
     """
-    Monkeypatch internal linecache.updatecache to work around RenPy python code
-    being in different files even in the same namespace. Assume filename is one
-    of '_ren.py', '.rpy' or '.rpym'.
-
-    This is needed because a lot of Python modules (namely traceback) assume
-    that the compiled python code is always comes from the python file and uses
-    linecache to get the source code, which tries to use PEP302 loader, which
-    for RenPy python code points to renpy.minstore.
+    Gets the text of a line, in a best-effort way, for debugging purposes. May
+    return just a newline, if the line doesn't exist.
     """
 
-    if filename.endswith(("_ren.py", ".rpy", ".rpym")):
-        # Assume filename also relative to basedir or renpy_base.
-        full_fn = renpy.lexer.unelide_filename(filename)
-        # If we can't find absolute path that way, assume we are
-        # in build, so we can't show source.
-        if not (os.path.isabs(full_fn) and os.path.exists(full_fn)):
-            linecache.cache.pop(filename, None)
-            return []
+    global line_text_filename
+    global line_text_cache
+
+    import linecache
+    full_filename = renpy.exports.unelide_filename(filename)
+
+
+    if full_filename != line_text_filename:
+
+        line_text_filename = full_filename
 
         try:
-            with open(full_fn, "rb") as f:
+
+            with open(full_filename, "rb") as f:
                 data = f.read().decode("utf-8", "python_strict")
 
-            if full_fn.endswith("_ren.py"):
+            if full_filename.endswith("_ren.py"):
                 data = ren_py_to_rpy(data, None)
 
             data += "\n\n"
 
-            lines = data.split("\n")
-            if lines and lines[0].startswith("\ufeff"):
-                lines[0] = lines[0][1:]
-
-            lines = [line.removesuffix("\r") + "\n" for line in lines]
-
-            stat = os.stat(full_fn)
-            linecache.cache[filename] = stat.st_size, stat.st_mtime, lines, full_fn
-            return lines
+            line_text_cache = data.split("\n")
 
         except Exception:
-            return []
+            line_text_cache = [ ]
 
-    return _python_updatecache(filename, module_globals)
+    if lineno <= len(line_text_cache):
+        return line_text_cache[lineno - 1] + "\n"
+    else:
+        return "\n"
 
 
-linecache.updatecache = _updatecache
+class ParseError(Exception):
 
+    def __init__(self, filename, number, msg, line=None, pos=None, first=False):
+        message = u"File \"%s\", line %d: %s" % (unicode_filename(filename), number, msg)
 
-class ParseError(SyntaxError):
-    """
-    Special exception type for syntax errors in Ren'Py.
-    This exception includes syntax errors of Python code, converted to
-    appropriate report style, and Ren'Py own syntax errors in user script.
-    """
+        if line:
+            if isinstance(line, list):
+                line = "".join(line)
 
-    _message: str | None = None
+            lines = line.split('\n')
 
-    def __init__(
-        self,
-        message: str,
-        filename: str,
-        lineno: int,
-        offset: int | None = None,
-        text: str | None = None,
-        end_lineno: int | None = None,
-        end_offset: int | None = None,
-    ):
-        super().__init__(message, (
-            unicode_filename(filename),
-            lineno, offset,
-            text,
-            end_lineno, end_offset))
+            if len(lines) > 1:
+                open_string = None
+                i = 0
 
-    @property
-    def message(self) -> str:
-        """
-        Fully formatted message of the error close to the result of
-        `traceback.print_exception_only`.
-        """
-        if self._message is None:
-            message = f'File "{self.filename}", line {self.lineno}: {self.msg}'
-            if self.text is not None:
-                # Neither Python nor this class does not support multiline syntax error code.
-                # Just strip the first line of provided code.
-                text = self.text.split("\n")[0]
+                while i < len(lines[0]):
+                    c = lines[0][i]
 
-                # Remove ending escape chars, so we can render it.
-                text = text.rstrip()
+                    if c == "\\":
+                        i += 1
+                    elif c == open_string:
+                        open_string = None
+                    elif open_string:
+                        pass
+                    elif c == '`' or c == '\'' or c == '"':
+                        open_string = c
 
-                # And also replace any escape chars at the start with an indent.
-                message += f'\n    {text.lstrip()}'
+                    i += 1
 
-                if self.offset is not None:
-                    offset = self.offset
+                if open_string:
+                    message += "\n(Perhaps you left out a %s at the end of the first line.)" % open_string
 
-                    # Fallback to single caret for cases end_offset is before offset.
-                    if self.end_offset is None or self.end_offset <= offset:
-                        end_offset = offset + 1
+            for l in lines:
+                message += "\n    " + l
+
+                if pos is not None:
+                    if pos <= len(l):
+                        message += "\n    " + " " * pos + "^"
+                        pos = None
                     else:
-                        end_offset = self.end_offset
+                        pos -= len(l)
 
-                    left_spaces = len(text) - len(text.lstrip())
-                    offset -= left_spaces
-                    end_offset -= left_spaces
+                if first:
+                    break
 
-                    if offset >= 1:
-                        caret_space = ' ' * (offset - 1)
-                        carets = '^' * (end_offset - offset)
-                        message += f"\n    {caret_space}{carets}"
+        self.message = message
 
-            for note in getattr(self, "__notes__", ()):
-                message += f"\n{note}"
+        Exception.__init__(self, message)
 
-            self._message = message
-
-        return self._message
+    def __unicode__(self):
+        return self.message
 
     def defer(self, queue):
         renpy.parser.deferred_parse_errors[queue].append(self.message)
@@ -383,8 +356,7 @@ def list_logical_lines(filename, filedata=None, linenumber=1, add_lines=False):
             c = data[pos]
 
             if c == u'\t':
-                raise ParseError("Tab characters are not allowed in Ren'Py scripts.",
-                                 filename, number)
+                raise ParseError(filename, number, "Tab characters are not allowed in Ren'Py scripts.")
 
             if c == u'\n' and not parendepth:
 
@@ -524,80 +496,14 @@ def list_logical_lines(filename, filedata=None, linenumber=1, add_lines=False):
             pos = end
 
             if (pos - startpos) > 65536:
-                err = ParseError(
-                    "Overly long logical line.",
-                    filename, start_number,
-                    text="".join(line))
-                err.add_note("Check strings and parenthesis.")
-                raise err
+                raise ParseError(filename, start_number, "Overly long logical line. (Check strings and parenthesis.)", line=line, first=True)
 
     if line:
-        err = ParseError("is not terminated with a newline.",
-                         filename, start_number,
-                         text="".join(line))
-        err.add_note("Check strings and parenthesis.")
-        raise err
+        raise ParseError(filename, start_number, "is not terminated with a newline. (Check strings and parenthesis.)", line=line, first=True)
 
     return rv
 
 
-def depth_split(l):
-    """
-    Returns the length of the line's prefix, and the rest of the line.
-    """
-
-    depth = 0
-    index = 0
-
-    while True:
-        if l[index] == ' ':
-            depth += 1
-            index += 1
-            continue
-
-        break
-
-    return depth, l[index:]
-
-# i, min_depth -> block, new_i
-def gll_core(lines, i, min_depth):
-    """
-    Recursively groups lines into blocks.
-
-    Given the line
-    """
-
-    rv = []
-    depth = None
-
-    while i < len(lines):
-
-        filename, number, text = lines[i]
-
-        line_depth, rest = depth_split(text)
-
-        # This catches a block exit.
-        if line_depth < min_depth:
-            break
-
-        if depth is None:
-            depth = line_depth
-
-        if depth != line_depth:
-            raise ParseError(
-                "Indentation mismatch.",
-                filename, number, text=text)
-
-        # Advance to the next line.
-        i += 1
-
-        # Try parsing a block associated with this line.
-        block, i = gll_core(lines, i, depth + 1)
-
-        rv.append((filename, number, rest, block))
-
-    return rv, i
-
 def group_logical_lines(lines):
     """
     This takes as input the list of logical line triples output from
@@ -607,16 +513,67 @@ def group_logical_lines(lines):
     no block is associated with this line.)
     """
 
+    # Returns the depth of a line, and the rest of the line.
+    def depth_split(l):
+
+        depth = 0
+        index = 0
+
+        while True:
+            if l[index] == ' ':
+                depth += 1
+                index += 1
+                continue
+
+            # if l[index] == '\t':
+            #    index += 1
+            #    depth = depth + 8 - (depth % 8)
+            #    continue
+
+            break
+
+        return depth, l[index:]
+
+    # i, min_depth -> block, new_i
+    def gll_core(i, min_depth):
+
+        rv = []
+        depth = None
+
+        while i < len(lines):
+
+            filename, number, text = lines[i]
+
+            line_depth, rest = depth_split(text)
+
+            # This catches a block exit.
+            if line_depth < min_depth:
+                break
+
+            if depth is None:
+                depth = line_depth
+
+            if depth != line_depth:
+                raise ParseError(filename, number, "Indentation mismatch.")
+
+            # Advance to the next line.
+            i += 1
+
+            # Try parsing a block associated with this line.
+            block, i = gll_core(i, depth + 1)
+
+            rv.append((filename, number, rest, block))
+
+        return rv, i
+
     if lines:
 
         filename, number, text = lines[0]
 
         if depth_split(text)[0] != 0:
-            raise ParseError(
-                "Unexpected indentation at start of file.",
-                filename, number, text=text)
+            raise ParseError(filename, number, "Unexpected indentation at start of file.")
 
-    return gll_core(lines, 0, 0)[0]
+    return gll_core(0, 0)[0]
 
 
 # A list of keywords which should not be parsed as names, because
@@ -854,35 +811,16 @@ class Lexer(object):
         except ParseError as e:
             renpy.parser.parse_errors.append(e.message)
 
-    def error(self, msg, start_pos: int | None = None, *, note: str | None = None):
+    def error(self, msg):
         """
         Convenience function for reporting a parse error at the current
         location.
-
-        If `start_pos` is passed, it should be index of character in line where
-        bad token starts, otherwise error will render single caret at current position.
-
-        If `note` is passed, it should be a string which is added to error message after
-        error line, and is often an appropriate place to add suggestion of fix.
         """
 
         if (self.line == -1) and self.block:
             self.filename, self.number, self.text, self.subblock = self.block[0]
 
-        if start_pos is None:
-            lineno = self.number
-            offset = self.pos + 1
-            end_lineno = end_offset = None
-        else:
-            lineno = end_lineno = self.number
-            offset = start_pos + 1
-            end_offset = self.pos + 1
-
-        raise ParseError(
-            msg, self.filename,
-            lineno, offset,
-            self.text,
-            end_lineno, end_offset)
+        raise ParseError(self.filename, self.number, msg, self.text, self.pos)
 
     def deferred_error(self, queue, msg):
         """
@@ -896,10 +834,7 @@ class Lexer(object):
         if (self.line == -1) and self.block:
             self.filename, self.number, self.text, self.subblock = self.block[0]
 
-        ParseError(
-            msg, self.filename,
-            self.number, self.pos + 1,
-            self.text).defer(queue)
+        ParseError(self.filename, self.number, msg, self.text, self.pos).defer(queue)
 
     def eol(self):
         """
@@ -1504,7 +1439,7 @@ class Lexer(object):
 
         return self.filename, self.number
 
-    def require(self, thing, name=None, **kwargs):
+    def require(self, thing, name=None):
         """
         Tries to parse thing, and reports an error if it cannot be done.
 
@@ -1517,14 +1452,10 @@ class Lexer(object):
             name = name or thing
             rv = self.match(thing)
         else:
-            rv = thing(**kwargs)
+            name = name or thing.__func__.__name__
+            rv = thing()
 
         if rv is None:
-            if isinstance(thing, basestring):
-                name = name or thing
-            else:
-                name = name or thing.__func__.__name__
-
             self.error("expected '%s' not found." % name)
 
         return rv
@@ -1551,20 +1482,6 @@ class Lexer(object):
         self.pos = len(self.text)
         return self.text[pos:].strip()
 
-    def _process_python_block(self, block, indent, rv, line_holder):
-        for _fn, ln, text, subblock in block:
-
-            while line_holder.line < ln:
-                rv.append(indent + '\n')
-                line_holder.line += 1
-
-            linetext = indent + text + '\n'
-
-            rv.append(linetext)
-            line_holder.line += linetext.count('\n')
-
-            self._process_python_block(subblock, indent + '    ', rv, line_holder)
-
     def python_block(self):
         """
         Returns the subblock of this code, and subblocks of that
@@ -1574,10 +1491,25 @@ class Lexer(object):
 
         rv = [ ]
 
-        line_holder = LineNumberHolder()
-        line_holder.line = self.number
+        o = LineNumberHolder()
+        o.line = self.number
 
-        self._process_python_block(self.subblock, '', rv, line_holder)
+        def process(block, indent):
+
+            for _fn, ln, text, subblock in block:
+
+                while o.line < ln:
+                    rv.append(indent + '\n')
+                    o.line += 1
+
+                linetext = indent + text + '\n'
+
+                rv.append(linetext)
+                o.line += linetext.count('\n')
+
+                process(subblock, indent + '    ')
+
+        process(self.subblock, '')
         return ''.join(rv)
 
     def arguments(self):
@@ -1645,7 +1577,7 @@ class Lexer(object):
         return sp
 
 
-def ren_py_to_rpy(text: str, filename: str | None) -> str:
+def ren_py_to_rpy(text, filename):
     """
     Transforms an _ren.py file into the equivalent .rpy file. This should retain line numbers.
 
@@ -1723,9 +1655,8 @@ def ren_py_to_rpy(text: str, filename: str | None) -> str:
             raise Exception('In {!r}, there are no """renpy blocks, so every line is ignored.'.format(filename))
 
         if state == RENPY:
-            raise Exception(
-                'In {!r}, there is a """renpy block at line {} that is not terminated by """.'.format(
-                    filename, open_linenumber))
+            raise Exception('In {!r}, there is a """renpy block at line {} that is not terminated by """.'.format(filename,
+                                                                                                                open_linenumber))
 
     rv = "\n".join(result)
 

renpy/loader.py

@@ -36,8 +36,6 @@ import io
 import unicodedata
 import time
 
-from importlib.util import spec_from_loader
-
 from pygame_sdl2.rwobject import RWopsIO
 
 from renpy.compat.pickle import loads
@@ -73,27 +71,17 @@ def get_path(fn):
 
 apks = [ ]
 game_apks = [ ]
-split_apks = [ ]
 
 if renpy.android:
     import android.apk # type: ignore
 
-    packs = [os.environ[i] for i in ["ANDROID_PACK_FF" + str(j+1) for j in range(4)] if i in os.environ and os.environ[i].endswith(".apk")]
-
     if renpy.config.renpy_base == renpy.config.basedir:
-        # Read the game data from the APKs.
+        # Read the game data from the APK.
 
         apks.append(android.apk.APK(prefix='assets/x-game/'))
-        game_apks.append(apks[-1])
-        for i in packs:
-            apks.append(android.apk.APK(apk=i, prefix='assets/game/'))
-            game_apks.append(apks[-1])
-            split_apks.append(apks[-1])
+        game_apks.append(apks[0])
 
     apks.append(android.apk.APK(prefix='assets/x-renpy/x-common/'))
-    for i in packs:
-        apks.append(android.apk.APK(apk=i, prefix='assets/renpy/common/'))
-        split_apks.append(apks[-1])
 
 
 # Files on disk should be checked before archives. Otherwise, among
@@ -372,8 +360,7 @@ def scandirfiles_from_apk(add, seen):
 
             # Strip off the "x-" in front of each filename, which is there
             # to ensure that aapt actually includes every file.
-            if apk not in split_apks:
-                f = "/".join(i[2:] for i in f.split("/"))
+            f = "/".join(i[2:] for i in f.split("/"))
 
             add(None, f, files, seen)
 
@@ -592,9 +579,7 @@ def load_from_apk(name):
     """
 
     for apk in apks:
-        prefixed_name = name
-        if apk not in split_apks:
-            prefixed_name = "/".join("x-" + i for i in name.split("/"))
+        prefixed_name = "/".join("x-" + i for i in name.split("/"))
 
         try:
             return apk.open(prefixed_name)
@@ -706,9 +691,7 @@ def loadable_core(name):
         pass
 
     for apk in apks:
-        prefixed_name = name
-        if apk not in split_apks:
-            prefixed_name = "/".join("x-" + i for i in name.split("/"))
+        prefixed_name = "/".join("x-" + i for i in name.split("/"))
         if prefixed_name in apk.info:
             loadable_cache[name] = True
             return True
@@ -835,16 +818,14 @@ class RenpyImporter(object):
 
         return None
 
-
-    def find_spec(self, fullname, path, target=None):
+    def find_module(self, fullname, path=None):
         if path is not None:
             for i in path:
                 if self.translate(fullname, i):
-                    return spec_from_loader(name=fullname, loader=RenpyImporter(i), origin=path)
+                    return RenpyImporter(i)
 
         if self.translate(fullname):
-            return spec_from_loader(name=fullname, loader=self, origin=path)
-
+            return self
 
     def load_module(self, fullname, mode="full"):
         """

renpy/loadsave.py

@@ -82,10 +82,16 @@ def save_dump(roots, log):
 
         elif isinstance(o, types.MethodType):
 
-            o_repr = "<method {0}.{1}>".format(o.__self__.__class__.__name__, o.__name__)
+            if PY2:
+                o_repr = "<method {0}.{1}>".format(o.__self__.__class__.__name__, o.__func__.__name__) # type: ignore
+            else:
+                o_repr = "<method {0}.{1}>".format(o.__self__.__class__.__name__, o.__name__)
 
         elif isinstance(o, types.FunctionType):
-            name = o.__qualname__ or o.__name__
+            if PY2:
+                name = o.__name__
+            else:
+                name = o.__qualname__ or o.__name__
 
             o_repr = o.__module__ + '.' + name
 

renpy/memory.py

@@ -121,7 +121,10 @@ def cycle_finder(o, name):
             o_repr = "<" + o.__class__.__name__ + ">"
 
         elif isinstance(o, types.MethodType):
-            o_repr = "<method {0}.{1}>".format(o.__self__.__class__.__name__, o.__name__)
+            if PY2:
+                o_repr = "<method {0}.{1}>".format(o.__self__.__class__.__name__, o.__func__.__name__) # type: ignore
+            else:
+                o_repr = "<method {0}.{1}>".format(o.__self__.__class__.__name__, o.__name__)
 
         elif isinstance(o, object):
             o_repr = "<{0}>".format(type(o).__name__)

renpy/minstore.py

@@ -201,3 +201,6 @@ __all__ = [
     'ui',
     'unicode',
 ]
+
+if PY2:
+    __all__ = [ bytes(i) for i in __all__ ] # type: ignore

renpy/object.py

@@ -26,8 +26,12 @@ from renpy.compat import PY2, basestring, bchr, bord, chr, open, pystr, range, r
 
 
 # Allow pickling NoneType.
-import builtins
-builtins.NoneType = type(None) # type: ignore
+if PY2:
+    import __builtin__ # type: ignore
+    __builtin__.NoneType = type(None)
+else:
+    import builtins
+    builtins.NoneType = type(None) # type: ignore
 
 
 class Object(object):

renpy/parameter.py

@@ -24,6 +24,7 @@ from __future__ import division, absolute_import, with_statement, print_function
 from renpy.compat import PY2, basestring, bchr, bord, chr, open, pystr, range, round, str, tobytes, unicode # *
 
 from itertools import chain as _chain
+import collections
 
 import renpy
 
@@ -42,7 +43,8 @@ class Parameter(object):
 
     empty = None
 
-    def __init__(self, name, kind, *, default=empty):
+    def __init__(self, name, kind, default=empty):
+        # default should only be passed by keyword, todo when PY3-only
         self.name = name
         self.kind = kind
         self.default = default
@@ -88,9 +90,10 @@ class ValuedParameter(Parameter):
 
     class empty: pass # singleton, should be picklable
 
-    def __init__(self, name, kind, *, default=empty):
+    def __init__(self, name, kind, default=empty):
+        # default should only be passed by keyword, todo when PY3-only
         # this method is redefined in order to change default's default value
-        super(ValuedParameter, self).__init__(name, kind, default=default)
+        super(ValuedParameter, self).__init__(name, kind, default)
 
     def default_value(self, *args, **kwargs):
         return self.default
@@ -120,7 +123,8 @@ class Signature(object):
         if parameters is None:
             self.parameters = {}
         else:
-            self.parameters = {param.name: param for param in parameters}
+            # when in PY3-only, turn this into MappingProxyType o dict
+            self.parameters = collections.OrderedDict((param.name, param) for param in parameters)
 
     @staticmethod
     def legacy_params(parameters, positional, extrapos, extrakw, last_posonly=None, first_kwonly=None):
@@ -237,6 +241,8 @@ class Signature(object):
         - applies the defaults automatically (and lazily, as per the above)
         """
 
+        # when in PY3-only, uncomment the "from None" in the raise statements
+
         if not renpy.config.developer:
             ignore_errors = True
 
@@ -244,7 +250,7 @@ class Signature(object):
 
         def _raise(exct, msg, *argz, **kwargz):
             if not ignore_errors:
-                raise exct(msg.format(*argz, **kwargz)) from None
+                raise exct(msg.format(*argz, **kwargz)) # from None
 
         # code mostly taken from stdlib's inspect.Signature._bind
         arguments = {}
@@ -298,7 +304,7 @@ class Signature(object):
                                 argtype = ''
                             msg = 'missing a required{argtype} argument: {arg!r}'
                             msg = msg.format(arg=param.name, argtype=argtype)
-                            raise TypeError(msg) from None
+                            raise TypeError(msg) # from None
             else:
                 # We have a positional argument to process
                 try:
@@ -356,7 +362,7 @@ class Signature(object):
                 if (not (partial or ignore_errors) and param.kind != param.VAR_POSITIONAL and
                                                     param.default is param.empty):
                     raise TypeError('missing a required argument: {arg!r}'. \
-                                    format(arg=param_name)) from None
+                                    format(arg=param_name)) # from None
 
             else:
                 if param.kind == param.POSITIONAL_ONLY:
@@ -452,8 +458,8 @@ def apply_arguments(parameters, args, kwargs, ignore_errors=False):
 class ArgumentInfo(renpy.object.Object):
 
     __version__ = 1
-    starred_indexes = frozenset()
-    doublestarred_indexes = frozenset()
+    starred_indexes = set()
+    doublestarred_indexes = set()
 
     def after_upgrade(self, version):
         if version < 1:
@@ -462,13 +468,16 @@ class ArgumentInfo(renpy.object.Object):
             extrakw = self.extrakw # type: ignore
             length = len(arguments) + bool(extrapos) + bool(extrakw)
             if extrapos:
-                self.starred_indexes = { length - 1 - bool(extrakw) }
+                self.starred_indexes = { length - 1 }
                 arguments.append((None, extrapos))
 
             if extrakw:
                 self.doublestarred_indexes = { length - 1 }
                 arguments.append((None, extrakw))
 
+            if extrapos and extrakw:
+                self.starred_indexes = { length - 2 }
+
     def __init__(self, arguments, starred_indexes=None, doublestarred_indexes=None):
 
         # A list of (keyword, expression) pairs.
@@ -476,12 +485,10 @@ class ArgumentInfo(renpy.object.Object):
         self.arguments = arguments
 
         # Indexes of arguments to be considered as * unpacking
-        if starred_indexes is not None:
-            self.starred_indexes = starred_indexes
+        self.starred_indexes = starred_indexes or set()
 
         # Indexes of arguments to be considered as ** unpacking.
-        if doublestarred_indexes is not None:
-            self.doublestarred_indexes = doublestarred_indexes
+        self.doublestarred_indexes = doublestarred_indexes or set()
 
     def evaluate(self, scope=None):
         """
@@ -534,4 +541,4 @@ class ArgumentInfo(renpy.object.Object):
 
 
 EMPTY_PARAMETERS = Signature()
-EMPTY_ARGUMENTS = ArgumentInfo((), None, None)
+EMPTY_ARGUMENTS = ArgumentInfo([ ], None, None)

renpy/parser.py

@@ -31,7 +31,7 @@ import time
 import renpy
 import renpy.ast as ast
 
-from renpy.parameter import EMPTY_ARGUMENTS, Parameter
+from renpy.parameter import Parameter
 
 from renpy.lexer import (
     list_logical_lines,
@@ -43,6 +43,7 @@ from renpy.lexer import (
     munge_filename,
     elide_filename,
     unelide_filename,
+    get_line_text,
     SubParse,
 )
 
@@ -97,20 +98,20 @@ def parse_image_name(l, string=False, nodash=False):
     return tuple(rv)
 
 
-def parse_simple_expression_list(l, image=False):
+def parse_simple_expression_list(l):
     """
     This parses a comma-separated list of simple_expressions, and
     returns a list of strings. It requires at least one
     simple_expression be present.
     """
 
-    rv = [ l.require(l.simple_expression, image=image) ]
+    rv = [ l.require(l.simple_expression) ]
 
     while True:
         if not l.match(','):
             break
 
-        e = l.simple_expression(image=image)
+        e = l.simple_expression()
 
         if not e:
             break
@@ -132,7 +133,7 @@ def parse_image_specifier(l):
     behind = [ ]
 
     if l.keyword("expression") or l.keyword("image"):
-        expression = l.require(l.simple_expression, image=True)
+        expression = l.require(l.simple_expression)
         image_name = (expression.strip(),)
     else:
         image_name = parse_image_name(l, True)
@@ -153,7 +154,7 @@ def parse_image_specifier(l):
             if at_list:
                 l.error("multiple at clauses are prohibited.")
             else:
-                at_list = parse_simple_expression_list(l, image=True)
+                at_list = parse_simple_expression_list(l)
 
             continue
 
@@ -171,7 +172,7 @@ def parse_image_specifier(l):
             if zorder is not None:
                 l.error("multiple zorder clauses are prohibited.")
             else:
-                zorder = l.require(l.simple_expression, image=True)
+                zorder = l.require(l.simple_expression)
 
             continue
 
@@ -468,9 +469,6 @@ def parse_arguments(l):
     if not l.match(r'\('):
         return None
 
-    if l.match(r'\)'):
-        return EMPTY_ARGUMENTS
-
     arguments = [ ]
     starred_indexes = set()
     doublestarred_indexes = set()
@@ -1762,10 +1760,8 @@ def report_parse_errors():
     full_text = ""
 
     f, error_fn = renpy.error.open_error_file("errors.txt", "w")
-
-    screen_parse_errors = []
     with f:
-        f.write("\ufeff")  # BOM
+        f.write("\ufeff") # BOM
 
         print("I'm sorry, but errors were detected in your script. Please correct the", file=f)
         print("errors listed below, and try again.", file=f)
@@ -1782,8 +1778,6 @@ def report_parse_errors():
             print("", file=f)
             print(i, file=f)
 
-            screen_parse_errors.append(i)
-
             try:
                 print("")
                 print(i)
@@ -1794,7 +1788,7 @@ def report_parse_errors():
         print("Ren'Py Version:", renpy.version, file=f)
         print(str(time.ctime()), file=f)
 
-    renpy.display.error.report_parse_errors(screen_parse_errors, error_fn)
+    renpy.display.error.report_parse_errors(full_text, error_fn)
 
     try:
         if renpy.game.args.command == "run" or renpy.game.args.errors_in_editor: # type: ignore

renpy/persistent.py

@@ -233,7 +233,7 @@ def init():
     disk, so that we can configure the savelocation system.
     """
 
-    if renpy.config.early_developer:
+    if renpy.config.early_developer and not PY2:
         init_debug_pickler()
 
     filename = os.path.join(renpy.config.savedir, "persistent.new") # type: ignore

renpy/py2analysis.py

@@ -0,0 +1,789 @@
+# Copyright 2004-2024 Tom Rothamel <pytom@bishoujo.us>
+#
+# Permission is hereby granted, free of charge, to any person
+# obtaining a copy of this software and associated documentation files
+# (the "Software"), to deal in the Software without restriction,
+# including without limitation the rights to use, copy, modify, merge,
+# publish, distribute, sublicense, and/or sell copies of the Software,
+# and to permit persons to whom the Software is furnished to do so,
+# subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+from __future__ import division, absolute_import, with_statement, print_function, unicode_literals
+from renpy.compat import PY2, basestring, bchr, bord, chr, open, pystr, range, round, str, tobytes, unicode # *
+
+
+
+import renpy # @UnusedImport
+from renpy.python import py_compile
+
+# Import the Python AST module, instead of the Ren'Py ast module.
+import ast
+
+import zlib
+
+from renpy.compat.pickle import loads, dumps
+
+# The set of names that should be treated as constants.
+always_constants = { 'True', 'False', 'None' }
+
+# The set of names that should be treated as pure functions.
+pure_functions = {
+    # Python 2 builtins.
+    "abs", "all", "any", "apply", "bin", "bool", "bytes", "callable", "chr",
+    "cmp", "dict", "divmod",
+    "filter", "float", "frozenset",
+    "getattr", "globals", "hasattr", "hash", "hex", "int", "isinstance",
+    "len", "list", "long", "map", "max", "min", "oct", "ord", "pow",
+    "range", "reduce", "repr", "round", "set", "sorted",
+    "str", "sum", "tuple", "unichr", "unicode", "vars", "zip",
+
+    # minstore.py
+    "_",
+    "_p",
+    "absolute",
+    "position",
+    "__renpy__list__",
+    "__renpy__dict__",
+    "__renpy__set__",
+
+    # defaultstore.py
+    "ImageReference", "Image", "Frame", "Solid", "LiveComposite", "LiveCrop",
+    "LiveTile", "Flatten", "Null", "Window", "Viewport", "DynamicDisplayable",
+    "ConditionSwitch", "ShowingSwitch", "Transform", "Animation", "Movie",
+    "Particles", "SnowBlossom", "Text", "ParameterizedText", "FontGroup",
+    "Drag", "Alpha", "AlphaMask", "Position", "Pan", "Move", "Motion", "Revolve", "Zoom",
+    "RotoZoom", "FactorZoom", "SizeZoom", "Fade", "Dissolve", "ImageDissolve",
+    "AlphaDissolve", "CropMove", "PushMove", "Pixellate", "OldMoveTransition",
+    "MoveTransition", "MoveFactory", "MoveIn", "MoveOut", "ZoomInOut",
+    "RevolveInOut", "MultipleTransition", "ComposeTransition", "Pause",
+    "SubTransition", "ADVSpeaker", "ADVCharacter", "Speaker", "Character",
+    "DynamicCharacter", "Fixed", "HBox", "VBox", "Grid", "AlphaBlend", "At",
+    "color", "Color",
+
+    # ui.py
+
+    "ui.returns",
+    "ui.jumps",
+    "ui.jumpsoutofcontext",
+    "ui.callsinnewcontext",
+    "ui.invokesinnewcontext",
+    "ui.gamemenus",
+
+    # renpy.py
+
+    "renpy.version_string",
+    "renpy.version_only",
+    "renpy.version_tuple",
+    "renpy.version_name",
+    "renpy.license",
+}
+
+constants = { "config", "style" } | always_constants | pure_functions
+
+# A set of names that should not be treated as global constants.
+not_constants = set()
+
+# The base set for the local constants.
+local_constants = set()
+
+
+def const(name):
+    """
+    :doc: const
+
+    Declares a variable in the store to be constant.
+
+    A variable is constant if nothing can change its value, or any value
+    reached by indexing it or accessing its attributes. Variables must
+    remain constant out of define, init, and translate python blocks.
+
+    `name`
+        A string giving the name of the variable to declare constant.
+    """
+
+    if name not in not_constants:
+        constants.add(name)
+
+
+def not_const(name):
+    """
+    :doc: const
+
+    Declares a name in the store to be not constant.
+
+    This undoes the effect of calls to :func:`renpy.const` and
+    :func:`renpy.pure`.
+
+    `name`
+        The name to declare not constant.
+    """
+
+    constants.discard(name)
+    pure_functions.discard(name)
+    not_constants.add(name)
+
+
+def pure(fn):
+    """
+    :doc: const
+
+    Declares a function as pure. A pure function must always return the
+    same value when it is called with the same arguments, outside of
+    define, init, and translate python blocks.
+
+    `fn`
+        The name of the function to declare pure. This may either be a string
+        containing the name of the function, or the function itself.
+        If a string is passed and the function is inside the module,
+        this string should contain the module name with the dot.
+
+    Returns `fn`, allowing this function to be used as a decorator.
+    """
+
+    name = fn
+
+    if not isinstance(name, basestring):
+        name = fn.__name__
+
+        module = fn.__module__
+        name = module + "." + name
+
+    if name.startswith("store."):
+        name = name[6:]
+
+    if name not in not_constants:
+        pure_functions.add(name)
+        constants.add(name)
+
+    return fn
+
+
+class Control(object):
+    """
+    Represents control flow.
+
+    `const`
+        True if this statement always executes.
+
+    `loop`
+        True if this corresponds to a loop.
+
+    `imagemap`
+        True if this control is in a non-constant imagemap.
+    """
+
+    def __init__(self, const, loop, imagemap):
+        self.const = const
+        self.loop = loop
+        self.imagemap = imagemap
+
+
+# Three levels of constness.
+GLOBAL_CONST = 2 # Expressions that are const everywhere.
+LOCAL_CONST = 1 # Expressions that are const with regard to a screen + parameters.
+NOT_CONST = 0 # Expressions that are not const.
+
+
+class DeltaSet(object):
+
+    def __init__(self, base, copy=None):
+        """
+        Represents a set that stores its contents as differences from a base
+        set.
+        """
+
+        self.base = base
+
+        if copy is not None:
+            self.added = set(copy.added)
+            self.removed = set(copy.removed)
+        else:
+            self.added = set()
+            self.removed = set()
+
+        self.changed = False
+
+    def add(self, v):
+
+        if v in self.removed:
+            self.removed.discard(v)
+            self.changed = True
+        elif v not in self.base and v not in self.added:
+            self.added.add(v)
+            self.changed = True
+
+    def discard(self, v):
+
+        if v in self.added:
+            self.added.discard(v)
+            self.changed = True
+        elif v in self.base and v not in self.removed:
+            self.removed.add(v)
+            self.changed = True
+
+    def __contains__(self, v):
+        return (v in self.added) or ((v in self.base) and (v not in self.removed))
+
+    def copy(self):
+        return DeltaSet(self.base, self)
+
+    def __iter__(self):
+
+        for i in self.base:
+            if i not in self.removed:
+                yield i
+
+        for i in self.added:
+            yield i
+
+
+class Analysis(object):
+    """
+    Represents the result of code analysis, and provides tools to perform
+    code analysis.
+    """
+
+    def __init__(self, parent=None):
+
+        # The parent context transcludes run in, or None if there is no parent
+        # context.
+        self.parent = parent
+
+        # Analyses of children, such a screens we use.
+        self.children = { }
+
+        # The variables we consider to be not-constant.
+        self.not_constant = DeltaSet(not_constants)
+
+        # Variables we consider to be locally constant.
+        self.local_constant = DeltaSet(local_constants)
+
+        # Variables we consider to be globally constant.
+        self.global_constant = DeltaSet(always_constants)
+
+        # The functions we consider to be pure.
+        self.pure_functions = DeltaSet(pure_functions)
+
+        # Represents what we know about the current control.
+        self.control = Control(True, False, False)
+
+        # The stack of const_flow values.
+        self.control_stack = [ self.control ]
+
+    def get_child(self, identifier):
+        if identifier in self.children:
+            return self.children[identifier]
+
+        rv = Analysis(self)
+        self.children[identifier] = rv
+
+        return rv
+
+    def push_control(self, const=True, loop=False, imagemap=False):
+        self.control = Control(self.control.const and const, loop, self.control.imagemap or imagemap)
+        self.control_stack.append(self.control) # type: ignore
+
+    def pop_control(self):
+        rv = self.control_stack.pop()
+        self.control = self.control_stack[-1]
+        return rv
+
+    def imagemap(self):
+        """
+        Returns NOT_CONST if we're in a non-constant imagemap.
+        """
+
+        if self.control.imagemap:
+            return NOT_CONST
+        else:
+            return GLOBAL_CONST
+
+    def exit_loop(self):
+        """
+        Call this to indicate the current loop is being exited by the
+        continue or break statements.
+        """
+
+        l = list(self.control_stack)
+        l.reverse()
+
+        for i in l:
+            i.const = False
+
+            if i.loop:
+                break
+
+    def at_fixed_point(self):
+        """
+        Returns True if we've reached a fixed point, where the analysis has
+        not changed since the last time we called this function.
+        """
+
+        for i in self.children.values():
+            if not i.at_fixed_point():
+                return False
+
+        if (self.not_constant.changed or
+                self.global_constant.changed or
+                self.local_constant.changed or
+                self.pure_functions.changed):
+
+            self.not_constant.changed = False
+            self.global_constant.changed = False
+            self.local_constant.changed = False
+            self.pure_functions.changed = False
+
+            return False
+
+        return True
+
+    def mark_constant(self, name):
+        """
+        Marks `name` as a potential local constant.
+        """
+
+        if not name in self.not_constant:
+            self.local_constant.add(name)
+            self.global_constant.discard(name)
+            self.pure_functions.discard(name)
+
+    def mark_not_constant(self, name):
+        """
+        Marks `name` as definitely not-constant.
+        """
+
+        self.not_constant.add(name)
+
+        self.pure_functions.discard(name)
+        self.local_constant.discard(name)
+        self.global_constant.discard(name)
+
+    def is_constant(self, node):
+        """
+        Returns true if `node` is constant for the purpose of screen
+        language. Node should be a python AST node.
+
+        Screen language ignores object identity for the purposes of
+        object equality.
+        """
+
+        def check_slice(slice): # @ReservedAssignment
+
+            if isinstance(slice, ast.Index):
+                return check_node(slice.value) # type: ignore
+
+            elif isinstance(slice, ast.Slice):
+                consts = [ ]
+
+                if slice.lower:
+                    consts.append(check_node(slice.lower))
+                if slice.upper:
+                    consts.append(check_node(slice.upper))
+                if slice.step:
+                    consts.append(check_node(slice.step))
+
+                if not consts:
+                    return GLOBAL_CONST
+                else:
+                    return min(consts)
+
+            return NOT_CONST
+
+        def check_name(node):
+            """
+            Check nodes that make up a name. This returns a pair:
+
+            * The first element is True if the node is constant, and False
+              otherwise.
+            * The second element is None if the node is constant or the name is
+              not known, and the name otherwise.
+            """
+
+            if isinstance(node, ast.Name):
+                const = NOT_CONST
+                name = node.id
+
+            elif isinstance(node, ast.Attribute):
+                const, name = check_name(node.value)
+
+                if name is not None:
+                    name = name + "." + node.attr
+
+            else:
+                return check_node(node), None
+
+            if name in self.not_constant:
+                return NOT_CONST, name
+            elif name in self.global_constant:
+                return GLOBAL_CONST, name
+            elif name in self.local_constant:
+                return LOCAL_CONST, name
+            else:
+                return const, name
+
+        def check_nodes(nodes):
+            """
+            Checks a list of nodes for constness.
+            """
+
+            nodes = list(nodes)
+
+            if not nodes:
+                return GLOBAL_CONST
+
+            return min(check_node(i) for i in nodes)
+
+        def check_node(node):
+            """
+            Returns true if the ast node `node` is constant.
+            """
+
+            # This handles children that do not exist.
+            if node is None:
+                return GLOBAL_CONST
+
+            # PY3: see if there are new node types.
+
+            if isinstance(node, (ast.Num, ast.Str)):
+                return GLOBAL_CONST
+
+            elif isinstance(node, (ast.List, ast.Tuple)):
+                return check_nodes(node.elts)
+
+            elif isinstance(node, (ast.Attribute, ast.Name)):
+                return check_name(node)[0]
+
+            elif isinstance(node, ast.BoolOp):
+                return check_nodes(node.values)
+
+            elif isinstance(node, ast.BinOp):
+                return min(
+                    check_node(node.left),
+                    check_node(node.right),
+                    )
+
+            elif isinstance(node, ast.UnaryOp):
+                return check_node(node.operand)
+
+            elif isinstance(node, ast.Call):
+                const, name = check_name(node.func)
+
+                # The function must have a name, and must be declared pure.
+                if (const != GLOBAL_CONST) or (name not in self.pure_functions):
+                    return NOT_CONST
+
+                consts = [ ]
+
+                # Arguments and keyword arguments must be pure.
+                consts.append(check_nodes(node.args))
+                consts.append(check_nodes(i.value for i in node.keywords))
+
+                if node.starargs is not None: # type: ignore
+                    consts.append(check_node(node.starargs)) # type: ignore
+
+                if node.kwargs is not None: # type: ignore
+                    consts.append(check_node(node.kwargs)) # type: ignore
+
+                return min(consts)
+
+            elif isinstance(node, ast.IfExp):
+                return min(
+                    check_node(node.test),
+                    check_node(node.body),
+                    check_node(node.orelse),
+                    )
+
+            elif isinstance(node, ast.Dict):
+                return min(
+                    check_nodes(node.keys),
+                    check_nodes(node.values)
+                    )
+
+            elif isinstance(node, ast.Set):
+                return check_nodes(node.elts)
+
+            elif isinstance(node, ast.Compare):
+                return min(
+                    check_node(node.left),
+                    check_nodes(node.comparators),
+                    )
+
+            elif isinstance(node, ast.Repr): # type: ignore
+                return check_node(node.value)
+
+            elif isinstance(node, ast.Subscript):
+                return min(
+                    check_node(node.value),
+                    check_slice(node.slice),
+                    )
+
+            return NOT_CONST
+
+        return check_node(node)
+
+    def is_constant_expr(self, expr):
+        """
+        Compiles `expr` into an AST node, then returns the result of
+        self.is_constant called on that node.
+        """
+
+        node, literal = ccache.ast_eval_literal(expr)
+
+        if literal:
+            return GLOBAL_CONST
+        else:
+            return self.is_constant(node)
+
+    def python(self, code):
+        """
+        Performs analysis on a block of python code.
+        """
+
+        nodes = ccache.ast_exec(code)
+
+        a = PyAnalysis(self)
+
+        for i in nodes:
+            a.visit(i)
+
+    def parameters(self, parameters):
+        """
+        Analyzes the parameters to the screen.
+        """
+
+        self.global_constant = DeltaSet(constants)
+
+        # As we have parameters, analyze with those parameters.
+
+        for name in parameters.parameters:
+            self.mark_not_constant(name)
+
+
+class PyAnalysis(ast.NodeVisitor):
+    """
+    This analyzes Python code to determine which variables should be
+    marked const, and which should be marked non-const.
+    """
+
+    def __init__(self, analysis):
+
+        self.analysis = analysis
+
+    def visit_Name(self, node):
+
+        if isinstance(node.ctx, ast.AugStore):
+            self.analysis.mark_not_constant(node.id)
+
+        elif isinstance(node.ctx, ast.Store):
+            if self.analysis.control.const:
+                self.analysis.mark_constant(node.id)
+            else:
+                self.analysis.mark_not_constant(node.id)
+
+    def visit_Assign(self, node):
+
+        const = self.analysis.is_constant(node.value)
+        self.analysis.push_control(const, False)
+
+        self.generic_visit(node)
+
+        self.analysis.pop_control()
+
+    def visit_AugAssign(self, node):
+
+        self.analysis.push_control(False, False)
+
+        self.generic_visit(node)
+
+        self.analysis.pop_control()
+
+    def visit_For(self, node):
+
+        const = self.analysis.is_constant(node.iter)
+
+        self.analysis.push_control(const=const, loop=True)
+        old_const = self.analysis.control.const
+
+        self.generic_visit(node)
+
+        if self.analysis.control.const != old_const:
+            self.generic_visit(node)
+
+        self.analysis.pop_control()
+
+    def visit_While(self, node):
+
+        const = self.analysis.is_constant(node.test)
+
+        self.analysis.push_control(const=const, loop=True)
+        old_const = self.analysis.control.const
+
+        self.generic_visit(node)
+
+        if self.analysis.control.const != old_const:
+            self.generic_visit(node)
+
+        self.analysis.pop_control()
+
+    def visit_If(self, node):
+        const = self.analysis.is_constant(node.test)
+        self.analysis.push_control(const, False)
+
+        self.generic_visit(node)
+
+        self.analysis.pop_control()
+
+    # The continue and break statements should be pretty rare, so if they
+    # occur, we mark everything later in the loop as non-const.
+
+    def visit_Break(self, node):
+        self.analysis.exit_loop()
+
+    def visit_Continue(self, node):
+        self.analysis.exit_loop()
+
+
+class CompilerCache(object):
+    """
+    Objects of this class are used to cache the compiliation of Python code.
+    """
+
+    def __init__(self):
+        self.ast_eval_cache = { }
+        self.ast_exec_cache = { }
+
+        # True if we've changed the caches.
+        self.updated = False
+
+        # The version of this object.
+        self.version = 1
+
+    def ast_eval_literal(self, expr):
+        """
+        Compiles an expression into an AST.
+        """
+
+        if isinstance(expr, renpy.ast.PyExpr):
+            filename = expr.filename
+            linenumber = expr.linenumber
+        else:
+            filename = None
+            linenumber = None
+
+        key = (expr, filename, linenumber)
+
+        rv = self.ast_eval_cache.get(key, None)
+
+        if rv is None:
+            expr = py_compile(expr, 'eval', ast_node=True)
+
+            try:
+                ast.literal_eval(expr)
+                literal = True
+            except Exception:
+                literal = False
+
+            rv = (expr, literal)
+
+            self.ast_eval_cache[key] = rv
+            self.updated = True
+
+        new_ccache.ast_eval_cache[key] = rv
+
+        return rv
+
+    def ast_eval(self, expr):
+        return self.ast_eval_literal(expr)[0]
+
+    def ast_exec(self, code):
+        """
+        Compiles a block into an AST.
+        """
+
+        if isinstance(code, renpy.ast.PyExpr):
+            key = (code, code.filename, code.linenumber)
+        else:
+            key = (code, None, None)
+
+        rv = self.ast_exec_cache.get(key, None)
+
+        if rv is None:
+            rv = py_compile(code, 'exec', ast_node=True)
+            self.ast_exec_cache[key] = rv
+            self.updated = True
+
+        new_ccache.ast_exec_cache[key] = rv
+
+        return rv
+
+
+ccache = CompilerCache()
+new_ccache = CompilerCache()
+
+CACHE_FILENAME = "cache/pyanalysis.rpyb"
+
+
+def load_cache():
+    if renpy.game.args.compile: # type: ignore
+        return
+
+    try:
+        with renpy.loader.load(CACHE_FILENAME) as f:
+            c = loads(zlib.decompress(f.read()))
+
+        if c.version == ccache.version:
+            ccache.ast_eval_cache.update(c.ast_eval_cache)
+            ccache.ast_exec_cache.update(c.ast_exec_cache)
+    except Exception:
+        pass
+
+
+def save_cache():
+    if not ccache.updated:
+        return
+
+    if renpy.macapp:
+        return
+
+    try:
+        data = zlib.compress(dumps(new_ccache, True), 3)
+
+        with open(renpy.loader.get_path(CACHE_FILENAME), "wb") as f:
+            f.write(data)
+    except Exception:
+        pass
+
+
+__all__ = [
+    "always_constants",
+    "Analysis",
+    "ccache",
+    "CompilerCache",
+    "const",
+    "constants",
+    "Control",
+    "DeltaSet",
+    "GLOBAL_CONST",
+    "load_cache",
+    "LOCAL_CONST",
+    "local_constants",
+    "new_ccache",
+    "not_const",
+    "NOT_CONST",
+    "not_constants",
+    "pure",
+    "pure_functions",
+    "PyAnalysis",
+    "save_cache",
+]

renpy/py3analysis.py

@@ -0,0 +1,861 @@
+# Copyright 2004-2024 Tom Rothamel <pytom@bishoujo.us>
+#
+# Permission is hereby granted, free of charge, to any person
+# obtaining a copy of this software and associated documentation files
+# (the "Software"), to deal in the Software without restriction,
+# including without limitation the rights to use, copy, modify, merge,
+# publish, distribute, sublicense, and/or sell copies of the Software,
+# and to permit persons to whom the Software is furnished to do so,
+# subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+from __future__ import division, absolute_import, with_statement, print_function, unicode_literals
+from renpy.compat import PY2, basestring, bchr, bord, chr, open, pystr, range, round, str, tobytes, unicode # *
+
+
+import builtins
+
+import renpy # @UnusedImport
+from renpy.python import py_compile
+
+# Import the Python AST module, instead of the Ren'Py ast module.
+import ast
+
+import zlib
+
+from renpy.compat.pickle import loads, dumps
+
+# The set of names that should be treated as constants.
+always_constants = { 'True', 'False', 'None' }
+
+# The set of names that should be treated as pure functions.
+pure_functions = {
+    # Python 3 builtins.
+    'abs', 'all', 'any', 'ascii', 'bin', 'bool', 'bytes', 'callable',
+    'chr', 'complex', 'dict', 'dir', 'divmod', 'enumerate', 'filter', 'float',
+    'format', 'frozenset', 'getattr', 'hasattr', 'hash', 'hex', 'int',
+    'isinstance', 'issubclass', 'len', 'list', 'map', 'max', 'min', 'oct',
+    'ord', 'pow', 'range', 'repr', 'reversed', 'round', 'set', 'slice', 'sorted',
+    'str', 'sum', 'tuple', 'type', 'zip',
+
+    # minstore.py
+    "_",
+    "_p",
+    "absolute",
+    "position",
+    "__renpy__list__",
+    "__renpy__dict__",
+    "__renpy__set__",
+
+    # defaultstore.py
+    "ImageReference", "Image", "Frame", "Solid", "LiveComposite", "LiveCrop",
+    "LiveTile", "Flatten", "Null", "Window", "Viewport", "DynamicDisplayable",
+    "ConditionSwitch", "ShowingSwitch", "Transform", "Animation", "Movie",
+    "Particles", "SnowBlossom", "Text", "ParameterizedText", "FontGroup",
+    "Drag", "Alpha", "AlphaMask", "Position", "Pan", "Move", "Motion", "Revolve", "Zoom",
+    "RotoZoom", "FactorZoom", "SizeZoom", "Fade", "Dissolve", "ImageDissolve",
+    "AlphaDissolve", "CropMove", "PushMove", "Pixellate", "OldMoveTransition",
+    "MoveTransition", "MoveFactory", "MoveIn", "MoveOut", "ZoomInOut",
+    "RevolveInOut", "MultipleTransition", "ComposeTransition", "Pause",
+    "SubTransition", "ADVSpeaker", "ADVCharacter", "Speaker", "Character",
+    "DynamicCharacter", "Fixed", "HBox", "VBox", "Grid", "AlphaBlend", "At",
+    "color", "Color",
+
+    # ui.py
+
+    "ui.returns",
+    "ui.jumps",
+    "ui.jumpsoutofcontext",
+    "ui.callsinnewcontext",
+    "ui.invokesinnewcontext",
+    "ui.gamemenus",
+
+    # renpy.py
+
+    "renpy.version_string",
+    "renpy.version_only",
+    "renpy.version_tuple",
+    "renpy.version_name",
+    "renpy.license",
+}
+
+constants = { "config", "style" } | always_constants | pure_functions
+
+# A set of names that should not be treated as global constants.
+not_constants = set()
+
+# The base set for the local constants.
+local_constants = set()
+
+
+def const(name):
+    """
+    :doc: const
+
+    Declares a variable in the store to be constant.
+
+    A variable is constant if nothing can change its value, or any value
+    reached by indexing it or accessing its attributes. Variables must
+    remain constant out of define, init, and translate python blocks.
+
+    `name`
+        A string giving the name of the variable to declare constant.
+    """
+
+    if name not in not_constants:
+        constants.add(name)
+
+
+def not_const(name):
+    """
+    :doc: const
+
+    Declares a name in the store to be not constant.
+
+    This undoes the effect of calls to :func:`renpy.const` and
+    :func:`renpy.pure`.
+
+    `name`
+        The name to declare not constant.
+    """
+
+    constants.discard(name)
+    pure_functions.discard(name)
+    not_constants.add(name)
+
+
+def pure(fn):
+    """
+    :doc: const
+
+    Declares a function as pure. A pure function must always return the
+    same value when it is called with the same arguments, outside of
+    define, init, and translate python blocks.
+
+    `fn`
+        The name of the function to declare pure. This may either be a string
+        containing the name of the function, or the function itself.
+        If a string is passed and the function is inside a module,
+        this string should contain the module name with the dot.
+
+    Returns `fn`, allowing this function to be used as a decorator.
+    """
+
+    name = fn
+
+    if not isinstance(name, basestring):
+        name = fn.__name__
+
+        module = fn.__module__
+        name = module + "." + name
+
+    if name.startswith("store."):
+        name = name[6:]
+
+    if name not in not_constants:
+        pure_functions.add(name)
+        constants.add(name)
+
+    return fn
+
+
+class Control(object):
+    """
+    Represents control flow.
+
+    `const`
+        True if this statement always executes.
+
+    `loop`
+        True if this corresponds to a loop.
+
+    `imagemap`
+        True if this control is in a non-constant imagemap.
+    """
+
+    def __init__(self, const, loop, imagemap):
+        self.const = const
+        self.loop = loop
+        self.imagemap = imagemap
+
+    def __repr__(self):
+        return "<Control const={0} loop={1} imagemap={2}>".format(self.const, self.loop, self.imagemap)
+
+
+# Three levels of constness.
+
+# An expression is globally constant if it will evaluate to the same value
+# whenever it is run.
+GLOBAL_CONST = 2
+
+# An expression is locally const if it will evaluate to the same value when
+# run in the same place - the same screen, with the same parameters, the same
+# statement, and the same iteration of a for loop.
+LOCAL_CONST = 1
+
+# An expression is not const if it wilk change it's value.
+NOT_CONST = 0
+
+
+class DeltaSet(object):
+
+    def __init__(self, base, copy=None):
+        """
+        Represents a set that stores its contents as differences from a base
+        set.
+        """
+
+        self.base = base
+
+        if copy is not None:
+            self.added = set(copy.added)
+            self.removed = set(copy.removed)
+        else:
+            self.added = set()
+            self.removed = set()
+
+        self.changed = False
+
+    def add(self, v):
+
+        if v in self.removed:
+            self.removed.discard(v)
+            self.changed = True
+        elif v not in self.base and v not in self.added:
+            self.added.add(v)
+            self.changed = True
+
+    def discard(self, v):
+
+        if v in self.added:
+            self.added.discard(v)
+            self.changed = True
+        elif v in self.base and v not in self.removed:
+            self.removed.add(v)
+            self.changed = True
+
+    def __contains__(self, v):
+        return (v in self.added) or ((v in self.base) and (v not in self.removed))
+
+    def copy(self):
+        return DeltaSet(self.base, self)
+
+    def __iter__(self):
+
+        for i in self.base:
+            if i not in self.removed:
+                yield i
+
+        for i in self.added:
+            yield i
+
+
+class Analysis(object):
+    """
+    Represents the result of code analysis, and provides tools to perform
+    code analysis.
+    """
+
+    def __init__(self, parent=None):
+
+        # The parent context transcludes run in, or None if there is no parent
+        # context.
+        self.parent = parent
+
+        # Analyses of children, such a screens we use.
+        self.children = { }
+
+        # The variables we consider to be not-constant.
+        self.not_constant = DeltaSet(not_constants)
+
+        # Variables we consider to be locally constant.
+        self.local_constant = DeltaSet(local_constants)
+
+        # Variables we consider to be globally constant.
+        self.global_constant = DeltaSet(always_constants)
+
+        # The functions we consider to be pure.
+        self.pure_functions = DeltaSet(pure_functions)
+
+        # Represents what we know about the current control.
+        self.control = Control(True, False, False)
+
+        # The stack of const_flow values.
+        self.control_stack = [ self.control ]
+
+    def get_child(self, identifier):
+        if identifier in self.children:
+            return self.children[identifier]
+
+        rv = Analysis(self)
+        self.children[identifier] = rv
+
+        return rv
+
+    def push_control(self, const=True, loop=False, imagemap=False):
+        self.control = Control(self.control.const and const, loop, self.control.imagemap or imagemap)
+        self.control_stack.append(self.control) # type: ignore
+
+    def pop_control(self):
+        rv = self.control_stack.pop()
+        self.control = self.control_stack[-1]
+        return rv
+
+    def imagemap(self):
+        """
+        Returns NOT_CONST if we're in a non-constant imagemap.
+        """
+
+        if self.control.imagemap:
+            return NOT_CONST
+        else:
+            return GLOBAL_CONST
+
+    def exit_loop(self):
+        """
+        Call this to indicate the current loop is being exited by the
+        continue or break statements.
+        """
+
+        l = list(self.control_stack)
+        l.reverse()
+
+        for i in l:
+            i.const = False
+
+            if i.loop:
+                break
+
+    def at_fixed_point(self):
+        """
+        Returns True if we've reached a fixed point, where the analysis has
+        not changed since the last time we called this function.
+        """
+
+        for i in self.children.values():
+            if not i.at_fixed_point():
+                return False
+
+        if (self.not_constant.changed or
+                self.global_constant.changed or
+                self.local_constant.changed or
+                self.pure_functions.changed):
+
+            self.not_constant.changed = False
+            self.global_constant.changed = False
+            self.local_constant.changed = False
+            self.pure_functions.changed = False
+
+            return False
+
+        return True
+
+    def mark_constant(self, name):
+        """
+        Marks `name` as a potential local constant.
+        """
+
+        if not name in self.not_constant:
+            self.local_constant.add(name)
+            self.global_constant.discard(name)
+            self.pure_functions.discard(name)
+
+    def mark_not_constant(self, name):
+        """
+        Marks `name` as definitely not-constant.
+        """
+
+        self.not_constant.add(name)
+
+        self.pure_functions.discard(name)
+        self.local_constant.discard(name)
+        self.global_constant.discard(name)
+
+    def is_constant(self, node):
+        """
+        Returns true if `node` is constant for the purpose of screen
+        language. Node should be a python AST node.
+
+        Screen language ignores object identity for the purposes of
+        object equality.
+        """
+
+        def check_name(node):
+            """
+            Check nodes that make up a name. This returns a pair:
+
+            * The first element is True if the node is constant, and False
+              otherwise.
+            * The second element is None if the node is constant or the name is
+              not known, and the name otherwise.
+            """
+
+            if isinstance(node, ast.Name):
+                const = NOT_CONST
+                name = node.id
+
+            elif isinstance(node, ast.Attribute):
+                const, name = check_name(node.value)
+
+                if name is not None:
+                    name = name + "." + node.attr
+
+            else:
+                return check_node(node), None
+
+            if name in self.not_constant:
+                return NOT_CONST, name
+            elif name in self.global_constant:
+                return GLOBAL_CONST, name
+            elif name in self.local_constant:
+                return LOCAL_CONST, name
+            else:
+                return const, name
+
+        def check_nodes(nodes):
+            """
+            Checks a list of nodes for constness.
+            """
+
+            nodes = list(nodes)
+
+            if not nodes:
+                return GLOBAL_CONST
+
+            return min(check_node(i) for i in nodes)
+
+        def check_node(node):
+            """
+            When given `node`, part of a Python expression, returns how
+            const the expression is.
+            """
+
+            # This handles children that do not exist.
+            if node is None:
+                return GLOBAL_CONST
+
+            # PY3: see if there are new node types.
+
+
+            if isinstance(node, ast.Constant):
+                return GLOBAL_CONST
+
+            elif isinstance(node, ast.BoolOp):
+                return check_nodes(node.values)
+
+            elif isinstance(node, ast.NamedExpr):
+                return check_node(node.value)
+
+            elif isinstance(node, ast.BinOp):
+                return min(
+                    check_node(node.left),
+                    check_node(node.right),
+                    )
+
+            elif isinstance(node, ast.UnaryOp):
+                return check_node(node.operand)
+
+
+            # ast.Lambda is NOT_CONST.
+
+            elif isinstance(node, ast.IfExp):
+                return min(
+                    check_node(node.test),
+                    check_node(node.body),
+                    check_node(node.orelse),
+                    )
+
+
+            elif isinstance(node, ast.Dict):
+                return min(
+                    check_nodes(node.keys),
+                    check_nodes(node.values)
+                    )
+
+            elif isinstance(node, ast.Set):
+                return check_nodes(node.elts)
+
+
+            # ast.ListComp is NOT_CONST.
+            # ast.SetComp is NOT_CONST.
+            # ast.DictComp is NOT_CONST.
+
+            # ast.GeneratorExp is NOT_CONST.
+
+            # ast.Await is NOT_CONST.
+            # ast.Yield is NOT_CONST.
+            # ast.YieldFrom is NOT_CONST.
+
+            elif isinstance(node, ast.Compare):
+                return min(
+                    check_node(node.left),
+                    check_nodes(node.comparators),
+                    )
+
+            elif isinstance(node, ast.Call):
+                const, name = check_name(node.func)
+
+                # The function must have a name, and must be declared pure.
+                if (const != GLOBAL_CONST) or (name not in self.pure_functions):
+                    return NOT_CONST
+
+                return min(
+                    check_nodes(node.args),
+                    check_nodes(i.value for i in node.keywords),
+                )
+
+
+            elif isinstance(node, ast.FormattedValue):
+                return min(
+                    check_node(node.value),
+                    check_node(node.format_spec),
+                )
+
+            elif isinstance(node, ast.JoinedStr):
+                return check_nodes(node.values)
+
+            elif isinstance(node, (ast.Attribute, ast.Name)):
+                return check_name(node)[0]
+
+            elif isinstance(node, ast.Subscript):
+                return min(
+                    check_node(node.value),
+                    check_node(node.slice),
+                    )
+
+            elif isinstance(node, ast.Starred):
+                return check_node(node.value)
+
+            elif isinstance(node, (ast.List, ast.Tuple)):
+                return check_nodes(node.elts)
+
+            elif isinstance(node, ast.Slice):
+                return min(
+                    check_node(node.lower),
+                    check_node(node.upper),
+                    check_node(node.step),
+                )
+
+            return NOT_CONST
+
+        return check_node(node)
+
+    def is_constant_expr(self, expr):
+        """
+        Compiles `expr` into an AST node, then returns the result of
+        self.is_constant called on that node.
+        """
+
+        node, literal = ccache.ast_eval_literal(expr)
+
+        if literal:
+            return GLOBAL_CONST
+        else:
+            return self.is_constant(node)
+
+    def python(self, code):
+        """
+        Performs analysis on a block of python code.
+        """
+
+        nodes = ccache.ast_exec(code)
+
+        a = PyAnalysis(self)
+
+        for i in nodes:
+            a.visit(i)
+
+    def parameters(self, parameters):
+        """
+        Analyzes the parameters to the screen.
+        """
+
+        self.global_constant = DeltaSet(constants)
+
+        # As we have parameters, analyze with those parameters.
+
+        for name in parameters.parameters:
+            self.mark_not_constant(name)
+
+
+class PyAnalysis(ast.NodeVisitor):
+    """
+    This analyzes Python code to determine which variables should be
+    marked const, and which should be marked non-const.
+    """
+
+    def __init__(self, analysis):
+
+        self.analysis = analysis
+
+    # Expressions that assign names.
+    def visit_Name(self, node):
+
+        if isinstance(node.ctx, ast.AugStore):
+            self.analysis.mark_not_constant(node.id)
+
+        elif isinstance(node.ctx, ast.Store):
+            if self.analysis.control.const:
+                self.analysis.mark_constant(node.id)
+            else:
+                self.analysis.mark_not_constant(node.id)
+
+    def visit_NamedExpr(self, node):
+
+        const = self.analysis.is_constant(node.value)
+        self.analysis.push_control(const, False)
+
+        self.generic_visit(node)
+
+        self.analysis.pop_control()
+
+
+    # Statements that assign names or control constness.
+    def visit_FunctionDef(self, node):
+        self.analysis.mark_constant(node.name)
+
+    def visit_AsyncFunctionDef(self, node):
+        self.analysis.mark_constant(node.name)
+
+    def visit_ClassDef(self, node):
+        self.analysis.mark_constant(node.name)
+
+    # Return can't assign a name.
+
+    # Delete doesn't assign a name - so it would be something else making
+    # the name non-const, not delete.
+
+
+    def visit_Assign(self, node):
+
+        const = self.analysis.is_constant(node.value)
+        self.analysis.push_control(const, False)
+
+        self.generic_visit(node)
+
+        self.analysis.pop_control()
+
+    def visit_AugAssign(self, node):
+
+        self.analysis.push_control(False, False)
+
+        self.generic_visit(node)
+
+        self.analysis.pop_control()
+
+    def visit_AnnAssign(self, node):
+
+        const = self.analysis.is_constant(node.value)
+        self.analysis.push_control(const, False)
+
+        self.generic_visit(node)
+
+        self.analysis.pop_control()
+
+    def visit_For(self, node): # type: (ast.For|ast.AsyncFor) -> None
+
+        const = self.analysis.is_constant(node.iter)
+
+        self.analysis.push_control(const=const, loop=True)
+        old_const = self.analysis.control.const
+
+        self.generic_visit(node) # All nodes in the loop depend on node.test.
+
+        if self.analysis.control.const != old_const:
+            self.generic_visit(node)
+
+        self.analysis.pop_control()
+
+    def visit_AsyncFor(self, node):
+        return self.visit_For(node)
+
+    def visit_While(self, node):
+
+        const = self.analysis.is_constant(node.test)
+
+        self.analysis.push_control(const=const, loop=True)
+        old_const = self.analysis.control.const
+
+        self.generic_visit(node) # All nodes in the loop depend on node.test.
+
+        if self.analysis.control.const != old_const:
+            self.generic_visit(node)
+
+        self.analysis.pop_control()
+
+    def visit_If(self, node):
+        const = self.analysis.is_constant(node.test)
+        self.analysis.push_control(const, False)
+
+        self.generic_visit(node)
+
+        self.analysis.pop_control()
+
+    # Nothing special for visit_With or visit_AsyncWith, when withitem is
+    # defined as below.
+
+    def visit_withitem(self, node):
+
+        const = self.analysis.is_constant(node.context_expr)
+        self.visit(node.context_expr)
+
+        self.analysis.push_control(const, False)
+
+        if node.optional_vars is not None:
+            self.visit(node.optional_vars)
+
+        self.analysis.pop_control()
+
+    # Match is barely implemented. We assume that it's always going to be
+    # performed on something non-constant, which means that every variable
+    # assigned inside the match is also non-constant. This is probably a
+    # reasonable assumption.
+    def visit_Match(self, node):
+        self.analysis.push_control(False)
+        self.generic_visit(node)
+        self.analysis.pop_control()
+
+    def visit_Try(self, node):
+
+        for i in node.handlers:
+            if i.name:
+                self.analysis.mark_not_constant(i.name)
+
+        self.generic_visit(node)
+
+    # Import and Import from can only assign to a variable in a way that
+    # keeps it constant.
+
+    # Global and NonLocal only make sense inside Python functions, and we don't
+    # analyze Python functions.
+
+    # Expr can be ignored, as it can't assign.
+
+    # The continue and break statements should be pretty rare, so if they
+    # occur, we mark everything later in the loop as non-const.
+    def visit_Break(self, node):
+        self.analysis.exit_loop()
+
+    def visit_Continue(self, node):
+        self.analysis.exit_loop()
+
+
+class CompilerCache(object):
+    """
+    Objects of this class are used to cache the compiliation of Python code.
+    """
+
+    def __init__(self):
+        self.ast_eval_cache = { }
+        self.ast_exec_cache = { }
+
+        # True if we've changed the caches.
+        self.updated = False
+
+        # The version of this object.
+        self.version = 1
+
+    def ast_eval_literal(self, expr):
+        """
+        Compiles an expression into an AST.
+        """
+
+        if isinstance(expr, renpy.ast.PyExpr):
+            filename = expr.filename
+            linenumber = expr.linenumber
+        else:
+            filename = None
+            linenumber = None
+
+        key = (expr, filename, linenumber)
+
+        rv = self.ast_eval_cache.get(key, None)
+
+        if rv is None:
+            expr = py_compile(expr, 'eval', ast_node=True)
+
+            try:
+                ast.literal_eval(expr)
+                literal = True
+            except Exception:
+                literal = False
+
+            rv = (expr, literal)
+
+            self.ast_eval_cache[key] = rv
+            self.updated = True
+
+        new_ccache.ast_eval_cache[key] = rv
+
+        return rv
+
+    def ast_eval(self, expr):
+        return self.ast_eval_literal(expr)[0]
+
+    def ast_exec(self, code):
+        """
+        Compiles a block into an AST.
+        """
+
+        if isinstance(code, renpy.ast.PyExpr):
+            key = (code, code.filename, code.linenumber)
+        else:
+            key = (code, None, None)
+
+        rv = self.ast_exec_cache.get(key, None)
+
+        if rv is None:
+            rv = py_compile(code, 'exec', ast_node=True)
+            self.ast_exec_cache[key] = rv
+            self.updated = True
+
+        new_ccache.ast_exec_cache[key] = rv
+
+        return rv
+
+
+ccache = CompilerCache()
+new_ccache = CompilerCache()
+
+CACHE_FILENAME = "cache/py3analysis.rpyb"
+
+
+def load_cache():
+    if renpy.game.args.compile: # type: ignore
+        return
+
+    try:
+        with renpy.loader.load(CACHE_FILENAME) as f:
+            c = loads(zlib.decompress(f.read()))
+
+        if c.version == ccache.version:
+            ccache.ast_eval_cache.update(c.ast_eval_cache)
+            ccache.ast_exec_cache.update(c.ast_exec_cache)
+    except Exception:
+        pass
+
+
+def save_cache():
+    if not ccache.updated:
+        return
+
+    if renpy.macapp:
+        return
+
+    try:
+        data = zlib.compress(dumps(new_ccache, True), 3)
+
+        with open(renpy.loader.get_path(CACHE_FILENAME), "wb") as f:
+            f.write(data)
+    except Exception:
+        pass

renpy/pyanalysis.py

@@ -23,848 +23,7 @@ from __future__ import division, absolute_import, with_statement, print_function
 from renpy.compat import PY2, basestring, bchr, bord, chr, open, pystr, range, round, str, tobytes, unicode # *
 
 
-import builtins
-
-import renpy # @UnusedImport
-from renpy.python import py_compile
-
-# Import the Python AST module, instead of the Ren'Py ast module.
-import ast
-
-import zlib
-
-from renpy.compat.pickle import loads, dumps
-
-# The set of names that should be treated as constants.
-always_constants = { 'True', 'False', 'None' }
-
-# The set of names that should be treated as pure functions.
-pure_functions = {
-    # Python 3 builtins.
-    'abs', 'all', 'any', 'ascii', 'bin', 'bool', 'bytes', 'callable',
-    'chr', 'complex', 'dict', 'dir', 'divmod', 'enumerate', 'filter', 'float',
-    'format', 'frozenset', 'getattr', 'hasattr', 'hash', 'hex', 'int',
-    'isinstance', 'issubclass', 'len', 'list', 'map', 'max', 'min', 'oct',
-    'ord', 'pow', 'range', 'repr', 'reversed', 'round', 'set', 'slice', 'sorted',
-    'str', 'sum', 'tuple', 'type', 'zip',
-
-    # minstore.py
-    "_",
-    "_p",
-    "absolute",
-    "position",
-    "__renpy__list__",
-    "__renpy__dict__",
-    "__renpy__set__",
-
-    # defaultstore.py
-    "ImageReference", "Image", "Frame", "Solid", "LiveComposite", "LiveCrop",
-    "LiveTile", "Flatten", "Null", "Window", "Viewport", "DynamicDisplayable",
-    "ConditionSwitch", "ShowingSwitch", "Transform", "Animation", "Movie",
-    "Particles", "SnowBlossom", "Text", "ParameterizedText", "FontGroup",
-    "Drag", "Alpha", "AlphaMask", "Position", "Pan", "Move", "Motion", "Revolve", "Zoom",
-    "RotoZoom", "FactorZoom", "SizeZoom", "Fade", "Dissolve", "ImageDissolve",
-    "AlphaDissolve", "CropMove", "PushMove", "Pixellate", "OldMoveTransition",
-    "MoveTransition", "MoveFactory", "MoveIn", "MoveOut", "ZoomInOut",
-    "RevolveInOut", "MultipleTransition", "ComposeTransition", "Pause",
-    "SubTransition", "ADVSpeaker", "ADVCharacter", "Speaker", "Character",
-    "DynamicCharacter", "Fixed", "HBox", "VBox", "Grid", "AlphaBlend", "At",
-    "color", "Color",
-
-    # ui.py
-
-    "ui.returns",
-    "ui.jumps",
-    "ui.jumpsoutofcontext",
-    "ui.callsinnewcontext",
-    "ui.invokesinnewcontext",
-    "ui.gamemenus",
-
-    # renpy.py
-
-    "renpy.version_string",
-    "renpy.version_only",
-    "renpy.version_tuple",
-    "renpy.version_name",
-    "renpy.license",
-}
-
-constants = { "config", "style" } | always_constants | pure_functions
-
-# A set of names that should not be treated as global constants.
-not_constants = set()
-
-# The base set for the local constants.
-local_constants = set()
-
-
-def const(name):
-    """
-    :doc: const
-
-    Declares a variable in the store to be constant.
-
-    A variable is constant if nothing can change its value, or any value
-    reached by indexing it or accessing its attributes. Variables must
-    remain constant out of define, init, and translate python blocks.
-
-    `name`
-        A string giving the name of the variable to declare constant.
-    """
-
-    if name not in not_constants:
-        constants.add(name)
-
-
-def not_const(name):
-    """
-    :doc: const
-
-    Declares a name in the store to be not constant.
-
-    This undoes the effect of calls to :func:`renpy.const` and
-    :func:`renpy.pure`.
-
-    `name`
-        The name to declare not constant.
-    """
-
-    constants.discard(name)
-    pure_functions.discard(name)
-    not_constants.add(name)
-
-
-def pure(fn):
-    """
-    :doc: const
-
-    Declares a function as pure. A pure function must always return the
-    same value when it is called with the same arguments, outside of
-    define, init, and translate python blocks.
-
-    `fn`
-        The name of the function to declare pure. This may either be a string
-        containing the name of the function, or the function itself.
-        If a string is passed and the function is inside a module,
-        this string should contain the module name with the dot.
-
-    Returns `fn`, allowing this function to be used as a decorator.
-    """
-
-    name = fn
-
-    if not isinstance(name, basestring):
-        name = fn.__name__
-
-        module = fn.__module__
-        name = module + "." + name
-
-    if name.startswith("store."):
-        name = name[6:]
-
-    if name not in not_constants:
-        pure_functions.add(name)
-        constants.add(name)
-
-    return fn
-
-
-class Control(object):
-    """
-    Represents control flow.
-
-    `const`
-        True if this statement always executes.
-
-    `loop`
-        True if this corresponds to a loop.
-
-    `imagemap`
-        True if this control is in a non-constant imagemap.
-    """
-
-    def __init__(self, const, loop, imagemap):
-        self.const = const
-        self.loop = loop
-        self.imagemap = imagemap
-
-    def __repr__(self):
-        return "<Control const={0} loop={1} imagemap={2}>".format(self.const, self.loop, self.imagemap)
-
-
-# Three levels of constness.
-
-# An expression is globally constant if it will evaluate to the same value
-# whenever it is run.
-GLOBAL_CONST = 2
-
-# An expression is locally const if it will evaluate to the same value when
-# run in the same place - the same screen, with the same parameters, the same
-# statement, and the same iteration of a for loop.
-LOCAL_CONST = 1
-
-# An expression is not const if it wilk change it's value.
-NOT_CONST = 0
-
-
-class DeltaSet(object):
-
-    def __init__(self, base, copy=None):
-        """
-        Represents a set that stores its contents as differences from a base
-        set.
-        """
-
-        self.base = base
-
-        if copy is not None:
-            self.added = set(copy.added)
-            self.removed = set(copy.removed)
-        else:
-            self.added = set()
-            self.removed = set()
-
-        self.changed = False
-
-    def add(self, v):
-
-        if v in self.removed:
-            self.removed.discard(v)
-            self.changed = True
-        elif v not in self.base and v not in self.added:
-            self.added.add(v)
-            self.changed = True
-
-    def discard(self, v):
-
-        if v in self.added:
-            self.added.discard(v)
-            self.changed = True
-        elif v in self.base and v not in self.removed:
-            self.removed.add(v)
-            self.changed = True
-
-    def __contains__(self, v):
-        return (v in self.added) or ((v in self.base) and (v not in self.removed))
-
-    def copy(self):
-        return DeltaSet(self.base, self)
-
-    def __iter__(self):
-
-        for i in self.base:
-            if i not in self.removed:
-                yield i
-
-        for i in self.added:
-            yield i
-
-
-class Analysis(object):
-    """
-    Represents the result of code analysis, and provides tools to perform
-    code analysis.
-    """
-
-    def __init__(self, parent=None):
-
-        # The parent context transcludes run in, or None if there is no parent
-        # context.
-        self.parent = parent
-
-        # Analyses of children, such a screens we use.
-        self.children = { }
-
-        # The variables we consider to be not-constant.
-        self.not_constant = DeltaSet(not_constants)
-
-        # Variables we consider to be locally constant.
-        self.local_constant = DeltaSet(local_constants)
-
-        # Variables we consider to be globally constant.
-        self.global_constant = DeltaSet(always_constants)
-
-        # The functions we consider to be pure.
-        self.pure_functions = DeltaSet(pure_functions)
-
-        # Represents what we know about the current control.
-        self.control = Control(True, False, False)
-
-        # The stack of const_flow values.
-        self.control_stack = [ self.control ]
-
-    def get_child(self, identifier):
-        if identifier in self.children:
-            return self.children[identifier]
-
-        rv = Analysis(self)
-        self.children[identifier] = rv
-
-        return rv
-
-    def push_control(self, const=True, loop=False, imagemap=False):
-        self.control = Control(self.control.const and const, loop, self.control.imagemap or imagemap)
-        self.control_stack.append(self.control) # type: ignore
-
-    def pop_control(self):
-        rv = self.control_stack.pop()
-        self.control = self.control_stack[-1]
-        return rv
-
-    def imagemap(self):
-        """
-        Returns NOT_CONST if we're in a non-constant imagemap.
-        """
-
-        if self.control.imagemap:
-            return NOT_CONST
-        else:
-            return GLOBAL_CONST
-
-    def exit_loop(self):
-        """
-        Call this to indicate the current loop is being exited by the
-        continue or break statements.
-        """
-
-        l = list(self.control_stack)
-        l.reverse()
-
-        for i in l:
-            i.const = False
-
-            if i.loop:
-                break
-
-    def at_fixed_point(self):
-        """
-        Returns True if we've reached a fixed point, where the analysis has
-        not changed since the last time we called this function.
-        """
-
-        for i in self.children.values():
-            if not i.at_fixed_point():
-                return False
-
-        if (self.not_constant.changed or
-                self.global_constant.changed or
-                self.local_constant.changed or
-                self.pure_functions.changed):
-
-            self.not_constant.changed = False
-            self.global_constant.changed = False
-            self.local_constant.changed = False
-            self.pure_functions.changed = False
-
-            return False
-
-        return True
-
-    def mark_constant(self, name):
-        """
-        Marks `name` as a potential local constant.
-        """
-
-        if not name in self.not_constant:
-            self.local_constant.add(name)
-            self.global_constant.discard(name)
-            self.pure_functions.discard(name)
-
-    def mark_not_constant(self, name):
-        """
-        Marks `name` as definitely not-constant.
-        """
-
-        self.not_constant.add(name)
-
-        self.pure_functions.discard(name)
-        self.local_constant.discard(name)
-        self.global_constant.discard(name)
-
-
-    def _check_name(self, node):
-        """
-        Check nodes that make up a name. This returns a pair:
-
-        * The first element is True if the node is constant, and False
-            otherwise.
-        * The second element is None if the node is constant or the name is
-            not known, and the name otherwise.
-        """
-
-        if isinstance(node, ast.Name):
-            const = NOT_CONST
-            name = node.id
-
-        elif isinstance(node, ast.Attribute):
-            const, name = self._check_name(node.value)
-
-            if name is not None:
-                name = name + "." + node.attr
-
-        else:
-            return self._check_node(node), None
-
-        if name in self.not_constant:
-            return NOT_CONST, name
-        elif name in self.global_constant:
-            return GLOBAL_CONST, name
-        elif name in self.local_constant:
-            return LOCAL_CONST, name
-        else:
-            return const, name
-
-    def _check_nodes(self, nodes):
-        """
-        Checks a list of nodes for constness.
-        """
-
-        nodes = list(nodes)
-
-        if not nodes:
-            return GLOBAL_CONST
-
-        return min(self._check_node(i) for i in nodes)
-
-    def _check_node(self, node):
-        """
-        When given `node`, part of a Python expression, returns how
-        const the expression is.
-        """
-
-        # This handles children that do not exist.
-        if node is None:
-            return GLOBAL_CONST
-
-        # PY3: see if there are new node types.
-
-
-        if isinstance(node, ast.Constant):
-            return GLOBAL_CONST
-
-        elif isinstance(node, ast.BoolOp):
-            return self._check_nodes(node.values)
-
-        elif isinstance(node, ast.NamedExpr):
-            return self._check_node(node.value)
-
-        elif isinstance(node, ast.BinOp):
-            return min(
-                self._check_node(node.left),
-                self._check_node(node.right),
-                )
-
-        elif isinstance(node, ast.UnaryOp):
-            return self._check_node(node.operand)
-
-
-        # ast.Lambda is NOT_CONST.
-
-        elif isinstance(node, ast.IfExp):
-            return min(
-                self._check_node(node.test),
-                self._check_node(node.body),
-                self._check_node(node.orelse),
-                )
-
-
-        elif isinstance(node, ast.Dict):
-            return min(
-                self._check_nodes(node.keys),
-                self._check_nodes(node.values)
-                )
-
-        elif isinstance(node, ast.Set):
-            return self._check_nodes(node.elts)
-
-
-        # ast.ListComp is NOT_CONST.
-        # ast.SetComp is NOT_CONST.
-        # ast.DictComp is NOT_CONST.
-
-        # ast.GeneratorExp is NOT_CONST.
-
-        # ast.Await is NOT_CONST.
-        # ast.Yield is NOT_CONST.
-        # ast.YieldFrom is NOT_CONST.
-
-        elif isinstance(node, ast.Compare):
-            return min(
-                self._check_node(node.left),
-                self._check_nodes(node.comparators),
-                )
-
-        elif isinstance(node, ast.Call):
-            const, name = self._check_name(node.func)
-
-            # The function must have a name, and must be declared pure.
-            if (const != GLOBAL_CONST) or (name not in self.pure_functions):
-                return NOT_CONST
-
-            return min(
-                self._check_nodes(node.args),
-                self._check_nodes(i.value for i in node.keywords),
-            )
-
-
-        elif isinstance(node, ast.FormattedValue):
-            return min(
-                self._check_node(node.value),
-                self._check_node(node.format_spec),
-            )
-
-        elif isinstance(node, ast.JoinedStr):
-            return self._check_nodes(node.values)
-
-        elif isinstance(node, (ast.Attribute, ast.Name)):
-            return self._check_name(node)[0]
-
-        elif isinstance(node, ast.Subscript):
-            return min(
-                self._check_node(node.value),
-                self._check_node(node.slice),
-                )
-
-        elif isinstance(node, ast.Starred):
-            return self._check_node(node.value)
-
-        elif isinstance(node, (ast.List, ast.Tuple)):
-            return self._check_nodes(node.elts)
-
-        elif isinstance(node, ast.Slice):
-            return min(
-                self._check_node(node.lower),
-                self._check_node(node.upper),
-                self._check_node(node.step),
-            )
-
-        return NOT_CONST
-
-    def is_constant(self, node):
-        """
-        Returns true if `node` is constant for the purpose of screen
-        language. Node should be a python AST node.
-
-        Screen language ignores object identity for the purposes of
-        object equality.
-        """
-
-        return self._check_node(node)
-
-    def is_constant_expr(self, expr):
-        """
-        Compiles `expr` into an AST node, then returns the result of
-        self.is_constant called on that node.
-        """
-
-        node, literal = ccache.ast_eval_literal(expr)
-
-        if literal:
-            return GLOBAL_CONST
-        else:
-            return self.is_constant(node)
-
-    def python(self, code):
-        """
-        Performs analysis on a block of python code.
-        """
-
-        nodes = ccache.ast_exec(code)
-
-        a = PyAnalysis(self)
-
-        for i in nodes:
-            a.visit(i)
-
-    def parameters(self, parameters):
-        """
-        Analyzes the parameters to the screen.
-        """
-
-        self.global_constant = DeltaSet(constants)
-
-        # As we have parameters, analyze with those parameters.
-
-        for name in parameters.parameters:
-            self.mark_not_constant(name)
-
-
-class PyAnalysis(ast.NodeVisitor):
-    """
-    This analyzes Python code to determine which variables should be
-    marked const, and which should be marked non-const.
-    """
-
-    def __init__(self, analysis):
-
-        self.analysis = analysis
-
-    # Expressions that assign names.
-    def visit_Name(self, node):
-
-        if isinstance(node.ctx, ast.AugStore):
-            self.analysis.mark_not_constant(node.id)
-
-        elif isinstance(node.ctx, ast.Store):
-            if self.analysis.control.const:
-                self.analysis.mark_constant(node.id)
-            else:
-                self.analysis.mark_not_constant(node.id)
-
-    def visit_NamedExpr(self, node):
-
-        const = self.analysis.is_constant(node.value)
-        self.analysis.push_control(const, False)
-
-        self.generic_visit(node)
-
-        self.analysis.pop_control()
-
-
-    # Statements that assign names or control constness.
-    def visit_FunctionDef(self, node):
-        self.analysis.mark_constant(node.name)
-
-    def visit_AsyncFunctionDef(self, node):
-        self.analysis.mark_constant(node.name)
-
-    def visit_ClassDef(self, node):
-        self.analysis.mark_constant(node.name)
-
-    # Return can't assign a name.
-
-    # Delete doesn't assign a name - so it would be something else making
-    # the name non-const, not delete.
-
-
-    def visit_Assign(self, node):
-
-        const = self.analysis.is_constant(node.value)
-        self.analysis.push_control(const, False)
-
-        self.generic_visit(node)
-
-        self.analysis.pop_control()
-
-    def visit_AugAssign(self, node):
-
-        self.analysis.push_control(False, False)
-
-        self.generic_visit(node)
-
-        self.analysis.pop_control()
-
-    def visit_AnnAssign(self, node):
-
-        const = self.analysis.is_constant(node.value)
-        self.analysis.push_control(const, False)
-
-        self.generic_visit(node)
-
-        self.analysis.pop_control()
-
-    def visit_For(self, node): # type: (ast.For|ast.AsyncFor) -> None
-
-        const = self.analysis.is_constant(node.iter)
-
-        self.analysis.push_control(const=const, loop=True)
-        old_const = self.analysis.control.const
-
-        self.generic_visit(node) # All nodes in the loop depend on node.test.
-
-        if self.analysis.control.const != old_const:
-            self.generic_visit(node)
-
-        self.analysis.pop_control()
-
-    def visit_AsyncFor(self, node):
-        return self.visit_For(node)
-
-    def visit_While(self, node):
-
-        const = self.analysis.is_constant(node.test)
-
-        self.analysis.push_control(const=const, loop=True)
-        old_const = self.analysis.control.const
-
-        self.generic_visit(node) # All nodes in the loop depend on node.test.
-
-        if self.analysis.control.const != old_const:
-            self.generic_visit(node)
-
-        self.analysis.pop_control()
-
-    def visit_If(self, node):
-        const = self.analysis.is_constant(node.test)
-        self.analysis.push_control(const, False)
-
-        self.generic_visit(node)
-
-        self.analysis.pop_control()
-
-    # Nothing special for visit_With or visit_AsyncWith, when withitem is
-    # defined as below.
-
-    def visit_withitem(self, node):
-
-        const = self.analysis.is_constant(node.context_expr)
-        self.visit(node.context_expr)
-
-        self.analysis.push_control(const, False)
-
-        if node.optional_vars is not None:
-            self.visit(node.optional_vars)
-
-        self.analysis.pop_control()
-
-    # Match is barely implemented. We assume that it's always going to be
-    # performed on something non-constant, which means that every variable
-    # assigned inside the match is also non-constant. This is probably a
-    # reasonable assumption.
-    def visit_MatchMapping(self, node):
-        if node.rest:
-            self.analysis.mark_not_constant(node.rest)
-
-    def visit_MatchStar(self, node):
-        if node.name is not None:
-            self.analysis.mark_not_constant(node.name)
-
-    def visit_MatchAs(self, node):
-        if node.name is not None:
-            self.analysis.mark_not_constant(node.name)
-
-    def visit_Try(self, node):
-
-        for i in node.handlers:
-            if i.name:
-                self.analysis.mark_not_constant(i.name)
-
-        self.generic_visit(node)
-
-    # Import and Import from can only assign to a variable in a way that
-    # keeps it constant.
-
-    # Global and NonLocal only make sense inside Python functions, and we don't
-    # analyze Python functions.
-
-    # Expr can be ignored, as it can't assign.
-
-    # The continue and break statements should be pretty rare, so if they
-    # occur, we mark everything later in the loop as non-const.
-    def visit_Break(self, node):
-        self.analysis.exit_loop()
-
-    def visit_Continue(self, node):
-        self.analysis.exit_loop()
-
-
-
-class CompilerCache(object):
-    """
-    Objects of this class are used to cache the compiliation of Python code.
-    """
-
-    def __init__(self):
-        self.ast_eval_cache = { }
-        self.ast_exec_cache = { }
-
-        # True if we've changed the caches.
-        self.updated = False
-
-        # The version of this object.
-        self.version = 1
-
-    def ast_eval_literal(self, expr):
-        """
-        Compiles an expression into an AST.
-        """
-
-        if isinstance(expr, renpy.ast.PyExpr):
-            filename = expr.filename
-            linenumber = expr.linenumber
-        else:
-            filename = None
-            linenumber = None
-
-        key = (expr, filename, linenumber)
-
-        rv = self.ast_eval_cache.get(key, None)
-
-        if rv is None:
-            expr = py_compile(expr, 'eval', ast_node=True)
-
-            try:
-                ast.literal_eval(expr)
-                literal = True
-            except Exception:
-                literal = False
-
-            rv = (expr, literal)
-
-            self.ast_eval_cache[key] = rv
-            self.updated = True
-
-        new_ccache.ast_eval_cache[key] = rv
-
-        return rv
-
-    def ast_eval(self, expr):
-        return self.ast_eval_literal(expr)[0]
-
-    def ast_exec(self, code):
-        """
-        Compiles a block into an AST.
-        """
-
-        if isinstance(code, renpy.ast.PyExpr):
-            key = (code, code.filename, code.linenumber)
-        else:
-            key = (code, None, None)
-
-        rv = self.ast_exec_cache.get(key, None)
-
-        if rv is None:
-            rv = py_compile(code, 'exec', ast_node=True)
-            self.ast_exec_cache[key] = rv
-            self.updated = True
-
-        new_ccache.ast_exec_cache[key] = rv
-
-        return rv
-
-
-ccache = CompilerCache()
-new_ccache = CompilerCache()
-
-CACHE_FILENAME = "cache/py3analysis.rpyb"
-
-
-def load_cache():
-    if renpy.game.args.compile: # type: ignore
-        return
-
-    try:
-        with renpy.loader.load(CACHE_FILENAME) as f:
-            c = loads(zlib.decompress(f.read()))
-
-        if c.version == ccache.version:
-            ccache.ast_eval_cache.update(c.ast_eval_cache)
-            ccache.ast_exec_cache.update(c.ast_exec_cache)
-    except Exception:
-        pass
-
-
-def save_cache():
-    if not ccache.updated:
-        return
-
-    if renpy.macapp:
-        return
-
-    try:
-        data = zlib.compress(dumps(new_ccache, True), 3)
-
-        with open(renpy.loader.get_path(CACHE_FILENAME), "wb") as f:
-            f.write(data)
-    except Exception:
-        pass
+if PY2:
+    from renpy.py2analysis import *
+else:
+    from renpy.py3analysis import *

renpy/python.py

@@ -26,7 +26,7 @@
 from __future__ import division, absolute_import, with_statement, print_function, unicode_literals
 from renpy.compat import PY2, basestring, bchr, bord, chr, open, pystr, range, round, str, tobytes, unicode # *
 
-from typing import Literal, Optional, Any, cast, overload
+from typing import Optional, Any
 import contextlib
 
 # Import the python ast module, not ours.
@@ -351,6 +351,13 @@ def reset_store_changes(name):
 # Code that replaces literals will calls to magic constructors.
 
 
+def b(s):
+    if PY2:
+        return s.encode("utf-8")
+    else:
+        return s
+
+
 class LoadedVariables(ast.NodeVisitor):
     """
     This is used to implement find_loaded_variables.
@@ -361,21 +368,15 @@ class LoadedVariables(ast.NodeVisitor):
             self.loaded.add(node.id)
         elif isinstance(node.ctx, ast.Store):
             self.stored.add(node.id)
+        elif PY2 and isinstance(node.ctx, ast.Param):
+            # there's no guarantee that ast.Param will keep existing in future versions of python3
+            # it's only present in asts made in py2
+            self.stored.add(node.id)
 
-    def visit_MatchMapping(self, node):
-        if node.rest:
-            self.stored.add(node.rest)
-
-    def visit_MatchStar(self, node):
-        if node.name is not None:
-            self.stored.add(node.name)
-
-    def visit_MatchAs(self, node):
-        if node.name is not None:
-            self.stored.add(node.name)
-
-    def visit_arg(self, node):
-        self.stored.add(node.arg)
+    if not PY2:
+        # we could remove this if, but the method wouldn't be called in py2 anyway
+        def visit_arg(self, node):
+            self.stored.add(node.arg)
 
     def find(self, node):
         self.loaded = set()
@@ -443,34 +444,6 @@ class WrapFormattedValue(ast.NodeTransformer):
 wrap_formatted_value = WrapFormattedValue().visit
 
 
-class FindStarredMatchPatterns(ast.NodeVisitor):
-    """
-    Given a match patten, return a list of (name, type) tuples, where type is "dict" or "list".
-    """
-
-    def __init__(self):
-        self.vars = [ ]
-
-    def visit_MatchStar(self, node: ast.MatchStar) -> Any:
-        self.generic_visit(node)
-        if node.name is not None:
-            self.vars.append((node.name, "list"))
-
-    def visit_MatchMapping(self, node: ast.MatchMapping) -> Any:
-        self.generic_visit(node)
-        if node.rest is not None:
-            self.vars.append((node.rest, "dict"))
-
-def find_starred_match_patterns(node):
-    """
-    Given a match pattern, return a list of (name, type) tuples, where type is "dict" or "list".
-    """
-
-    visitor = FindStarredMatchPatterns()
-    visitor.visit(node)
-    return visitor.vars
-
-
 class WrapNode(ast.NodeTransformer):
 
 
@@ -497,26 +470,53 @@ class WrapNode(ast.NodeTransformer):
         call_args =[ ]
 
         for var in variables:
-            lambda_args.append(ast.arg(arg=var))
+            if PY2:
+                lambda_args.append(ast.Name(id=var, ctx=ast.Param()))
+            else:
+                lambda_args.append(ast.arg(arg=var))
+
             call_args.append(ast.Name(id=var, ctx=ast.Load()))
 
-        return ast.Call(
-            func=ast.Lambda(
-                args=ast.arguments(
-                    posonlyargs=[ ],
-                    args=lambda_args,
-                    kwonlyargs=[ ],
-                    kw_defaults=[ ],
-                    defaults=[ ],
+        if PY2:
+
+            return ast.Call(
+                func=ast.Lambda(
+                    args=ast.arguments(
+                        args=lambda_args,
+                        vararg=None,
+                        kwarg=None,
+                        defaults=[]
+                    ),
+                    body=node,
                 ),
-                body=node,
-            ),
-            args=call_args,
-            keywords=[ ],
-        )
+                args=call_args,
+                keywords=[ ],
+                starargs=None,
+                kwargs=None,
+            )
+
+        else:
+
+            return ast.Call(
+                func=ast.Lambda(
+                    args=ast.arguments(
+                        posonlyargs=[ ],
+                        args=lambda_args,
+                        kwonlyargs=[ ],
+                        kw_defaults=[ ],
+                        defaults=[ ],
+                    ),
+                    body=node,
+                ),
+                args=call_args,
+                keywords=[ ],
+            )
 
     def wrap_starred_assign(self, n, targets):
 
+        if PY2:
+            return n
+
         starred = find_starred_variables(targets)
 
         if not starred:
@@ -528,13 +528,15 @@ class WrapNode(ast.NodeTransformer):
 
             call = ast.Call(
                 func=ast.Name(
-                    id="__renpy__list__",
+                    id=b("__renpy__list__"),
                     ctx=ast.Load()
                     ),
                 args=[
                     ast.Name(id=var, ctx=ast.Load())
                 ],
-                keywords=[ ])
+                keywords=[ ],
+                starargs=None,
+                kwargs=None)
 
             assign = ast.Assign(
                 targets=[ ast.Name(id=var, ctx=ast.Store()) ],
@@ -552,6 +554,9 @@ class WrapNode(ast.NodeTransformer):
 
     def wrap_starred_for(self, node):
 
+        if PY2:
+            return node
+
         starred = find_starred_variables([ node.target ])
 
         if not starred:
@@ -561,13 +566,15 @@ class WrapNode(ast.NodeTransformer):
 
             call = ast.Call(
                 func=ast.Name(
-                    id="__renpy__list__",
+                    id=b("__renpy__list__"),
                     ctx=ast.Load()
                     ),
                 args=[
                     ast.Name(id=var, ctx=ast.Load())
                 ],
-                keywords=[ ])
+                keywords=[ ],
+                starargs=None,
+                kwargs=None)
 
             assign = ast.Assign(
                 targets=[ ast.Name(id=var, ctx=ast.Store()) ],
@@ -581,6 +588,9 @@ class WrapNode(ast.NodeTransformer):
 
     def wrap_starred_with(self, node):
 
+        if PY2:
+            return node
+
         optional_vars = [ ]
 
         for i in node.items:
@@ -599,13 +609,15 @@ class WrapNode(ast.NodeTransformer):
 
             call = ast.Call(
                 func=ast.Name(
-                    id="__renpy__list__",
+                    id=b("__renpy__list__"),
                     ctx=ast.Load()
                     ),
                 args=[
                     ast.Name(id=var, ctx=ast.Load())
                 ],
-                keywords=[ ])
+                keywords=[ ],
+                starargs=None,
+                kwargs=None)
 
             assign = ast.Assign(
                 targets=[ ast.Name(id=var, ctx=ast.Store()) ],
@@ -616,52 +628,6 @@ class WrapNode(ast.NodeTransformer):
 
         return node
 
-    def wrap_match_case(self, node):
-
-        starred = find_starred_match_patterns(node.pattern)
-        if not starred:
-            return node
-
-        for var, kind in reversed(starred):
-
-            if kind == "list":
-                call = ast.Call(
-                    func=ast.Name(
-                        id="__renpy__list__",
-                        ctx=ast.Load()
-                        ),
-                    args=[
-                        ast.Name(id=var, ctx=ast.Load())
-                    ],
-                    keywords=[ ])
-
-                assign = ast.Assign(
-                    targets=[ ast.Name(id=var, ctx=ast.Store()) ],
-                    value=call,
-                )
-
-                node.body.insert(0, assign)
-
-            elif kind == "dict":
-                call = ast.Call(
-                    func=ast.Name(
-                        id="__renpy__dict__",
-                        ctx=ast.Load()
-                        ),
-                    args=[
-                        ast.Name(id=var, ctx=ast.Load())
-                    ],
-                    keywords=[ ])
-
-                assign = ast.Assign(
-                    targets=[ ast.Name(id=var, ctx=ast.Store()) ],
-                    value=call,
-                )
-
-                node.body.insert(0, assign)
-
-        return node
-
     def visit_Assign(self, n):
         n = self.generic_visit(n)
         return self.wrap_starred_assign(n, n.targets) # type: ignore
@@ -690,7 +656,7 @@ class WrapNode(ast.NodeTransformer):
         n = self.generic_visit(n)
 
         if not n.bases: # type: ignore
-            n.bases.append(ast.Name(id="object", ctx=ast.Load())) # type: ignore
+            n.bases.append(ast.Name(id=b("object"), ctx=ast.Load())) # type: ignore
 
         return n
 
@@ -700,75 +666,80 @@ class WrapNode(ast.NodeTransformer):
     def visit_SetComp(self, n):
         return ast.Call(
             func=ast.Name(
-                id="__renpy__set__",
+                id=b("__renpy__set__"),
                 ctx=ast.Load()
                 ),
             args=[ self.wrap_generator(n) ],
-            keywords=[ ])
+            keywords=[ ],
+            starargs=None,
+            kwargs=None)
 
     def visit_Set(self, n):
 
         return ast.Call(
             func=ast.Name(
-                id="__renpy__set__",
+                id=b("__renpy__set__"),
                 ctx=ast.Load()
                 ),
             args=[ self.generic_visit(n) ],
-            keywords=[ ])
+            keywords=[ ],
+            starargs=None,
+            kwargs=None)
 
     def visit_ListComp(self, n):
 
         return ast.Call(
             func=ast.Name(
-                id="__renpy__list__",
+                id=b("__renpy__list__"),
                 ctx=ast.Load()
                 ),
             args=[ self.wrap_generator(n) ],
-            keywords=[ ])
+            keywords=[ ],
+            starargs=None,
+            kwargs=None)
 
     def visit_List(self, n):
-
         if not isinstance(n.ctx, ast.Load):
             return self.generic_visit(n)
 
         return ast.Call(
             func=ast.Name(
-                id="__renpy__list__",
+                id=b("__renpy__list__"),
                 ctx=ast.Load()
                 ),
             args=[ self.generic_visit(n) ],
-            keywords=[ ])
+            keywords=[ ],
+            starargs=None,
+            kwargs=None)
 
     def visit_DictComp(self, n):
         return ast.Call(
             func=ast.Name(
-                id="__renpy__dict__",
+                id=b("__renpy__dict__"),
                 ctx=ast.Load()
                 ),
             args=[ self.wrap_generator(n) ],
-            keywords=[ ])
+            keywords=[ ],
+            starargs=None,
+            kwargs=None)
 
     def visit_Dict(self, n):
 
         return ast.Call(
             func=ast.Name(
-                id="__renpy__dict__",
+                id=b("__renpy__dict__"),
                 ctx=ast.Load()
                 ),
             args=[ self.generic_visit(n) ],
-            keywords=[ ])
+            keywords=[ ],
+            starargs=None,
+            kwargs=None)
+
 
     def visit_FormattedValue(self, n):
         n = wrap_formatted_value(n)
         return self.generic_visit(n)
 
-    def visit_Match(self, node : ast.Match) -> Any:
-        n : ast.Match = self.generic_visit(node) # type: ignore
-        n.cases = [ self.wrap_match_case(i) for i in n.cases ]
-        return n
-
-
-
 wrap_node = WrapNode()
 
 
@@ -876,86 +847,35 @@ py_compile_cache = { }
 old_py_compile_cache = { }
 
 
-def fix_locations(
-    node: ast.AST,
-    lineno_increment: int = 0,
-    col_offset: int = 0,
-    strip_ends: bool = False,
-    _parent_offset=(0, 0),
-    _first=True,
-):
+def fix_locations(node, lineno, col_offset):
     """
-    This function does `ast.increment_lineno`, `ast.fix_missing_locations`
-    and `[end_]col_offset` increments in one pass.
-
-    `lineno_increment`
-        Amount of lines to increment `lineno` by.
-
-    `col_offset`
-        0-indexed offset of source code. This should be used in case
-        source code have extra indent relative to compiled source.
-
-    `strip_ends`
-        If `True`, removes `end_lineno` and `end_col_offset`
-        so traceback does not render incorrect carets for given nodes tree.
+    Assigns locations to the given node, and all of its children, adding
+    any missing line numbers and column offsets.
     """
 
-    if _first:
-        if not isinstance(node, (ast.Module, ast.Expression)):
-            raise ValueError(f"'fix_locations' called with unsupported top level node: {node}")
+    start = max(
+        (lineno, col_offset),
+        (getattr(node, "lineno", None) or 1, getattr(node, "col_offset", None) or 0)
+    )
 
-    # TypeIgnore is a special case where lineno is not an attribute
-    # but rather a field of the node itself.
-    if isinstance(node, ast.TypeIgnore):
-        node.lineno = getattr(node, 'lineno', 0) + lineno_increment
-        return
+    lineno, col_offset = start
 
-    parent_lineno, parent_col_offset = _parent_offset
+    node.lineno = lineno
+    node.col_offset = col_offset
 
-    # lineno and col_offset must exist on every node.
-    # If there are none, we assume ast creator knows what they
-    # are doing and we can just assign to parent values.
-    if 'lineno' in node._attributes:
-        node = cast("ast.expr", node)
-        node.lineno = getattr(node, 'lineno', parent_lineno)
-        parent_lineno = node.lineno
-        node.lineno += lineno_increment
-
-    if 'col_offset' in node._attributes:
-        node = cast("ast.expr", node)
-        node.col_offset = getattr(node, 'col_offset', parent_col_offset)
-        parent_col_offset = node.col_offset
-        node.col_offset += col_offset
+    ends = [ start, (getattr(node, "end_lineno", None) or 1, getattr(node, "end_col_offset", None) or 0) ]
 
     for child in ast.iter_child_nodes(node):
-        fix_locations(
-            child,
-            lineno_increment,
-            col_offset,
-            strip_ends,
-            (parent_lineno, parent_col_offset),
-            False)
+        fix_locations(child, lineno, col_offset)
+        ends.append((child.end_lineno, child.end_col_offset))
 
-    # end_lineno and end_col_offset are optional, and we remove them if
-    # strip_ends is True (for generated code that doesn't have source).
-    if 'end_lineno' in node._attributes:
-        node = cast("ast.expr", node)
-        if strip_ends:
-            node.end_lineno = None
-        else:
-            end_lineno = getattr(node, 'end_lineno', None)
-            node.end_lineno = (end_lineno or parent_lineno) + lineno_increment
+    end = max(ends)
 
-    if 'end_col_offset' in node._attributes:
-        node = cast("ast.expr", node)
-        if strip_ends:
-            node.end_col_offset = None
-        else:
-            end_col_offset = getattr(node, 'end_col_offset', None)
-            node.end_col_offset = (end_col_offset or parent_col_offset) + col_offset
+    node.end_lineno = end[0]
+    node.end_col_offset = end[1]
 
 
-def quote_eval(s: str):
+def quote_eval(s):
     """
     Quotes a string for `eval`. This is necessary when it's in certain places,
     like as part of an argument string. We need to stick a single backslash
@@ -1050,15 +970,16 @@ compile_filename = ""
 
 def py_compile(source, mode, filename='<none>', lineno=1, ast_node=False, cache=True, py=None):
     """
-    Compiles the given source code using the supplied code generator.
+    Compiles the given source code using the supplied codegenerator.
     Lists, List Comprehensions, and Dictionaries are wrapped when
     appropriate.
 
     `source`
-        The source code, as a either a string or PyExpr.
+        The source code, as a either a string, pyexpr, or ast module
+        node.
 
     `mode`
-        One of "exec", "hide" or "eval".
+        One of "exec" or "eval".
 
     `filename`
         The filename the source comes from. If a pyexpr is given, the
@@ -1079,6 +1000,9 @@ def py_compile(source, mode, filename='<none>', lineno=1, ast_node=False, cache=
     if ast_node:
         cache = False
 
+    if isinstance(source, ast.Module):
+        return compile(source, filename, mode)
+
     if isinstance(source, renpy.ast.PyExpr):
         filename = source.filename
         lineno = source.linenumber
@@ -1087,7 +1011,10 @@ def py_compile(source, mode, filename='<none>', lineno=1, ast_node=False, cache=
             py = source.py
 
     if py is None:
-        py = 3
+        if PY2:
+            py = 2
+        else:
+            py = 3
 
     if cache:
         key = (lineno, filename, str(source), mode, renpy.script.MAGIC)
@@ -1136,26 +1063,42 @@ def py_compile(source, mode, filename='<none>', lineno=1, ast_node=False, cache=
             py_mode = mode
 
         flags = file_compiler_flags.get(filename, 0)
-        flags |= new_compile_flags
 
-        try:
-            with save_warnings():
-                tree = compile(source, filename, py_mode, ast.PyCF_ONLY_AST | flags, 1)
-        except SyntaxError as orig_e:
+        if (not PY2) or flags:
+
+            flags |= new_compile_flags
 
             try:
-                fixed_source = renpy.compat.fixes.fix_tokens(source)
                 with save_warnings():
-                    tree = compile(fixed_source, filename, py_mode, ast.PyCF_ONLY_AST | flags, 1)
+                    tree = compile(source, filename, py_mode, ast.PyCF_ONLY_AST | flags, 1)
+            except SyntaxError as orig_e:
+
+                try:
+                    fixed_source = renpy.compat.fixes.fix_tokens(source)
+                    with save_warnings():
+                        tree = compile(fixed_source, filename, py_mode, ast.PyCF_ONLY_AST | flags, 1)
+                except Exception:
+                    raise orig_e
+
+        else:
+
+            try:
+                flags = new_compile_flags
+                with save_warnings():
+                    tree = compile(source, filename, py_mode, ast.PyCF_ONLY_AST | flags, 1)
             except Exception:
-                raise orig_e
+                flags = old_compile_flags
+                source = escape_unicode(source)
+                with save_warnings():
+                    tree = compile(source, filename, py_mode, ast.PyCF_ONLY_AST | flags, 1)
 
         tree = wrap_node.visit(tree)
 
         if mode == "hide":
             wrap_hide(tree)
 
-        fix_locations(tree, line_offset, 0)
+        fix_locations(tree, 1, 0)
+        ast.increment_lineno(tree, lineno - 1)
 
         line_offset = 0
 
@@ -1168,7 +1111,7 @@ def py_compile(source, mode, filename='<none>', lineno=1, ast_node=False, cache=
         except SyntaxError as orig_e:
             try:
                 tree = renpy.compat.fixes.fix_ast(tree)
-                fix_locations(tree)
+                fix_locations(tree, 1, 0)
                 with save_warnings():
                     rv = compile(tree, filename, py_mode, flags, 1)
             except Exception:
@@ -1309,6 +1252,20 @@ class StoreProxy(object):
 def method_unpickle(obj, name):
     return getattr(obj, name)
 
+if PY2:
+
+    # Code for pickling bound methods.
+    def method_pickle(method):
+        name = method.im_func.__name__
+
+        obj = method.im_self
+
+        if obj is None:
+            obj = method.im_class
+
+        return method_unpickle, (obj, name)
+
+    copyreg.pickle(types.MethodType, method_pickle)
 
 # Code for pickling modules.
 

renpy/revertable.py

@@ -65,10 +65,18 @@ copyreg._reconstructor = _reconstructor # type: ignore
 # this to check to see if a background-save is valid.
 mutate_flag = True
 
+# In Python 2 functools.wraps does not check for the existence of WRAPPER_ASSIGNMENTS elements,
+# so for the C-defined methods we have AttributeError: 'method_descriptor' object has no attribute '__module__'.
+# To work around this, we only keep attributes that surely exist.
+if PY2:
+    def _method_wrapper(method):
+        return functools.wraps(method, ("__name__", "__doc__"), ())
+else:
+    _method_wrapper = functools.wraps # type: ignore
 
 def mutator(method):
 
-    @functools.wraps(method)
+    @_method_wrapper(method)
     def do_mutation(self, *args, **kwargs):
 
         global mutate_flag
@@ -168,7 +176,11 @@ class RevertableList(list):
         list.__init__(self, *args)
 
     __delitem__ = mutator(list.__delitem__)
+    if PY2:
+        __delslice__ = mutator(list.__delslice__) # type: ignore
     __setitem__ = mutator(list.__setitem__)
+    if PY2:
+        __setslice__ = mutator(list.__setslice__) # type: ignore
     __iadd__ = mutator(list.__iadd__)
     __imul__ = mutator(list.__imul__)
     append = mutator(list.append)
@@ -181,7 +193,7 @@ class RevertableList(list):
 
     def wrapper(method): # type: ignore
 
-        @functools.wraps(method)
+        @_method_wrapper(method)
         def newmethod(*args, **kwargs):
             l = method(*args, **kwargs) # type: ignore
             if l is NotImplemented:
@@ -191,6 +203,8 @@ class RevertableList(list):
         return newmethod
 
     __add__ = wrapper(list.__add__) # type: ignore
+    if PY2:
+        __getslice__ = wrapper(list.__getslice__) # type: ignore
     __mul__ = wrapper(list.__mul__) # type: ignore
     __rmul__ = wrapper(list.__rmul__) # type: ignore
 
@@ -274,31 +288,58 @@ class RevertableDict(dict):
     setdefault = mutator(dict.setdefault)
     update = mutator(dict.update)
 
-    itervalues = dict.values
-    iterkeys = dict.keys
-    iteritems = dict.items
+    if PY2:
 
-    def has_key(self, key):
-        return (key in self)
+        def keys(self):
+            rv = dict.keys(self)
 
-    # https://peps.python.org/pep-0584 methods
-    def __or__(self, other):
-        if not isinstance(other, dict):
-            return NotImplemented
-        rv = RevertableDict(self)
-        rv.update(other)
-        return rv
+            if (sys._getframe(1).f_code.co_flags & FUTURE_FLAGS) != FUTURE_FLAGS:
+                rv = RevertableList(rv)
 
-    def __ror__(self, other):
-        if not isinstance(other, dict):
-            return NotImplemented
-        rv = RevertableDict(other)
-        rv.update(self)
-        return rv
+            return rv
 
-    def __ior__(self, other):
-        self.update(other)
-        return self
+        def values(self):
+            rv = dict.values(self)
+
+            if (sys._getframe(1).f_code.co_flags & FUTURE_FLAGS) != FUTURE_FLAGS:
+                rv = RevertableList(rv)
+
+            return rv
+
+        def items(self):
+            rv = dict.items(self)
+
+            if (sys._getframe(1).f_code.co_flags & FUTURE_FLAGS) != FUTURE_FLAGS:
+                rv = RevertableList(rv)
+
+            return rv
+
+    else:
+        itervalues = dict.values
+        iterkeys = dict.keys
+        iteritems = dict.items
+
+        def has_key(self, key):
+            return (key in self)
+
+        # https://peps.python.org/pep-0584 methods
+        def __or__(self, other):
+            if not isinstance(other, dict):
+                return NotImplemented
+            rv = RevertableDict(self)
+            rv.update(other)
+            return rv
+
+        def __ror__(self, other):
+            if not isinstance(other, dict):
+                return NotImplemented
+            rv = RevertableDict(other)
+            rv.update(self)
+            return rv
+
+        def __ior__(self, other):
+            self.update(other)
+            return self
 
     def copy(self):
         rv = RevertableDict()
@@ -388,7 +429,7 @@ class RevertableSet(set):
 
     def wrapper(method): # type: ignore
 
-        @functools.wraps(method)
+        @_method_wrapper(method)
         def newmethod(*args, **kwargs):
             rv = method(*args, **kwargs) # type: ignore
             if isinstance(rv, set):
@@ -507,7 +548,7 @@ class MultiRevertable(object):
 
 def checkpointing(method):
 
-    @functools.wraps(method)
+    @_method_wrapper(method)
     def do_checkpoint(self, *args, **kwargs):
 
         renpy.game.context().force_checkpoint = True
@@ -519,7 +560,7 @@ def checkpointing(method):
 
 def list_wrapper(method):
 
-    @functools.wraps(method)
+    @_method_wrapper(method)
     def newmethod(*args, **kwargs):
         l = method(*args, **kwargs)
         return RevertableList(l)
@@ -551,7 +592,10 @@ class RollbackRandom(random.Random):
 
     setstate = checkpointing(mutator(random.Random.setstate))
 
-    choices = list_wrapper(random.Random.choices)
+    if PY2:
+        jumpahead = checkpointing(mutator(random.Random.jumpahead)) # type: ignore
+    else:
+        choices = list_wrapper(random.Random.choices)
     sample = list_wrapper(random.Random.sample)
 
     getrandbits = checkpointing(mutator(random.Random.getrandbits))
@@ -579,7 +623,8 @@ class DetRandom(random.Random):
         super(DetRandom, self).__init__()
         self.stack = [ ]
 
-    choices = list_wrapper(random.Random.choices)
+    if not PY2:
+        choices = list_wrapper(random.Random.choices)
     sample = list_wrapper(random.Random.sample)
 
     def random(self):

renpy/rollback.py

@@ -47,7 +47,10 @@ import renpy
 class StoreDeleted(object):
 
     def __reduce__(self):
-        return "deleted"
+        if PY2:
+            return b"deleted"
+        else:
+            return "deleted"
 
 
 deleted = StoreDeleted()

renpy/script.py

@@ -47,10 +47,20 @@ script_version = renpy.script_version
 # The version of the bytecode cache.
 BYTECODE_VERSION = 1
 
-from importlib.util import MAGIC_NUMBER as MAGIC
+# The python magic code.
+if PY2:
+    import heapq
+    import imp
+    MAGIC = imp.get_magic()
 
-# Change this to force a recompile when required.
-MAGIC += b'_v3.1'
+    # Change this to force a recompile when required.
+    MAGIC += b'_v2.1'
+
+else:
+    from importlib.util import MAGIC_NUMBER as MAGIC
+
+    # Change this to force a recompile when required.
+    MAGIC += b'_v3.1'
 
 # A string at the start of each rpycv2 file.
 RPYC2_HEADER = b"RENPY RPC2"
@@ -213,7 +223,10 @@ class Script(object):
 
             base, ext = os.path.splitext(short_fn)
 
-            hex_checksum = checksum[:8].hex()
+            if PY2:
+                hex_checksum = checksum[:8].encode("hex")
+            else:
+                hex_checksum = checksum[:8].hex()
 
             target_fn = os.path.join(
                 backupdir,
@@ -560,13 +573,12 @@ class Script(object):
                     if renpy.config.allow_duplicate_labels:
                         return
 
-                    import linecache
                     self.duplicate_labels.append(
                         u'The label {} is defined twice, at File "{}", line {}:\n{}and File "{}", line {}:\n{}'.format(
                             bad_name, old_node.filename, old_node.linenumber,
-                            linecache.getline(old_node.filename, old_node.linenumber),
+                            renpy.lexer.get_line_text(old_node.filename, old_node.linenumber),
                             bad_node.filename, bad_node.linenumber,
-                            linecache.getline(bad_node.filename, bad_node.linenumber),
+                            renpy.lexer.get_line_text(bad_node.filename, bad_node.linenumber),
                         ))
 
         self.update_bytecode()
@@ -1016,14 +1028,18 @@ class Script(object):
                         code = renpy.python.py_compile_eval_bytecode(i.source, filename=i.location[0], lineno=i.location[1], py=i.py)
 
                 except SyntaxError as e:
-                    assert e.filename is not None
-                    assert e.lineno is not None
+
+                    text = e.text
+
+                    if text is None:
+                        text = ''
+
                     pem = renpy.parser.ParseError(
-                        e.msg,
-                        e.filename,
-                        e.lineno, e.offset,
-                        e.text,
-                        e.end_lineno, e.end_offset)
+                        filename=e.filename,
+                        number=e.lineno,
+                        msg=e.msg,
+                        line=text,
+                        pos=e.offset)
 
                     renpy.parser.parse_errors.append(pem.message)
 

renpy/sl2/slast.py

@@ -1377,6 +1377,106 @@ class SLDisplayable(SLBlock):
             i.dump_const(prefix + "  ")
 
 
+class SLAddImage(SLDisplayable):
+    def __init__(self, loc, name):
+        super().__init__(loc, None, name=name, unique=False)
+
+    def execute(self, context):
+        debug = context.debug
+
+        cache = context.old_cache.get(self.serial, None) or context.miss_cache.get(self.serial, None)
+
+        # We use and check 'displayable', 'constant' and 'old_showif'
+        # fields, and rest are left unchanged.
+        if not isinstance(cache, SLCache):
+            cache = SLCache()
+
+        context.new_cache[self.serial] = cache
+
+        if debug:
+            self.debug_line()
+
+        # Add image does not use style or scope.
+        if cache.constant is not None:
+            d = cache.constant
+
+            if context.showif is not None:
+                d = self.wrap_in_showif(d, context, cache)
+
+            context.children.append(d)
+
+            if debug:
+                profile_log.write("    reused constant displayable")
+
+            return
+
+        # Create the context.
+        ctx = SLContext(context)
+
+        # True if we encountered an exception that we're recovering from
+        # due to being in prediction mode.
+        fail = False
+
+        # Result transform to display.
+        d = None
+
+        try:
+            ctx.keywords = {}
+
+            SLBlock.keywords(self, ctx)
+
+            d = ctx.keywords.pop("at")
+            assert not ctx.keywords, ctx.keywords
+
+            if debug:
+                self.report_arguments(cache, (), {}, None)
+
+            d._location = self.location
+
+            cache.copy_on_change = False # We no longer need to copy on change.
+
+            if debug:
+                if self.constant:
+                    profile_log.write("    created constant displayable")
+                else:
+                    profile_log.write("    created displayable")
+
+        except Exception:
+            if not context.predicting:
+                raise
+            fail = True
+
+        if self.variable is not None:
+            context.scope[self.variable] = d
+
+        old_d = cache.displayable
+
+        if old_d is not None:
+            d.take_state(old_d)
+            d.take_execution_state(old_d)
+
+        # If a failure occurred during prediction, predict main (if known),
+        # and return.
+        if fail:
+            predict_displayable(d)
+            context.fail = True
+            return
+
+        cache.displayable = d
+
+        if ctx.fail:
+            context.fail = True
+
+        else:
+            if self.constant:
+                cache.constant = d
+
+        if context.showif is not None:
+            d = self.wrap_in_showif(d, context, cache)
+
+        context.children.append(d)
+
+
 class SLIf(SLNode):
     """
     A screen language AST node that corresponds to an If/Elif/Else statement.

renpy/sl2/slparser.py

@@ -194,21 +194,42 @@ class Parser(object):
                 self.keyword[i.prefix + j + i.name] = i
 
         elif isinstance(i, Parser):
-            self.children[i.name] = i
+            self.children[tuple(i.name.split())] = i
 
     def parse_statement(self, loc, l, layout_mode=False, keyword=True):
-        word = l.word() or l.match(r'\$')
-
-        if word and word in self.children:
-            if layout_mode:
-                c = self.children[word].parse_layout(loc, l, self, keyword)
+        # Look for one-line python only once.
+        if l.match(r'\$'):
+            if ("$", ) in self.children:
+                assert not layout_mode, "One-line python can not be in layout mode."
+                return self.children["$", ].parse(loc, l, self, keyword)
             else:
-                c = self.children[word].parse(loc, l, self, keyword)
+                return None
 
-            return c
-        else:
+        name = ()
+        while True:
+            cp = l.checkpoint()
+            word = l.word()
+            if not word:
+                break
+
+            # Look for more specialized child parser.
+            new_name = (*name, word)
+            if new_name not in self.children:
+                l.revert(cp)
+                break
+
+            name = new_name
+
+        if not name:
             return None
 
+        if layout_mode:
+            c = self.children[name].parse_layout(loc, l, self, keyword)
+        else:
+            c = self.children[name].parse(loc, l, self, keyword)
+
+        return c
+
     def parse_layout(self, loc, l, parent, keyword):
         l.error("The %s statement cannot be used as a container for the has statement." % self.name)
 
@@ -229,7 +250,16 @@ class Parser(object):
 
         raise Exception("Not Implemented")
 
-    def parse_contents(self, l, target, layout_mode=False, can_has=False, can_tag=False, block_only=False, keyword=True):
+    def parse_contents(
+        self,
+        l, target,
+        layout_mode=False,
+        can_has=False,
+        can_tag=False,
+        block_only=False,
+        keyword=True,
+        line_only=False,
+    ):
         """
         Parses the remainder of the current line of `l`, and all of its subblock,
         looking for keywords and children.
@@ -247,6 +277,10 @@ class Parser(object):
 
         `block_only`
             If true, only parse the block and not the initial properties.
+
+        `line_only`
+            If true, only parse initial properties and not the block.
+            Lexer ends before the colon (if any).
         """
 
         seen_keywords = set()
@@ -330,10 +364,16 @@ class Parser(object):
             # If not block_only, we allow keyword arguments on the starting
             # line.
             while True:
+                cp = l.checkpoint()
                 if l.match(':'):
-                    l.expect_eol()
-                    l.expect_block(self.name)
-                    block = True
+                    # If line_only, we stop before the colon.
+                    if line_only:
+                        l.revert(cp)
+                        block = False
+                    else:
+                        l.expect_eol()
+                        l.expect_block(self.name)
+                        block = True
                     break
 
                 if l.eol():
@@ -700,6 +740,24 @@ class DisplayableParser(Parser):
         return rv
 
 
+class AddImageParser(Parser):
+    def __init__(self, name):
+        super().__init__(name)
+        self.variable = True
+
+    def parse(self, loc, l, parent, keyword):
+        rv = slast.SLAddImage(loc, self.name)
+
+        self.parse_contents(l, rv, line_only=True)
+        l.require(':')
+        l.expect_eol()
+        l.expect_block("ATL block")
+        rv.atl_transform = renpy.atl.parse_atl(l.subblock_lexer()) # type: ignore
+        return rv
+
+AddImageParser("add image")
+
+
 class IfParser(Parser):
 
     def __init__(self, name, node_type, parent_contents):

renpy/text/font.py

@@ -457,11 +457,7 @@ class ScaledImageFont(ImageFont):
             w, h = v.get_size()
             nw = scale(w)
             nh = scale(h)
-
-            if renpy.config.nearest_neighbor:
-                self.chars[k] = renpy.display.pgrender.transform_scale(v, (nw, nh))
-            else:
-                self.chars[k] = renpy.display.scale.smoothscale(v, (nw, nh))
+            self.chars[k] = renpy.display.scale.smoothscale(v, (nw, nh))
 
 
 def register_sfont(name=None, size=None, bold=False, italics=False, underline=False,

renpy/text/text.py

@@ -1406,8 +1406,7 @@ class Layout(object):
                     if value[0] in "+-":
                         push().size += int(value)
                     elif value[0] == "*":
-                        ts = push()
-                        ts.size = int(float(value[1:]) * ts.size)
+                        push().size = int(float(value[1:]) * push().size)
                     else:
                         push().size = int(value)
 
@@ -1736,13 +1735,6 @@ class Layout(object):
             The text shader.
         """
 
-        first_shader = ts
-
-        for l in lines:
-            for g in l.glyphs:
-                first_shader = g.shader
-                break
-
         tw, th = tex.get_size()
 
         if lines:
@@ -1753,17 +1745,12 @@ class Layout(object):
         # The number of glyphs in the mesh.
         n_glyphs = sum(len(l.glyphs) for l in lines)
 
-        mesh = renpy.gl2.gl2mesh2.Mesh2.text_mesh(n_glyphs + 2 * len(lines))
+        mesh = renpy.gl2.gl2mesh2.Mesh2.text_mesh(n_glyphs)
 
         # The y coordinate of the top line.
+
         top = 0
 
-        # The index of the last glyph to be shown.
-        last_index = 0
-
-        # The time of the last glyph to be shown.
-        last_time = 0.0
-
         for line in lines:
 
             # Line bottom is the y coordinate of the bottom line.
@@ -1779,39 +1766,8 @@ class Layout(object):
                 first_glyph = None
                 last_glyph = None
 
-            left = 0
-
-            if first_glyph:
-                right = first_glyph.x + self.add_left
-            else:
-                right = tw
-
-            # Generate a psuedo-glyph for the text to the left of the line.
-            # These pseudo-glyphs are used to make sure that outlines of lines above and below
-            # are displayed.
-
-            if outline:
-                if right > 0 and (ts == first_shader):
-
-                    cx = 0 + right / 2
-                    cy = outline + line.baseline
-
-                    mesh.add_glyph(
-                        tw, th,
-                        cx, cy,
-                        last_index,
-                        left, top, right, bottom,
-                        last_time, last_time,
-                        line.baseline, line.height - line.baseline,
-                        self.add_left, self.add_top,
-                    )
-
-            # Generate the actual glyphs.
-
             for g in line.glyphs:
 
-                left = right
-
                 if g.time == -1:
                     continue
 
@@ -1819,11 +1775,17 @@ class Layout(object):
                 if (g.shader is not ts) and (g.shader != ts):
                     continue
 
+                # The x-coordinate of the left edge of the glyph.
+                if g is first_glyph:
+                    left = g.x - self.add_left
+                else:
+                    left = g.x + outline
+
                 # The x-coordinate of the right edge of the glyph.
                 if g is last_glyph:
-                    right = g.x + g.advance + outline * 2 + self.add_left + self.add_right
+                    right = g.x + g.advance + outline * 2 + self.add_right
                 else:
-                    right = g.x + g.advance + outline + self.add_left
+                    right = g.x + g.advance + outline
 
                 if left < 0:
                     left = 0
@@ -1832,7 +1794,7 @@ class Layout(object):
 
                 # The center coordinates of the glyph. These aren't the
                 # actual center, but the center of the baseline.
-                cx = g.x + g.advance / 2 + self.add_left
+                cx = g.x + g.advance / 2
                 cy = outline + line.baseline
 
                 duration = g.duration
@@ -1853,33 +1815,8 @@ class Layout(object):
                     left, top, right, bottom,
                     left_time, right_time,
                     g.ascent, g.descent,
-                    self.add_left, self.add_top,
                 )
 
-                last_time = g.time
-                last_index = g.index
-
-            # Handle the empty space to the right of the last glyph.
-            if outline:
-
-                if right < tw and (ts == first_shader):
-
-                    left = right
-                    right = tw
-
-                    cx = left + right / 2
-                    cy = outline + line.baseline
-
-                    mesh.add_glyph(
-                        tw, th,
-                        cx, cy,
-                        last_index,
-                        left, top, right, bottom,
-                        last_time, last_time,
-                        line.baseline, line.height - line.baseline,
-                        self.add_left, self.add_top,
-                    )
-
             top = bottom
 
         main = (depth == 0)

renpy/translation/__init__.py

@@ -198,10 +198,10 @@ class ScriptTranslator(object):
                     if n.identifier in self.default_translates:
                         old_node = self.default_translates[n.identifier]
 
-                        renpy.lexer.ParseError(
-                            f"Line with id {n.identifier} appears twice. "
-                            f"The other line is {old_node.filename}:{old_node.linenumber}",
-                            n.filename, n.linenumber,).defer("duplicate_id")
+                        renpy.lexer.ParseError(n.filename, n.linenumber, "Line with id %s appears twice. The other line is %s:%d" % (
+                                    n.identifier,
+                                    old_node.filename, old_node.linenumber)
+                                    ).defer("duplicate_id")
 
                     self.default_translates[n.identifier] = n
                     self.file_translates[filename].append((label, n))

renpy/ui.py

@@ -442,7 +442,10 @@ screen = None # type: renpy.display.screen.ScreenDisplayable|None
 class Wrapper(renpy.object.Object):
 
     def __reduce__(self):
-        return self.name
+        if PY2:
+            return bytes(self.name) # type: ignore
+        else:
+            return self.name
 
     def __init__(self, function, one=False, many=False, imagemap=False, replaces=False, style=None, **kwargs):
 

renpy/versions.py

@@ -51,8 +51,8 @@ class Version(object):
 
 Version("main", 3, "8.4.0", "TBD")
 
-Version("fix", 3, "8.3.3", "Second Star to the Right")
-Version("fix", 2, "7.8.3", "Straight on Till Morning")
+Version("fix", 3, "8.3.1", "Second Star to the Right")
+Version("fix", 2, "7.8.1", "Straight on Till Morning")
 
 
 def make_dict(branch, suffix="00000000", official=False, nightly=False):

run.sh

@@ -2,22 +2,14 @@
 # This builds out of date modules using the default C compiler, and then
 # runs them.
 
-set -e
-
 export RENPY_CYTHON=cython
 
-ROOT="$(dirname $(realpath $0))"
+try () {
+    "$@" || exit -1
+}
+
 QUIET=${RENPY_QUIET- --quiet}
 
-renpy_branch=$(git branch --show-current)
-pushd "$ROOT/pygame_sdl2" >/dev/null
-pygame_sdl2_branch=$(git branch --show-current)
-popd >/dev/null
-
-if [ "$renpy_branch" != "$pygame_sdl2_branch" ]; then
-    echo "warning: Ren'Py branch: $renpy_branch, pygame_sdl2 branch: $pygame_sdl2_branch"
-fi
-
 if [ -n "$RENPY_COVERAGE" ]; then
     variant="renpy-coverage"
 else
@@ -33,20 +25,22 @@ if [ -z "$VIRTUAL_ENV" ] ; then
     exit 1
 fi
 
-BUILD_J="-j $(nproc)"
-ADAPT_TO_SETUPTOOLS="--old-and-unmanageable"
+PY_VERSION=$(python -c 'import sys; print(sys.version_info.major)')
+
+if [ $PY_VERSION != "2" ]; then
+    BUILD_J="-j $(nproc)"
+    ADAPT_TO_SETUPTOOLS="--old-and-unmanageable"
+fi
+
+ROOT="$(dirname $(realpath $0))"
 
 setup () {
     pushd $1 >/dev/null
 
-    python setup.py $QUIET \
+    try python setup.py $QUIET \
         build -b build/lib.$variant -t build/tmp.$variant $BUILD_J \
         $RENPY_BUILD_ARGS install $ADAPT_TO_SETUPTOOLS
 
-    if [ -e install_headers.py ]; then
-        python install_headers.py $VIRTUAL_ENV
-    fi
-
     popd >/dev/null
 }
 
@@ -54,8 +48,6 @@ if [ -e "$ROOT/pygame_sdl2" ]; then
     setup "$ROOT/pygame_sdl2/"
 fi
 
-
-
 if [ -e "$ROOT/cubism" ]; then
     export CUBISM="$ROOT/cubism"
     export CUBISM_PLATFORM=${CUBISM_PLATFORM:-linux/x86_64}