From ac83e934a13b6ac35d354506667e3e4c4416fbce Mon Sep 17 00:00:00 2001
From: Kassy <78169097+Kassy2048@users.noreply.github.com>
Date: Thu, 23 Jan 2025 21:01:32 +0100
Subject: [PATCH] Limit the number of cython processes ran in parallel

Running all the cython tasks in parallel at once is a bit violent. On my Docker
container for instance, it makes some processes be killed before they complete
because of resources exhaustion.

Using a ThreadPoolExecutor instance to limit the number of parallel tasks
guarantees there will be at least as much processes at the number of CPU plus 4
(up to 32) so that should be as fast as before without risking to reach the
host resources limit.
---
 scripts/setuplib.py | 23 +++++++++--------------
 1 file changed, 9 insertions(+), 14 deletions(-)

diff --git a/scripts/setuplib.py b/scripts/setuplib.py
index 9a5419ce9..e59d2339a 100644
--- a/scripts/setuplib.py
+++ b/scripts/setuplib.py
@@ -32,6 +32,7 @@ import warnings
 import pathlib
 import platform
 import subprocess
+from concurrent.futures import ThreadPoolExecutor
 
 import setuptools
 
@@ -302,24 +303,18 @@ def generate_all_cython():
     Run all of the cython that needs to be generated.
     """
 
-    threads = [ ]
-
-    for args in generate_cython_queue:
-
-        if "RENPY_CYTHON_SINGLETHREAD" in os.environ:
+    if "RENPY_CYTHON_SINGLETHREAD" in os.environ:
+        for args in generate_cython_queue:
             generate_cython(*args)
             if cython_failure:
                 sys.exit(1)
-        else:
-            t = threading.Thread(target=generate_cython, args=args)
-            t.start()
-            threads.append(t)
+    else:
+        with ThreadPoolExecutor() as executor:
+            for args in generate_cython_queue:
+                executor.submit(generate_cython, *args)
 
-    for t in threads:
-        t.join()
-
-    if cython_failure:
-        sys.exit(1)
+        if cython_failure:
+            sys.exit(1)
 
 
 def find_unnecessary_gen():