Limit the number of cython processes ran in parallel

Running all the cython tasks in parallel at once is a bit violent. On my Docker
container for instance, it makes some processes be killed before they complete
because of resources exhaustion.

Using a ThreadPoolExecutor instance to limit the number of parallel tasks
guarantees there will be at least as much processes at the number of CPU plus 4
(up to 32) so that should be as fast as before without risking to reach the
host resources limit.
This commit is contained in:
Kassy
2025-01-23 21:01:32 +01:00
parent 9ce1334039
commit ac83e934a1
+9 -14
View File
@@ -32,6 +32,7 @@ import warnings
import pathlib
import platform
import subprocess
from concurrent.futures import ThreadPoolExecutor
import setuptools
@@ -302,24 +303,18 @@ def generate_all_cython():
Run all of the cython that needs to be generated.
"""
threads = [ ]
for args in generate_cython_queue:
if "RENPY_CYTHON_SINGLETHREAD" in os.environ:
if "RENPY_CYTHON_SINGLETHREAD" in os.environ:
for args in generate_cython_queue:
generate_cython(*args)
if cython_failure:
sys.exit(1)
else:
t = threading.Thread(target=generate_cython, args=args)
t.start()
threads.append(t)
else:
with ThreadPoolExecutor() as executor:
for args in generate_cython_queue:
executor.submit(generate_cython, *args)
for t in threads:
t.join()
if cython_failure:
sys.exit(1)
if cython_failure:
sys.exit(1)
def find_unnecessary_gen():