Limit the number of cython processes ran in parallel
Running all the cython tasks in parallel at once is a bit violent. On my Docker container for instance, it makes some processes be killed before they complete because of resources exhaustion. Using a ThreadPoolExecutor instance to limit the number of parallel tasks guarantees there will be at least as much processes at the number of CPU plus 4 (up to 32) so that should be as fast as before without risking to reach the host resources limit.
This commit is contained in:
+9
-14
@@ -32,6 +32,7 @@ import warnings
|
||||
import pathlib
|
||||
import platform
|
||||
import subprocess
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
import setuptools
|
||||
|
||||
@@ -302,24 +303,18 @@ def generate_all_cython():
|
||||
Run all of the cython that needs to be generated.
|
||||
"""
|
||||
|
||||
threads = [ ]
|
||||
|
||||
for args in generate_cython_queue:
|
||||
|
||||
if "RENPY_CYTHON_SINGLETHREAD" in os.environ:
|
||||
if "RENPY_CYTHON_SINGLETHREAD" in os.environ:
|
||||
for args in generate_cython_queue:
|
||||
generate_cython(*args)
|
||||
if cython_failure:
|
||||
sys.exit(1)
|
||||
else:
|
||||
t = threading.Thread(target=generate_cython, args=args)
|
||||
t.start()
|
||||
threads.append(t)
|
||||
else:
|
||||
with ThreadPoolExecutor() as executor:
|
||||
for args in generate_cython_queue:
|
||||
executor.submit(generate_cython, *args)
|
||||
|
||||
for t in threads:
|
||||
t.join()
|
||||
|
||||
if cython_failure:
|
||||
sys.exit(1)
|
||||
if cython_failure:
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def find_unnecessary_gen():
|
||||
|
||||
Reference in New Issue
Block a user