Make x.py work again in most (all?) cases
Wrap all of x.py in `if __name__ == '__main__':` to avoid problems with `multiprocessing` Make the pool sizing better
This commit is contained in:
parent
4b87ed9d0f
commit
30119498be
2 changed files with 40 additions and 19 deletions
|
@ -28,6 +28,20 @@ if platform_is_win32():
|
||||||
else:
|
else:
|
||||||
EXE_SUFFIX = ""
|
EXE_SUFFIX = ""
|
||||||
|
|
||||||
|
def get_cpus():
|
||||||
|
if hasattr(os, "sched_getaffinity"):
|
||||||
|
return len(os.sched_getaffinity(0))
|
||||||
|
if hasattr(os, "cpu_count"):
|
||||||
|
cpus = os.cpu_count()
|
||||||
|
if cpus is not None:
|
||||||
|
return cpus
|
||||||
|
try:
|
||||||
|
return cpu_count()
|
||||||
|
except NotImplementedError:
|
||||||
|
return 1
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
def get(base, url, path, checksums, verbose=False):
|
def get(base, url, path, checksums, verbose=False):
|
||||||
with tempfile.NamedTemporaryFile(delete=False) as temp_file:
|
with tempfile.NamedTemporaryFile(delete=False) as temp_file:
|
||||||
temp_path = temp_file.name
|
temp_path = temp_file.name
|
||||||
|
@ -540,11 +554,15 @@ class RustBuild(object):
|
||||||
|
|
||||||
# Unpack the tarballs in parallle.
|
# Unpack the tarballs in parallle.
|
||||||
# In Python 2.7, Pool cannot be used as a context manager.
|
# In Python 2.7, Pool cannot be used as a context manager.
|
||||||
p = Pool(min(len(tarballs_download_info), cpu_count()))
|
pool_size = min(len(tarballs_download_info), get_cpus())
|
||||||
|
if self.verbose:
|
||||||
|
print('Choosing a pool size of', pool_size, 'for the unpacking of the tarballs')
|
||||||
|
p = Pool(pool_size)
|
||||||
try:
|
try:
|
||||||
p.map(unpack_component, tarballs_download_info)
|
p.map(unpack_component, tarballs_download_info)
|
||||||
finally:
|
finally:
|
||||||
p.close()
|
p.close()
|
||||||
|
p.join()
|
||||||
|
|
||||||
if self.should_fix_bins_and_dylibs():
|
if self.should_fix_bins_and_dylibs():
|
||||||
self.fix_bin_or_dylib("{}/bin/cargo".format(bin_root))
|
self.fix_bin_or_dylib("{}/bin/cargo".format(bin_root))
|
||||||
|
|
3
x.py
3
x.py
|
@ -4,6 +4,9 @@
|
||||||
|
|
||||||
# This file is only a "symlink" to bootstrap.py, all logic should go there.
|
# This file is only a "symlink" to bootstrap.py, all logic should go there.
|
||||||
|
|
||||||
|
# Parts of `bootstrap.py` use the `multiprocessing` module, so this entry point
|
||||||
|
# must use the normal `if __name__ == '__main__':` convention to avoid problems.
|
||||||
|
if __name__ == '__main__':
|
||||||
import os
|
import os
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue