From dfc2e065a2e71011017077e549cd2f9bf4944c54 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Sat, 11 Feb 2023 22:07:52 -0800 Subject: [PATCH] gh-89792: Limit test_tools freeze test build parallelism based on the number of cores (#101841) unhardcode freeze test build parallelism. base it on the number of cpus, don't use more than max(2, os.cpu_count()/3). --- .../2023-02-11-20-28-08.gh-issue-89792.S-Y5BZ.rst | 7 ++++--- Tools/freeze/test/freeze.py | 15 ++++++++++++--- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/Misc/NEWS.d/next/Tests/2023-02-11-20-28-08.gh-issue-89792.S-Y5BZ.rst b/Misc/NEWS.d/next/Tests/2023-02-11-20-28-08.gh-issue-89792.S-Y5BZ.rst index a3a3070d7f3790..9de278919ef2f8 100644 --- a/Misc/NEWS.d/next/Tests/2023-02-11-20-28-08.gh-issue-89792.S-Y5BZ.rst +++ b/Misc/NEWS.d/next/Tests/2023-02-11-20-28-08.gh-issue-89792.S-Y5BZ.rst @@ -1,3 +1,4 @@ -``test_tools`` now copies up to 10x less source data to a temporary -directory during the ``freeze`` test by ignoring git metadata and other -artifacts. +``test_tools`` now copies up to 10x less source data to a temporary directory +during the ``freeze`` test by ignoring git metadata and other artifacts. It +also limits its python build parallelism based on os.cpu_count instead of hard +coding it as 8 cores. diff --git a/Tools/freeze/test/freeze.py b/Tools/freeze/test/freeze.py index 0ae983b15c98f0..b4c76ff36a873b 100644 --- a/Tools/freeze/test/freeze.py +++ b/Tools/freeze/test/freeze.py @@ -163,16 +163,25 @@ def prepare(script=None, outdir=None): if not MAKE: raise UnsupportedError('make') + cores = os.cpu_count() + if cores and cores >= 3: + # this test is most often run as part of the whole suite with a lot + # of other tests running in parallel, from 1-2 vCPU systems up to + # people's NNN core beasts. Don't attempt to use it all. + parallel = f'-j{cores*2//3}' + else: + parallel = '-j2' + # Build python. - print(f'building python in {builddir}...') + print(f'building python {parallel=} in {builddir}...') if os.path.exists(os.path.join(srcdir, 'Makefile')): # Out-of-tree builds require a clean srcdir. _run_quiet([MAKE, '-C', srcdir, 'clean']) - _run_quiet([MAKE, '-C', builddir, '-j8']) + _run_quiet([MAKE, '-C', builddir, parallel]) # Install the build. print(f'installing python into {prefix}...') - _run_quiet([MAKE, '-C', builddir, '-j8', 'install']) + _run_quiet([MAKE, '-C', builddir, 'install']) python = os.path.join(prefix, 'bin', 'python3') return outdir, scriptfile, python