From 8974ca8f2440f6c715d74de2b74f655ae48548d0 Mon Sep 17 00:00:00 2001 From: Campbell Barton Date: Fri, 6 Oct 2023 20:22:06 +1100 Subject: [PATCH] Tools: set the number of jobs to the CPU count for various utilities Previously this was the double the CPU count because: - Modern CPU's from AMD & Intel support SMT/hyper-threading which present twice as many cores, doubling again has little to no benefit. - Using 2x or 4x the number of physical cores number can use a lot of memory on systems with many cores which are becoming more common. --- build_files/cmake/cmake_static_check_clang.py | 2 +- tools/check_source/check_licenses.py | 2 +- tools/utils/addr2line_backtrace.py | 2 +- tools/utils/authors_git_gen.py | 2 +- tools/utils/autopep8_clean.py | 2 +- tools/utils/credits_git_gen.py | 2 +- tools/utils_maintenance/code_clean.py | 2 +- tools/utils_maintenance/modules/batch_edit_text.py | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/build_files/cmake/cmake_static_check_clang.py b/build_files/cmake/cmake_static_check_clang.py index fd46d0d7840..0f39e816098 100644 --- a/build_files/cmake/cmake_static_check_clang.py +++ b/build_files/cmake/cmake_static_check_clang.py @@ -492,7 +492,7 @@ def run_checks_on_project( import multiprocessing if jobs <= 0: - jobs = multiprocessing.cpu_count() * 2 + jobs = multiprocessing.cpu_count() if jobs > 1: with multiprocessing.Pool(processes=jobs) as pool: diff --git a/tools/check_source/check_licenses.py b/tools/check_source/check_licenses.py index bfa64369235..f1d5d6c1e81 100644 --- a/tools/check_source/check_licenses.py +++ b/tools/check_source/check_licenses.py @@ -557,7 +557,7 @@ def main() -> None: import multiprocessing job_total = multiprocessing.cpu_count() - pool = multiprocessing.Pool(processes=job_total * 2) + pool = multiprocessing.Pool(processes=job_total) pool.map(operation_wrap, filepath_args) else: for filepath in [ diff --git a/tools/utils/addr2line_backtrace.py b/tools/utils/addr2line_backtrace.py index a45a97b2f33..9fea0de8ea1 100755 --- a/tools/utils/addr2line_backtrace.py +++ b/tools/utils/addr2line_backtrace.py @@ -205,7 +205,7 @@ def main() -> None: jobs = args.jobs if jobs <= 0: - jobs = multiprocessing.cpu_count() * 2 + jobs = multiprocessing.cpu_count() base_path = args.base if not base_path: diff --git a/tools/utils/authors_git_gen.py b/tools/utils/authors_git_gen.py index 9c247e07a47..65201636971 100755 --- a/tools/utils/authors_git_gen.py +++ b/tools/utils/authors_git_gen.py @@ -367,7 +367,7 @@ def main() -> None: if jobs <= 0: # Clamp the value, higher values give errors with too many open files. # Allow users to manually pass very high values in as they might want to tweak system limits themselves. - jobs = min(multiprocessing.cpu_count() * 2, 400) + jobs = min(multiprocessing.cpu_count(), 400) credits.process(GitCommitIter(args.source_dir, commit_range), jobs=jobs) diff --git a/tools/utils/autopep8_clean.py b/tools/utils/autopep8_clean.py index ab2cf935fa2..50d059e4021 100755 --- a/tools/utils/autopep8_clean.py +++ b/tools/utils/autopep8_clean.py @@ -106,7 +106,7 @@ def main() -> None: if USE_MULTIPROCESS: import multiprocessing job_total = multiprocessing.cpu_count() - pool = multiprocessing.Pool(processes=job_total * 2) + pool = multiprocessing.Pool(processes=job_total) pool.map(autopep8_format_file, paths) else: for f in paths: diff --git a/tools/utils/credits_git_gen.py b/tools/utils/credits_git_gen.py index 722ac4e1aba..e4922dae361 100755 --- a/tools/utils/credits_git_gen.py +++ b/tools/utils/credits_git_gen.py @@ -358,7 +358,7 @@ def main() -> None: if jobs <= 0: # Clamp the value, higher values give errors with too many open files. # Allow users to manually pass very high values in as they might want to tweak system limits themselves. - jobs = min(multiprocessing.cpu_count() * 2, 400) + jobs = min(multiprocessing.cpu_count(), 400) credits.process(GitCommitIter(args.source_dir, commit_range), jobs=jobs) diff --git a/tools/utils_maintenance/code_clean.py b/tools/utils_maintenance/code_clean.py index 6dbf37f0c3a..79083346b56 100755 --- a/tools/utils_maintenance/code_clean.py +++ b/tools/utils_maintenance/code_clean.py @@ -1882,7 +1882,7 @@ def run_edits_on_directory( return 1 if jobs <= 0: - jobs = multiprocessing.cpu_count() * 2 + jobs = multiprocessing.cpu_count() if args is None: # Error will have been reported. diff --git a/tools/utils_maintenance/modules/batch_edit_text.py b/tools/utils_maintenance/modules/batch_edit_text.py index 72a2c720a61..99fe4c42d43 100644 --- a/tools/utils_maintenance/modules/batch_edit_text.py +++ b/tools/utils_maintenance/modules/batch_edit_text.py @@ -57,7 +57,7 @@ def run( ] import multiprocessing job_total = multiprocessing.cpu_count() - pool = multiprocessing.Pool(processes=job_total * 2) + pool = multiprocessing.Pool(processes=job_total) pool.starmap(operation_wrap, args) else: for directory in directories: