From 0c3565a046cc040aed12a8e113d019e4fbe8f4d0 Mon Sep 17 00:00:00 2001 From: kai lin Date: Tue, 31 Mar 2026 16:19:53 -0400 Subject: [PATCH 1/3] init --- tools/scripts/codegen/format_util.py | 39 ++++++++++++++++++---------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/tools/scripts/codegen/format_util.py b/tools/scripts/codegen/format_util.py index 050cd7c38396..9ea8b7a34c41 100644 --- a/tools/scripts/codegen/format_util.py +++ b/tools/scripts/codegen/format_util.py @@ -9,32 +9,43 @@ import os import pathlib import re +import subprocess from subprocess import list2cmdline, run from tempfile import NamedTemporaryFile CLANG_FORMAT_VERSION = '18.1.6' CLANG_FORMAT_INCLUDE_REGEX = re.compile(r'^.*\.(cpp|h)$') - def format_directories(directory_list): """Format C++ files in the given directories using clang-format""" - filepaths_file = NamedTemporaryFile(delete=False) - + cpu_count = os.cpu_count() or 1 + + filepaths_files = [NamedTemporaryFile(delete=False) for _ in range(cpu_count)] + + index = 0 for root_dir in directory_list: if os.path.exists(root_dir): for dirpath, dirnames, filenames in os.walk(root_dir): for filename in filenames: filepath = pathlib.Path(dirpath, filename).as_posix() if CLANG_FORMAT_INCLUDE_REGEX.match(filename): - filepaths_file.write(f"{filepath}\n".encode()) - - filepaths_file.close() - - cmd = ['pipx', 'run', f'clang-format=={CLANG_FORMAT_VERSION}', - f'--files={filepaths_file.name}', '-i', '-style=file:.clang-format'] - - print(f"Formatting generated files: {list2cmdline(cmd)}") - run(cmd) - + filepaths_files[index % cpu_count].write(f"{filepath}\n".encode()) + index += 1 + + for filepaths_file in filepaths_files: + filepaths_file.close() + + processes = [] + for filepaths_file in filepaths_files: + cmd = ['pipx', 'run', f'clang-format=={CLANG_FORMAT_VERSION}', + f'--files={filepaths_file.name}', '-i', '-style=file:.clang-format'] + p = subprocess.Popen(cmd) + processes.append(p) + print(f"Formatting generated files: {list2cmdline(cmd)}") + + for p in processes: + p.wait() + # Clean up temp file - os.unlink(filepaths_file.name) \ No newline at end of file + for filepaths_file in filepaths_files: + os.unlink(filepaths_file.name) \ No newline at end of file From d239eaf20c0e55b0ee9af304c8025455d23ad11b Mon Sep 17 00:00:00 2001 From: kai lin Date: Tue, 31 Mar 2026 16:47:36 -0400 Subject: [PATCH 2/3] adding warmup --- tools/scripts/codegen/format_util.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/scripts/codegen/format_util.py b/tools/scripts/codegen/format_util.py index 9ea8b7a34c41..0c0116c4741b 100644 --- a/tools/scripts/codegen/format_util.py +++ b/tools/scripts/codegen/format_util.py @@ -35,6 +35,10 @@ def format_directories(directory_list): for filepaths_file in filepaths_files: filepaths_file.close() + # Warm up pipx cache to avoid race conditions when spawning parallel processes + subprocess.run(['pipx', 'run', f'clang-format=={CLANG_FORMAT_VERSION}', '--version'], + check=True, capture_output=True) + processes = [] for filepaths_file in filepaths_files: cmd = ['pipx', 'run', f'clang-format=={CLANG_FORMAT_VERSION}', From f86a49a7b26b830cbd99099482159137b34bfbf7 Mon Sep 17 00:00:00 2001 From: kai lin Date: Wed, 1 Apr 2026 16:39:29 -0400 Subject: [PATCH 3/3] error code --- tools/scripts/codegen/format_util.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/tools/scripts/codegen/format_util.py b/tools/scripts/codegen/format_util.py index 0c0116c4741b..c6585ae543df 100644 --- a/tools/scripts/codegen/format_util.py +++ b/tools/scripts/codegen/format_util.py @@ -35,20 +35,25 @@ def format_directories(directory_list): for filepaths_file in filepaths_files: filepaths_file.close() - # Warm up pipx cache to avoid race conditions when spawning parallel processes - subprocess.run(['pipx', 'run', f'clang-format=={CLANG_FORMAT_VERSION}', '--version'], - check=True, capture_output=True) - processes = [] for filepaths_file in filepaths_files: cmd = ['pipx', 'run', f'clang-format=={CLANG_FORMAT_VERSION}', f'--files={filepaths_file.name}', '-i', '-style=file:.clang-format'] - p = subprocess.Popen(cmd) - processes.append(p) + p = subprocess.Popen(cmd, stderr=subprocess.PIPE) + processes.append((p, cmd)) print(f"Formatting generated files: {list2cmdline(cmd)}") - for p in processes: - p.wait() + failed = False + for p, cmd in processes: + _, stderr = p.communicate() + if p.returncode != 0: + print(f"ERROR: clang-format failed with exit code {p.returncode}: {list2cmdline(cmd)}") + if stderr: + print(f" stderr: {stderr.decode().strip()}") + failed = True + + if failed: + raise RuntimeError("One or more clang-format processes failed") # Clean up temp file for filepaths_file in filepaths_files: