From 3f1b74507366632867fc837f9f0487509772678a Mon Sep 17 00:00:00 2001 From: aicodeguard Date: Fri, 19 Sep 2025 13:52:22 +0800 Subject: [PATCH] Tools changes Signed-off-by: aicodeguard --- Tools/build/.ruff.toml | 41 +++ Tools/build/.warningignore_macos | 9 + Tools/build/.warningignore_ubuntu | 5 + Tools/build/check_extension_modules.py | 15 +- Tools/build/check_warnings.py | 316 +++++++++++++++++++ Tools/build/deepfreeze.py | 21 +- Tools/build/freeze_modules.py | 12 +- Tools/build/generate-build-details.py | 194 ++++++++++++ Tools/build/generate_global_objects.py | 13 +- Tools/build/generate_levenshtein_examples.py | 3 +- Tools/build/generate_re_casefix.py | 4 +- Tools/build/generate_sbom.py | 21 +- Tools/build/generate_sre_constants.py | 2 +- Tools/build/generate_stdlib_module_names.py | 2 - Tools/build/generate_token.py | 12 +- Tools/build/mypy.ini | 1 + Tools/build/parse_html5_entities.py | 18 +- Tools/build/regen-configure.sh | 2 +- Tools/build/smelly.py | 11 +- Tools/build/stable_abi.py | 80 +++-- Tools/build/umarshal.py | 18 +- Tools/c-analyzer/TODO | 8 +- Tools/c-analyzer/c_analyzer/datafiles.py | 7 +- Tools/c-analyzer/c_parser/parser/__init__.py | 2 +- Tools/c-analyzer/c_parser/parser/_regexes.py | 11 + Tools/c-analyzer/cpython/_analyzer.py | 14 + Tools/c-analyzer/cpython/_parser.py | 14 +- Tools/c-analyzer/cpython/globals-to-fix.tsv | 69 +--- Tools/c-analyzer/cpython/ignored.tsv | 42 ++- Tools/clinic/.ruff.toml | 3 +- Tools/clinic/libclinic/clanguage.py | 8 +- Tools/clinic/libclinic/converter.py | 2 +- Tools/clinic/libclinic/converters.py | 308 +++++++++++------- Tools/clinic/libclinic/dsl_parser.py | 90 ++---- Tools/clinic/libclinic/function.py | 1 + Tools/clinic/libclinic/parse_args.py | 165 +++++----- 36 files changed, 1086 insertions(+), 458 deletions(-) create mode 100644 Tools/build/.ruff.toml create mode 100644 Tools/build/.warningignore_macos create mode 100644 Tools/build/.warningignore_ubuntu create mode 100644 Tools/build/check_warnings.py create mode 100644 Tools/build/generate-build-details.py diff --git a/Tools/build/.ruff.toml b/Tools/build/.ruff.toml new file mode 100644 index 0000000..dcbf293 --- /dev/null +++ b/Tools/build/.ruff.toml @@ -0,0 +1,41 @@ +extend = "../../.ruff.toml" # Inherit the project-wide settings + +[per-file-target-version] +"deepfreeze.py" = "py311" # requires `code.co_exceptiontable` +"stable_abi.py" = "py311" # requires 'tomllib' + +[format] +preview = true +docstring-code-format = true + +[lint] +select = [ + "C4", # flake8-comprehensions + "E", # pycodestyle + "F", # pyflakes + "I", # isort + "ISC", # flake8-implicit-str-concat + "LOG", # flake8-logging + "PGH", # pygrep-hooks + "PT", # flake8-pytest-style + "PYI", # flake8-pyi + "RUF100", # Ban unused `# noqa` comments + "UP", # pyupgrade + "W", # pycodestyle + "YTT", # flake8-2020 +] +ignore = [ + "E501", # Line too long + "F541", # f-string without any placeholders + "PYI024", # Use `typing.NamedTuple` instead of `collections.namedtuple` + "PYI025", # Use `from collections.abc import Set as AbstractSet` + "UP038", # Use `X | Y` in `isinstance` call instead of `(X, Y)` +] + +[lint.per-file-ignores] +"{check_extension_modules,freeze_modules}.py" = [ + "UP031", # Use format specifiers instead of percent format +] +"generate_{re_casefix,sre_constants,token}.py" = [ + "UP031", # Use format specifiers instead of percent format +] diff --git a/Tools/build/.warningignore_macos b/Tools/build/.warningignore_macos new file mode 100644 index 0000000..d7b62bc --- /dev/null +++ b/Tools/build/.warningignore_macos @@ -0,0 +1,9 @@ +# Files listed will be ignored by the compiler warning checker +# for the macOS/build and test job. +# Keep lines sorted lexicographically to help avoid merge conflicts. +# Format example: +# /path/to/file (number of warnings in file) +Modules/expat/siphash.h 7 +Modules/expat/xmlparse.c 13 +Modules/expat/xmltok.c 3 +Modules/expat/xmltok_impl.c 26 diff --git a/Tools/build/.warningignore_ubuntu b/Tools/build/.warningignore_ubuntu new file mode 100644 index 0000000..469c727 --- /dev/null +++ b/Tools/build/.warningignore_ubuntu @@ -0,0 +1,5 @@ +# Files listed will be ignored by the compiler warning checker +# for the Ubuntu/build and test job. +# Keep lines sorted lexicographically to help avoid merge conflicts. +# Format example: +# /path/to/file (number of warnings in file) diff --git a/Tools/build/check_extension_modules.py b/Tools/build/check_extension_modules.py index e0c7a92..668db8d 100644 --- a/Tools/build/check_extension_modules.py +++ b/Tools/build/check_extension_modules.py @@ -30,10 +30,15 @@ import sys import sysconfig import warnings - from collections.abc import Iterable -from importlib._bootstrap import _load as bootstrap_load # type: ignore[attr-defined] -from importlib.machinery import BuiltinImporter, ExtensionFileLoader, ModuleSpec +from importlib._bootstrap import ( # type: ignore[attr-defined] + _load as bootstrap_load, +) +from importlib.machinery import ( + BuiltinImporter, + ExtensionFileLoader, + ModuleSpec, +) from importlib.util import spec_from_file_location, spec_from_loader from typing import NamedTuple @@ -201,7 +206,7 @@ def print_three_column(modinfos: list[ModuleInfo]) -> None: # guarantee zip() doesn't drop anything while len(names) % 3: names.append("") - for l, m, r in zip(names[::3], names[1::3], names[2::3]): + for l, m, r in zip(names[::3], names[1::3], names[2::3]): # noqa: E741 print("%-*s %-*s %-*s" % (longest, l, longest, m, longest, r)) if verbose and self.builtin_ok: @@ -433,7 +438,7 @@ def check_module_import(self, modinfo: ModuleInfo) -> None: except ImportError as e: logger.error("%s failed to import: %s", modinfo.name, e) raise - except Exception as e: + except Exception: if not hasattr(_imp, 'create_dynamic'): logger.warning("Dynamic extension '%s' ignored", modinfo.name) return diff --git a/Tools/build/check_warnings.py b/Tools/build/check_warnings.py new file mode 100644 index 0000000..3f49d8e --- /dev/null +++ b/Tools/build/check_warnings.py @@ -0,0 +1,316 @@ +""" +Parses compiler output from Clang or GCC and checks that warnings +exist only in files that are expected to have warnings. +""" + +import argparse +import re +import sys +from collections import defaultdict +from pathlib import Path +from typing import NamedTuple + + +class IgnoreRule(NamedTuple): + file_path: str + count: int + ignore_all: bool = False + is_directory: bool = False + + +def parse_warning_ignore_file(file_path: str) -> set[IgnoreRule]: + """ + Parses the warning ignore file and returns a set of IgnoreRules + """ + files_with_expected_warnings = set() + with Path(file_path).open(encoding="UTF-8") as ignore_rules_file: + files_with_expected_warnings = set() + for i, line in enumerate(ignore_rules_file): + line = line.strip() + if line and not line.startswith("#"): + line_parts = line.split() + if len(line_parts) >= 2: + file_name = line_parts[0] + count = line_parts[1] + ignore_all = count == "*" + is_directory = file_name.endswith("/") + + # Directories must have a wildcard count + if is_directory and count != "*": + print( + f"Error parsing ignore file: {file_path} " + f"at line: {i}" + ) + print( + f"Directory {file_name} must have count set to *" + ) + sys.exit(1) + if ignore_all: + count = 0 + + files_with_expected_warnings.add( + IgnoreRule( + file_name, int(count), ignore_all, is_directory + ) + ) + + return files_with_expected_warnings + + +def extract_warnings_from_compiler_output( + compiler_output: str, + compiler_output_type: str, + path_prefix: str = "", +) -> list[dict]: + """ + Extracts warnings from the compiler output based on compiler + output type. Removes path prefix from file paths if provided. + Compatible with GCC and Clang compiler output. + """ + # Choose pattern and compile regex for particular compiler output + if compiler_output_type == "gcc": + regex_pattern = ( + r"(?P.*):(?P\d+):(?P\d+): warning: " + r"(?P.*?)(?: (?P