bpo-42955: Add Python/module_names.h (GH-24258)

Add a private list of all stdlib modules: _Py_module_names.

* Add Tools/scripts/generate_module_names.py script.
* Makefile: Add "make regen-module-names" command.
* setup.py: Add --list-module-names option.
* GitHub Action and Travis CI also runs "make regen-module-names",
  not ony "make regen-all", to ensure that the module names remains
  up to date.

1 files changed, 200 insertions, 0 deletions

diff --git a/Tools/scripts/generate_module_names.py b/Tools/scripts/generate_module_names.py
new file mode 100644
index 00000000000..985a1a5e5a2
--- /dev/null
+++ b/Tools/scripts/generate_module_names.py
@@ -0,0 +1,200 @@
+# This script lists the names of standard library modules
+# to update Python/module_names.h
+import os.path
+import re
+import subprocess
+import sys
+import sysconfig
+
+
+SRC_DIR = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
+STDLIB_PATH = os.path.join(SRC_DIR, 'Lib')
+MODULES_SETUP = os.path.join(SRC_DIR, 'Modules', 'Setup')
+SETUP_PY = os.path.join(SRC_DIR, 'setup.py')
+
+IGNORE = {
+    '__init__',
+    '__pycache__',
+    'site-packages',
+
+    # Helper modules of public modules.
+    # For example, sysconfig uses _osx_support.
+    '_aix_support',
+    '_collections_abc',
+    '_compat_pickle',
+    '_compression',
+    '_markupbase',
+    '_osx_support',
+    '_sitebuiltins',
+    '_strptime',
+    '_threading_local',
+    '_weakrefset',
+
+    # Used to bootstrap setup.py
+    '_bootsubprocess',
+
+    # pure Python implementation
+    '_py_abc',
+    '_pydecimal',
+    '_pyio',
+
+    # test modules
+    '__phello__.foo',
+    '_ctypes_test',
+    '_testbuffer',
+    '_testcapi',
+    '_testconsole',
+    '_testimportmultiple',
+    '_testinternalcapi',
+    '_testmultiphase',
+    '_xxtestfuzz',
+    'distutils.tests',
+    'idlelib.idle_test',
+    'lib2to3.tests',
+    'test',
+    'xxlimited',
+    'xxlimited_35',
+    'xxsubtype',
+}
+
+# Windows extension modules
+WINDOWS_MODULES = (
+    '_msi',
+    '_testconsole',
+    '_winapi',
+    'msvcrt',
+    'nt',
+    'winreg',
+    'winsound'
+)
+
+
+def write_comment(fp, comment):
+    print(f"// {comment}", file=fp)
+
+
+def write_modules(fp, names):
+    for name in sorted(names):
+        if name in IGNORE:
+            continue
+        print(f'"{name}",', file=fp)
+    print(file=fp)
+
+
+def list_builtin_modules(fp):
+    write_comment(fp, "Built-in modules")
+    write_modules(fp, sys.builtin_module_names)
+
+
+# Pure Python modules (Lib/*.py)
+def list_python_modules(fp):
+    write_comment(fp, "Pure Python modules (Lib/*.py)")
+    names = []
+    for filename in os.listdir(STDLIB_PATH):
+        if not filename.endswith(".py"):
+            continue
+        name = filename.removesuffix(".py")
+        names.append(name)
+    write_modules(fp, names)
+
+
+def _list_sub_packages(path, names, parent=None):
+    for name in os.listdir(path):
+        package_path = os.path.join(path, name)
+        if name in IGNORE:
+            continue
+        if not os.path.isdir(package_path):
+            continue
+        if not any(package_file.endswith(".py")
+                   for package_file in os.listdir(package_path)):
+            continue
+        if parent:
+            qualname = f"{parent}.{name}"
+        else:
+            qualname = name
+        if qualname in IGNORE:
+            continue
+        names.append(qualname)
+        _list_sub_packages(package_path, names, qualname)
+
+
+# Packages and sub-packages
+def list_packages(fp):
+    write_comment(fp, "Packages and sub-packages")
+    names = []
+    _list_sub_packages(STDLIB_PATH, names)
+    write_modules(fp, names)
+
+
+# Windows extensions
+def list_windows_extensions(fp):
+    write_comment(fp, "Windows extension modules")
+    write_modules(fp, WINDOWS_MODULES)
+
+
+# Extension modules built by setup.py
+def list_setup(fp):
+    cmd = [sys.executable, SETUP_PY, "-q", "build", "--list-module-names"]
+    output = subprocess.check_output(cmd)
+    output = output.decode("utf8")
+    names = output.splitlines()
+
+    write_comment(fp, "Extension modules built by setup.py")
+    write_modules(fp, names)
+
+
+# Built-in and extension modules built by Modules/Setup
+def list_modules_setup(fp):
+    assign_var = re.compile("^[A-Z]+=")
+
+    names = []
+    with open(MODULES_SETUP, encoding="utf-8") as modules_fp:
+        for line in modules_fp:
+            # Strip comment
+            line = line.partition("#")[0]
+            line = line.rstrip()
+            if not line:
+                continue
+            if assign_var.match(line):
+                # Ignore "VAR=VALUE"
+                continue
+            if line in ("*disabled*", "*shared*"):
+                continue
+            parts = line.split()
+            if len(parts) < 2:
+                continue
+            # "errno errnomodule.c" => write "errno"
+            name = parts[0]
+            names.append(name)
+
+    write_comment(fp, "Built-in and extension modules built by Modules/Setup")
+    write_modules(fp, names)
+
+
+def list_modules(fp):
+    print("// Auto-generated by Tools/scripts/generate_module_names.py.", file=fp)
+    print(file=fp)
+    print("static const char* _Py_module_names[] = {", file=fp)
+    print(file=fp)
+
+    list_builtin_modules(fp)
+    list_python_modules(fp)
+    list_packages(fp)
+    list_setup(fp)
+    list_modules_setup(fp)
+    list_windows_extensions(fp)
+
+    print("};", file=fp)
+
+
+def main():
+    if not sysconfig.is_python_build():
+        print(f"ERROR: {sys.executable} is not a Python build",
+              file=sys.stderr)
+        sys.exit(1)
+
+    list_modules(sys.stdout)
+
+
+if __name__ == "__main__":
+    main()