dist.sphinxext: new sphinx extension

Small sphinx extension to generate docs from argparse scripts.
Simplifies all `conf.py` across all pkgcore stack.

Signed-off-by: Arthur Zamarin <arthurzam@gentoo.org>

5 files changed, 133 insertions, 112 deletions

diff --git a/.coveragerc b/.coveragerc
index 18303775..b24d5133 100644
--- a/.coveragerc
+++ b/.coveragerc
@@ -1,7 +1,7 @@
 [run]
 source = snakeoil
 branch = True
-omit = src/*, tests/*
+omit = src/*, tests/*, src/snakeoil/dist/*
 
 [paths]
 source = **/site-packages/snakeoil
diff --git a/src/snakeoil/dist/distutils_extensions.py b/src/snakeoil/dist/distutils_extensions.py
index f919826c..a5f6789e 100644
--- a/src/snakeoil/dist/distutils_extensions.py
+++ b/src/snakeoil/dist/distutils_extensions.py
@@ -18,7 +18,6 @@ import subprocess
 import sys
 import textwrap
 from contextlib import ExitStack, contextmanager, redirect_stderr, redirect_stdout
-from datetime import datetime
 from multiprocessing import cpu_count
 
 from setuptools import find_packages
@@ -120,38 +119,8 @@ def module_version(moduledir=MODULEDIR):
 
     Based on the assumption that a module defines __version__.
     """
-    version = None
-    try:
-        with open(os.path.join(moduledir, '__init__.py'), encoding='utf-8') as f:
-            version = re.search(
-                r'^__version__\s*=\s*[\'"]([^\'"]*)[\'"]',
-                f.read(), re.MULTILINE).group(1)
-    except IOError as e:
-        if e.errno == errno.ENOENT:
-            pass
-        else:
-            raise
-
-    if version is None:
-        raise RuntimeError(f'Cannot find version for module: {MODULE_NAME}')
-
-    # use versioning scheme similar to setuptools_scm for untagged versions
-    git_version = get_git_version(REPODIR)
-    if git_version:
-        tag = git_version['tag']
-        if tag is None:
-            commits = git_version['commits']
-            rev = git_version['rev'][:7]
-            date = datetime.strptime(git_version['date'], '%a, %d %b %Y %H:%M:%S %z')
-            date = datetime.strftime(date, '%Y%m%d')
-            if commits is not None:
-                version += f'.dev{commits}'
-            version += f'+g{rev}.d{date}'
-        elif tag != version:
-            raise DistutilsError(
-                f'unmatched git tag {tag!r} and {MODULE_NAME} version {version!r}')
-
-    return version
+    from .utilities import module_version
+    return module_version(REPODIR, moduledir)
 
 
 def generate_verinfo(target_dir):
@@ -266,76 +235,6 @@ def data_mapping(host_prefix, path, skip=None):
                                if os.path.join(root, x) not in skip])
 
 
-def pkg_config(*packages, **kw):
-    """Translate pkg-config data to compatible Extension parameters.
-
-    Example usage:
-
-    >>> from distutils.extension import Extension
-    >>> from pkgdist import pkg_config
-    >>>
-    >>> ext_kwargs = dict(
-    ...     include_dirs=['include'],
-    ...     extra_compile_args=['-std=c++11'],
-    ... )
-    >>> extensions = [
-    ...     Extension('foo', ['foo.c']),
-    ...     Extension('bar', ['bar.c'], **pkg_config('lcms2')),
-    ...     Extension('ext', ['ext.cpp'], **pkg_config(('nss', 'libusb-1.0'), **ext_kwargs)),
-    ... ]
-    """
-    flag_map = {
-        '-I': 'include_dirs',
-        '-L': 'library_dirs',
-        '-l': 'libraries',
-    }
-
-    try:
-        tokens = subprocess.check_output(
-            ['pkg-config', '--libs', '--cflags'] + list(packages)).split()
-    except OSError as e:
-        sys.stderr.write(f'running pkg-config failed: {e.strerror}\n')
-        sys.exit(1)
-
-    for token in tokens:
-        token = token.decode()
-        if token[:2] in flag_map:
-            kw.setdefault(flag_map.get(token[:2]), []).append(token[2:])
-        else:
-            kw.setdefault('extra_compile_args', []).append(token)
-    return kw
-
-
-def cython_pyx(path=MODULEDIR):
-    """Return all available cython extensions under a given path."""
-    for root, _dirs, files in os.walk(path):
-        for f in files:
-            if f.endswith('.pyx'):
-                yield str(os.path.join(root, f))
-
-
-def cython_exts(path=MODULEDIR, build_opts=None):
-    """Prepare all cython extensions under a given path to be built."""
-    if build_opts is None:
-        build_opts = {'depends': [], 'include_dirs': []}
-    exts = []
-
-    for ext in cython_pyx(path):
-        cythonized = os.path.splitext(ext)[0] + '.c'
-        if os.path.exists(cythonized):
-            ext_path = cythonized
-        else:
-            ext_path = ext
-
-        # strip package dir
-        module = ext_path.rpartition(PACKAGEDIR)[-1].lstrip(os.path.sep)
-        # strip file extension and translate to module namespace
-        module = os.path.splitext(module)[0].replace(os.path.sep, '.')
-        exts.append(Extension(module, [ext_path], **build_opts))
-
-    return exts
-
-
 class sdist(dst_sdist.sdist):
     """sdist command wrapper to bundle generated files for release."""
 
diff --git a/src/snakeoil/dist/generate_docs.py b/src/snakeoil/dist/generate_docs.py
index 6ee2268d..9def23e5 100644
--- a/src/snakeoil/dist/generate_docs.py
+++ b/src/snakeoil/dist/generate_docs.py
@@ -24,13 +24,13 @@ def _generate_custom(project, docdir, gendir):
     custom_dir = os.path.join(docdir, 'generate')
     print(f"Generating custom docs for {project} in {gendir!r}")
 
-    for root, dirs, files in os.walk(custom_dir):
+    for root, _dirs, files in os.walk(custom_dir):
         subdir = root.split(custom_dir, 1)[1].strip('/')
         if subdir:
             try:
                 os.mkdir(os.path.join(gendir, subdir))
-            except OSError as e:
-                if e.errno != errno.EEXIST:
+            except OSError as exc:
+                if exc.errno != errno.EEXIST:
                     raise
 
         for script in sorted(x for x in files if not x.startswith(('.', '_'))):
@@ -44,8 +44,7 @@ def _generate_custom(project, docdir, gendir):
                 module.main(fake_file, docdir=docdir, gendir=gendir)
 
             fake_file.seek(0)
-            data = fake_file.read()
-            if data:
+            if data := fake_file.read():
                 rst = os.path.join(gendir, subdir, os.path.splitext(script)[0] + '.rst')
                 print(f"generating {rst}")
                 with open(rst, 'w') as f:
@@ -66,7 +65,7 @@ def generate_man(repo_dir, package_dir, module):
 
     # Replace '-' with '_' due to python namespace contraints.
     generated_man_pages = [
-        ('%s.scripts.' % (module) + s.replace('-', '_'), s) for s in scripts
+        (f"{module}.scripts.{s.replace('-', '_')}", s) for s in scripts
     ]
 
     # generate specified man pages for scripts
@@ -88,5 +87,4 @@ def generate_html(repo_dir, package_dir, module):
                         os.path.join(package_dir, module),
                         os.path.join(package_dir, module, 'test'),
                         os.path.join(package_dir, module, 'scripts')]):
-        raise RuntimeError(
-            'API doc generation failed for %s' % (module,))
+        raise RuntimeError(f'API doc generation failed for {module}')
diff --git a/src/snakeoil/dist/sphinxext.py b/src/snakeoil/dist/sphinxext.py
new file mode 100644
index 00000000..7d04c49d
--- /dev/null
+++ b/src/snakeoil/dist/sphinxext.py
@@ -0,0 +1,80 @@
+"""small sphinx extension to generate docs from argparse scripts"""
+
+import sys
+from importlib import import_module
+from pathlib import Path
+
+from sphinx.application import Sphinx
+from sphinx.ext.apidoc import main as sphinx_apidoc
+
+from .generate_docs import _generate_custom
+from .generate_man_rsts import ManConverter
+from .utilities import module_version
+
+if sys.version_info >= (3, 11):
+    import tomllib
+else:
+    import tomli as tomllib
+
+
+def prepare_scripts_man(repo_dir: Path, man_pages: list[tuple]):
+    # Workaround for sphinx doing include directive path mangling in
+    # order to interpret absolute paths "correctly", but at the same
+    # time causing relative paths to fail. This just bypasses the
+    # sphinx mangling and lets docutils handle include directives
+    # directly which works as expected.
+    from docutils.parsers.rst.directives.misc import Include as BaseInclude
+    from sphinx.directives.other import Include
+    Include.run = BaseInclude.run
+
+    with open(repo_dir / 'pyproject.toml', 'rb') as file:
+        pyproj = tomllib.load(file)
+
+    authors_list = [
+        f'{author["name"]} <{author["email"]}>' for author in pyproj['project']['authors']
+    ]
+
+    for i, man_page in enumerate(man_pages):
+        if man_page[3] is None:
+            m = list(man_page)
+            m[3] = authors_list
+            man_pages[i] = tuple(m)
+
+    man_gen_dir = str(repo_dir / 'doc' / 'generated')
+
+    for name, entry in pyproj['project']['scripts'].items():
+        module: str = entry.split(':')[0]
+        man_pages.append((f'man/{name}', name, import_module(module).__doc__.strip().split('\n', 1)[0], authors_list, 1))
+        ManConverter.regen_if_needed(man_gen_dir, module.replace('__init__', name), out_name=name)
+
+
+def generate_html(repo_dir: Path, module: str):
+    """Generate API rst docs for a project.
+
+    This uses sphinx-apidoc to auto-generate all the required rst files.
+    """
+    apidir = repo_dir / 'doc' / 'api'
+    package_dir = repo_dir / 'src' / module
+    sphinx_apidoc(['-Tef', '-o', str(apidir),
+                   str(package_dir), str(package_dir / 'test'),
+                   str(package_dir / 'scripts')])
+
+
+def doc_backend(app: Sphinx):
+    repo_dir = Path(app.config.repodir)
+    if not app.config.version:
+        app.config.version = module_version(repo_dir, repo_dir / 'src' / app.config.project)
+
+    prepare_scripts_man(repo_dir, app.config.man_pages)
+
+    if app.builder.name in ('man', 'html'):
+        docdir = repo_dir / 'doc'
+        _generate_custom(app.config.project, str(docdir), str(docdir / 'generated'))
+
+    if app.builder.name == 'html':
+        generate_html(repo_dir, app.config.project)
+
+
+def setup(app: Sphinx):
+    app.connect('builder-inited', doc_backend)
+    app.add_config_value(name="repodir", default=Path.cwd(), rebuild=True)
diff --git a/src/snakeoil/dist/utilities.py b/src/snakeoil/dist/utilities.py
new file mode 100644
index 00000000..3c595b92
--- /dev/null
+++ b/src/snakeoil/dist/utilities.py
@@ -0,0 +1,44 @@
+import errno
+import os
+import re
+from datetime import datetime
+
+from ..version import get_git_version
+
+def module_version(repodir, moduledir):
+    """Determine a module's version.
+
+    Based on the assumption that a module defines __version__.
+    """
+    version = None
+    try:
+        with open(os.path.join(moduledir, '__init__.py'), encoding='utf-8') as f:
+            version = re.search(
+                r'^__version__\s*=\s*[\'"]([^\'"]*)[\'"]',
+                f.read(), re.MULTILINE).group(1)
+    except IOError as exc:
+        if exc.errno == errno.ENOENT:
+            pass
+        else:
+            raise
+
+    if version is None:
+        raise RuntimeError(f'Cannot find version for module in: {moduledir}')
+
+    # use versioning scheme similar to setuptools_scm for untagged versions
+    git_version = get_git_version(str(repodir))
+    if git_version:
+        tag = git_version['tag']
+        if tag is None:
+            commits = git_version['commits']
+            rev = git_version['rev'][:7]
+            date = datetime.strptime(git_version['date'], '%a, %d %b %Y %H:%M:%S %z')
+            date = datetime.strftime(date, '%Y%m%d')
+            if commits is not None:
+                version += f'.dev{commits}'
+            version += f'+g{rev}.d{date}'
+        elif tag != version:
+            raise RuntimeError(
+                f'unmatched git tag {tag!r} and module version {version!r}')
+
+    return version