Add get_repo_revision_history function and repo_revisions file

The history of synced revisions is provided by a new
get_repo_revision_history function and corresponding
/var/lib/portage/repo_revisions file, with history
limit currently capped at 25 revisions. If a change
is detected and the current process has permission
to update the repo_revisions file, then the file will
be updated with any newly detected revisions.
For volatile repos the revisions may be unordered,
which makes them unusable for the purposes of the
revision history, so the revisions of volatile repos
are not tracked. This functions detects revisions
which are not yet visible to the current process due
to the sync-rcu option.

The emaint revisions --purgerepos and --purgeallrepos
options allow revisions for some or all repos to be
easily purged from the history. For example, the
emerge-webrsync script uses this emaint commmand to
purge the revision history of the gentoo repo when
the emerge-webrsync --revert option is used to roll
back to a previous snapshot:

    emaint revisions --purgerepos="${repo_name}"

Bug: https://bugs.gentoo.org/924772
Signed-off-by: Zac Medico <zmedico@gentoo.org>

12 files changed, 387 insertions, 7 deletions

diff --git a/bin/emerge-webrsync b/bin/emerge-webrsync
index 99da05543..caa4986da 100755
--- a/bin/emerge-webrsync
+++ b/bin/emerge-webrsync
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# Copyright 1999-2023 Gentoo Authors
+# Copyright 1999-2024 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 # Author: Karl Trygve Kalleberg <karltk@gentoo.org>
 # Rewritten from the old, Perl-based emerge-webrsync script
@@ -732,6 +732,7 @@ main() {
 	[[ ${do_debug} -eq 1 ]] && set -x
 
 	if [[ -n ${revert_date} ]] ; then
+		emaint revisions --purgerepos="${repo_name}"
 		do_snapshot 1 "${revert_date}"
 	else
 		do_latest_snapshot
diff --git a/lib/portage/const.py b/lib/portage/const.py
index 2154213b7..c9a71009a 100644
--- a/lib/portage/const.py
+++ b/lib/portage/const.py
@@ -51,6 +51,7 @@ PRIVATE_PATH = "var/lib/portage"
 WORLD_FILE = f"{PRIVATE_PATH}/world"
 WORLD_SETS_FILE = f"{PRIVATE_PATH}/world_sets"
 CONFIG_MEMORY_FILE = f"{PRIVATE_PATH}/config"
+REPO_REVISIONS = f"{PRIVATE_PATH}/repo_revisions"
 NEWS_LIB_PATH = "var/lib/gentoo"
 
 # these variables get EPREFIX prepended automagically when they are
diff --git a/lib/portage/emaint/modules/meson.build b/lib/portage/emaint/modules/meson.build
index 48f4f77d8..33b396be9 100644
--- a/lib/portage/emaint/modules/meson.build
+++ b/lib/portage/emaint/modules/meson.build
@@ -12,5 +12,6 @@ subdir('logs')
 subdir('merges')
 subdir('move')
 subdir('resume')
+subdir('revisions')
 subdir('sync')
 subdir('world')
diff --git a/lib/portage/emaint/modules/revisions/__init__.py b/lib/portage/emaint/modules/revisions/__init__.py
new file mode 100644
index 000000000..c51cbb4bf
--- /dev/null
+++ b/lib/portage/emaint/modules/revisions/__init__.py
@@ -0,0 +1,36 @@
+# Copyright 2024 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+doc = """Purge repo_revisions history file."""
+__doc__ = doc
+
+
+module_spec = {
+    "name": "revisions",
+    "description": doc,
+    "provides": {
+        "purgerevisions": {
+            "name": "revisions",
+            "sourcefile": "revisions",
+            "class": "PurgeRevisions",
+            "description": "Purge repo_revisions history",
+            "functions": ["purgeallrepos", "purgerepos"],
+            "func_desc": {
+                "repo": {
+                    "long": "--purgerepos",
+                    "help": "(revisions module only): --purgerepos  Purge revisions for the specified repo(s)",
+                    "status": "Purging %s",
+                    "action": "store",
+                    "func": "purgerepos",
+                },
+                "allrepos": {
+                    "long": "--purgeallrepos",
+                    "help": "(revisions module only): --purgeallrepos  Purge revisions for all repos",
+                    "status": "Purging %s",
+                    "action": "store_true",
+                    "func": "purgeallrepos",
+                },
+            },
+        },
+    },
+}
diff --git a/lib/portage/emaint/modules/revisions/meson.build b/lib/portage/emaint/modules/revisions/meson.build
new file mode 100644
index 000000000..9d4c61ec4
--- /dev/null
+++ b/lib/portage/emaint/modules/revisions/meson.build
@@ -0,0 +1,8 @@
+py.install_sources(
+    [
+        'revisions.py',
+        '__init__.py',
+    ],
+    subdir : 'portage/emaint/modules/revisions',
+    pure : not native_extensions
+)
diff --git a/lib/portage/emaint/modules/revisions/revisions.py b/lib/portage/emaint/modules/revisions/revisions.py
new file mode 100644
index 000000000..7078b2a8b
--- /dev/null
+++ b/lib/portage/emaint/modules/revisions/revisions.py
@@ -0,0 +1,95 @@
+# Copyright 2024 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+import json
+import os
+
+import portage
+
+
+class PurgeRevisions:
+    short_desc = "Purge repo_revisions history file."
+
+    @staticmethod
+    def name():
+        return "revisions"
+
+    def __init__(self, settings=None):
+        """Class init function
+
+        @param settings: optional portage.config instance to get EROOT from.
+        """
+        self._settings = settings
+
+    @property
+    def settings(self):
+        return self._settings or portage.settings
+
+    def purgeallrepos(self, **kwargs):
+        """Purge revisions for all repos"""
+        repo_revisions_file = os.path.join(
+            self.settings["EROOT"], portage.const.REPO_REVISIONS
+        )
+        msgs = []
+        try:
+            os.stat(repo_revisions_file)
+        except FileNotFoundError:
+            pass
+        except OSError as e:
+            msgs.append(f"{repo_revisions_file}: {e}")
+        else:
+            repo_revisions_lock = None
+            try:
+                repo_revisions_lock = portage.locks.lockfile(repo_revisions_file)
+                os.unlink(repo_revisions_file)
+            except FileNotFoundError:
+                pass
+            except OSError as e:
+                msgs.append(f"{repo_revisions_file}: {e}")
+            finally:
+                if repo_revisions_lock is not None:
+                    portage.locks.unlockfile(repo_revisions_lock)
+        return (not msgs, msgs)
+
+    def purgerepos(self, **kwargs):
+        """Purge revisions for specified repos"""
+        options = kwargs.get("options", None)
+        if options:
+            repo_names = options.get("purgerepos", "")
+        if isinstance(repo_names, str):
+            repo_names = repo_names.split()
+
+        repo_revisions_file = os.path.join(
+            self.settings["EROOT"], portage.const.REPO_REVISIONS
+        )
+        msgs = []
+        try:
+            os.stat(repo_revisions_file)
+        except FileNotFoundError:
+            pass
+        except OSError as e:
+            msgs.append(f"{repo_revisions_file}: {e}")
+        else:
+            repo_revisions_lock = None
+            try:
+                repo_revisions_lock = portage.locks.lockfile(repo_revisions_file)
+                with open(repo_revisions_file, encoding="utf8") as f:
+                    if os.fstat(f.fileno()).st_size:
+                        previous_revisions = json.load(f)
+                repo_revisions = (
+                    {} if previous_revisions is None else previous_revisions.copy()
+                )
+                for repo_name in repo_names:
+                    repo_revisions.pop(repo_name, None)
+                if not repo_revisions:
+                    os.unlink(repo_revisions_file)
+                elif repo_revisions != previous_revisions:
+                    f = portage.util.atomic_ofstream(repo_revisions_file)
+                    json.dump(repo_revisions, f, ensure_ascii=False, sort_keys=True)
+                    f.close()
+            except OSError as e:
+                msgs.append(f"{repo_revisions_file}: {e}")
+            finally:
+                if repo_revisions_lock is not None:
+                    portage.locks.unlockfile(repo_revisions_lock)
+        return (not msgs, msgs)
diff --git a/lib/portage/sync/controller.py b/lib/portage/sync/controller.py
index da593e1a8..1d55c8a5d 100644
--- a/lib/portage/sync/controller.py
+++ b/lib/portage/sync/controller.py
@@ -1,4 +1,4 @@
-# Copyright 2014-2020 Gentoo Authors
+# Copyright 2014-2024 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 import sys
@@ -8,6 +8,11 @@ import pwd
 import warnings
 
 import portage
+
+portage.proxy.lazyimport.lazyimport(
+    globals(),
+    "portage.sync.revision_history:get_repo_revision_history",
+)
 from portage import os
 from portage.progress import ProgressBar
 
@@ -170,6 +175,7 @@ class SyncManager:
         status = None
         taskmaster = TaskHandler(callback=self.do_callback)
         taskmaster.run_tasks(tasks, func, status, options=task_opts)
+        get_repo_revision_history(self.settings["EROOT"], [repo])
 
         if master_hooks or self.updatecache_flg or not repo.sync_hooks_only_on_change:
             hooks_enabled = True
diff --git a/lib/portage/sync/meson.build b/lib/portage/sync/meson.build
index a39f1e3cf..59af12561 100644
--- a/lib/portage/sync/meson.build
+++ b/lib/portage/sync/meson.build
@@ -4,6 +4,7 @@ py.install_sources(
         'controller.py',
         'getaddrinfo_validate.py',
         'old_tree_timestamp.py',
+        'revision_history.py',
         'syncbase.py',
         '__init__.py',
     ],
diff --git a/lib/portage/sync/revision_history.py b/lib/portage/sync/revision_history.py
new file mode 100644
index 000000000..3d909d94e
--- /dev/null
+++ b/lib/portage/sync/revision_history.py
@@ -0,0 +1,133 @@
+# Copyright 2024 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+import json
+import os
+from typing import Optional
+
+import portage
+from portage.locks import lockfile, unlockfile
+from portage.repository.config import RepoConfig
+from portage.util.path import first_existing
+
+_HISTORY_LIMIT = 25
+
+
+def get_repo_revision_history(
+    eroot: str, repos: Optional[list[RepoConfig]] = None
+) -> dict[str, list[str]]:
+    """
+    Get revision history of synced repos. Returns a dict that maps
+    a repo name to list of revisions in descending order by time.
+    If a change is detected and the current process has permission
+    to update the repo_revisions file, then the file will be updated
+    with any newly detected revisions.
+
+    This functions detects revisions which are not yet visible to the
+    current process due to the sync-rcu option.
+
+    @param eroot: EROOT to query
+    @type eroot: string
+    @param repos: list of RepoConfig instances to check for new revisions
+    @type repos: list
+    @rtype: dict
+    @return: mapping of repo name to list of revisions in descending
+             order by time
+    """
+    items = []
+    for repo in repos or ():
+        if repo.volatile:
+            items.append((repo, None))
+            continue
+        if repo.sync_type:
+            try:
+                sync_mod = portage.sync.module_controller.get_class(repo.sync_type)
+            except portage.exception.PortageException:
+                continue
+        else:
+            continue
+        repo_location_orig = repo.location
+        try:
+            if repo.user_location is not None:
+                # Temporarily override sync-rcu behavior which pins the
+                # location to a previous snapshot, since we want the
+                # latest available revision here.
+                repo.location = repo.user_location
+            status, repo_revision = sync_mod().retrieve_head(options={"repo": repo})
+        except NotImplementedError:
+            repo_revision = None
+        else:
+            repo_revision = repo_revision.strip() if status == os.EX_OK else None
+        finally:
+            repo.location = repo_location_orig
+
+        if repo_revision is not None:
+            items.append((repo, repo_revision))
+
+    return _maybe_update_revisions(eroot, items)
+
+
+def _update_revisions(repo_revisions, items):
+    modified = False
+    for repo, repo_revision in items:
+        if repo.volatile:
+            # For volatile repos the revisions may be unordered,
+            # which makes them unusable here where revisions are
+            # intended to be ordered, so discard them.
+            rev_list = repo_revisions.pop(repo.name, None)
+            if rev_list:
+                modified = True
+            continue
+
+        rev_list = repo_revisions.setdefault(repo.name, [])
+        if not rev_list or rev_list[0] != repo_revision:
+            rev_list.insert(0, repo_revision)
+            del rev_list[_HISTORY_LIMIT:]
+            modified = True
+    return modified
+
+
+def _maybe_update_revisions(eroot, items):
+    repo_revisions_file = os.path.join(eroot, portage.const.REPO_REVISIONS)
+    repo_revisions_lock = None
+    try:
+        previous_revisions = None
+        try:
+            with open(repo_revisions_file, encoding="utf8") as f:
+                if os.fstat(f.fileno()).st_size:
+                    previous_revisions = json.load(f)
+        except FileNotFoundError:
+            pass
+
+        repo_revisions = {} if previous_revisions is None else previous_revisions.copy()
+        modified = _update_revisions(repo_revisions, items)
+
+        # If modified then do over with lock if permissions allow.
+        if modified and os.access(
+            first_existing(os.path.dirname(repo_revisions_file)), os.W_OK
+        ):
+            # This is a bit redundant since the config._init_dirs method
+            # is supposed to create PRIVATE_PATH with these permissions.
+            portage.util.ensure_dirs(
+                os.path.dirname(repo_revisions_file),
+                gid=portage.data.portage_gid,
+                mode=0o2750,
+                mask=0o2,
+            )
+            repo_revisions_lock = lockfile(repo_revisions_file)
+            previous_revisions = None
+            with open(repo_revisions_file, encoding="utf8") as f:
+                if os.fstat(f.fileno()).st_size:
+                    previous_revisions = json.load(f)
+            repo_revisions = (
+                {} if previous_revisions is None else previous_revisions.copy()
+            )
+            _update_revisions(repo_revisions, items)
+            f = portage.util.atomic_ofstream(repo_revisions_file)
+            json.dump(repo_revisions, f, ensure_ascii=False, sort_keys=True)
+            f.close()
+    finally:
+        if repo_revisions_lock is not None:
+            unlockfile(repo_revisions_lock)
+
+    return repo_revisions
diff --git a/lib/portage/tests/sync/test_sync_local.py b/lib/portage/tests/sync/test_sync_local.py
index aeeb5d0b1..91649398d 100644
--- a/lib/portage/tests/sync/test_sync_local.py
+++ b/lib/portage/tests/sync/test_sync_local.py
@@ -1,7 +1,8 @@
-# Copyright 2014-2023 Gentoo Authors
+# Copyright 2014-2024 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 import datetime
+import json
 import subprocess
 import sys
 import textwrap
@@ -9,8 +10,9 @@ import textwrap
 import portage
 from portage import os, shutil, _shell_quote
 from portage import _unicode_decode
-from portage.const import PORTAGE_PYM_PATH, TIMESTAMP_FORMAT
+from portage.const import PORTAGE_PYM_PATH, REPO_REVISIONS, TIMESTAMP_FORMAT
 from portage.process import find_binary
+from portage.sync.revision_history import get_repo_revision_history
 from portage.tests import TestCase
 from portage.tests.resolver.ResolverPlayground import ResolverPlayground
 from portage.util import ensure_dirs
@@ -43,6 +45,7 @@ class SyncLocalTestCase(TestCase):
 			sync-rcu = %(sync-rcu)s
 			sync-rcu-store-dir = %(EPREFIX)s/var/repositories/test_repo_rcu_storedir
 			auto-sync = %(auto-sync)s
+			volatile = no
 			%(repo_extra_keys)s
 		"""
         )
@@ -50,7 +53,7 @@ class SyncLocalTestCase(TestCase):
         profile = {"eapi": ("5",), "package.use.stable.mask": ("dev-libs/A flag",)}
 
         ebuilds = {
-            "dev-libs/A-0": {},
+            "dev-libs/A-0": {"EAPI": "8"},
             "sys-apps/portage-3.0": {"IUSE": "+python_targets_python3_8"},
         }
 
@@ -81,7 +84,7 @@ class SyncLocalTestCase(TestCase):
         rcu_store_dir = os.path.join(eprefix, "var/repositories/test_repo_rcu_storedir")
 
         cmds = {}
-        for cmd in ("emerge", "emaint"):
+        for cmd in ("egencache", "emerge", "emaint"):
             for bindir in (self.bindir, self.sbindir):
                 path = os.path.join(str(bindir), cmd)
                 if os.path.exists(path):
@@ -298,6 +301,21 @@ class SyncLocalTestCase(TestCase):
                 ),
             ),
             (repo.location, git_cmd + ("init-db",)),
+            # Ensure manifests and cache are valid after
+            # previous calls to alter_ebuild.
+            (
+                homedir,
+                cmds["egencache"]
+                + (
+                    f"--repo={repo.name}",
+                    "--update",
+                    "--update-manifests",
+                    "--sign-manifests=n",
+                    "--strict-manifests=n",
+                    f"--repositories-configuration={settings['PORTAGE_REPOSITORIES']}",
+                    f"--jobs={portage.util.cpuinfo.get_cpu_count()}",
+                ),
+            ),
             (repo.location, git_cmd + ("add", ".")),
             (repo.location, git_cmd + ("commit", "-a", "-m", "add whole repo")),
         )
@@ -314,6 +332,54 @@ class SyncLocalTestCase(TestCase):
             (homedir, lambda: shutil.rmtree(os.path.join(repo.location, ".git"))),
         )
 
+        def get_revision_history(sync_type="git"):
+            # Override volatile to False here because it gets set
+            # True by RepoConfig when repo.location is not root
+            # or portage owned.
+            try:
+                volatile_orig = repo.volatile
+                repo.volatile = False
+                sync_type_orig = repo.sync_type
+                repo.sync_type = sync_type
+                revision_history = get_repo_revision_history(eroot, repos=[repo])
+            finally:
+                repo.sync_type = sync_type_orig
+                repo.volatile = volatile_orig
+
+            return revision_history
+
+        repo_revisions_cmds = (
+            (homedir, lambda: self.assertTrue(bool(get_revision_history()))),
+            (
+                homedir,
+                lambda: self.assertTrue(
+                    os.path.exists(os.path.join(eroot, REPO_REVISIONS))
+                ),
+            ),
+            (homedir, cmds["emaint"] + ("revisions", f"--purgerepos={repo.name}")),
+            (
+                homedir,
+                lambda: self.assertFalse(
+                    os.path.exists(os.path.join(eroot, REPO_REVISIONS))
+                ),
+            ),
+            (homedir, lambda: self.assertTrue(bool(get_revision_history()))),
+            (
+                homedir,
+                lambda: self.assertTrue(
+                    os.path.exists(os.path.join(eroot, REPO_REVISIONS))
+                ),
+            ),
+            (homedir, cmds["emaint"] + ("revisions", "--purgeallrepos")),
+            (
+                homedir,
+                lambda: self.assertFalse(
+                    os.path.exists(os.path.join(eroot, REPO_REVISIONS))
+                ),
+            ),
+            (homedir, lambda: self.assertTrue(bool(get_revision_history()))),
+        )
+
         def hg_init_global_config():
             with open(os.path.join(homedir, ".hgrc"), "w") as f:
                 f.write(f"[ui]\nusername = {committer_name} <{committer_email}>\n")
@@ -451,6 +517,7 @@ class SyncLocalTestCase(TestCase):
                 + sync_type_git_shallow
                 + upstream_git_commit
                 + sync_cmds
+                + repo_revisions_cmds
                 + mercurial_tests
             ):
                 if hasattr(cmd, "__call__"):
diff --git a/man/emaint.1 b/man/emaint.1
index 2abba9d47..86d5e8973 100644
--- a/man/emaint.1
+++ b/man/emaint.1
@@ -1,4 +1,4 @@
-.TH "EMAINT" "1" "Feb 2021" "Portage @VERSION@" "Portage"
+.TH "EMAINT" "1" "Mar 2024" "Portage @VERSION@" "Portage"
 .SH NAME
 emaint \- performs package management related system health checks and maintenance
 .SH SYNOPSIS
@@ -54,6 +54,11 @@ Perform package move updates for installed packages.
 .br
 OPTIONS: check, fix
 .TP
+.BR revisions
+Purge repo_revisions history file.
+.br
+OPTIONS: purgerepos, purgeallrepos
+.TP
 .BR sync
 Perform sync actions on specified repositories.
 .br
@@ -86,6 +91,13 @@ deleted.
 .TP
 .BR \-y ", " \-\-yes
 Do not prompt for emerge invocations.
+.SH OPTIONS revisions command only
+.TP
+.BR \-\-purgeallrepos
+Purge revisions for all repos.
+.TP
+.BR \-\-purgerepos \ \fIREPO\fR
+Purge revisions for the specified repo(s).
 .SH OPTIONS sync command only
 .TP
 .BR \-a ", " \-\-auto
@@ -121,6 +133,10 @@ Contains the paths and md5sums of all the config files being tracked.
 .B /var/lib/portage/failed-merges
 Contains the packages and timestamps of any failed merges being cleaned from
 the system, and to be re-emerged.
+.TP
+.B /var/lib/portage/repo_revisions
+Contains the most recent repository revisions obtained via either
+\fBemaint sync\fR or \fBemerge \-\-sync\fR.
 .SH "SEE ALSO"
 .BR emerge (1),
 .BR portage (5)
diff --git a/man/portage.5 b/man/portage.5
index 3b8329bfb..66437d8f8 100644
--- a/man/portage.5
+++ b/man/portage.5
@@ -134,6 +134,7 @@ database to track installed packages
 .BR /var/lib/portage/
 .nf
 config
+repo_revisions
 world
 world_sets
 .fi
@@ -1901,6 +1902,20 @@ Hashes which are used to determine whether files in config protected
 directories have been modified since being installed.  Files which have not
 been modified will automatically be unmerged.
 .TP
+.BR repo_revisions
+Contains the most recent repository revisions obtained via either
+\fBemaint sync\fR or \fBemerge \-\-sync\fR. The format is a JSON
+object which maps a repo name to list of revisions in descending
+order by time. In cases when revisions are not ordered by time,
+the volatile attribute should be set in \fBrepos.conf\fR in order
+to prevent unordered revisions from being stored in the
+repo_revisions file. The \fBemaint revisions\fR command can be
+used to purge revisions for specific repos, which should be done
+in any case when there is a need to roll back to an older
+revision (the \fBemerge\-webrsync \-\-revert\fR option calls
+\fBemaint revisions\fR in order to purge all revision history
+for the repository).
+.TP
 .BR world
 Every time you emerge a package, the package that you requested is
 recorded here.  Then when you run `emerge world \-up`, the list of