binpackages: new sign&sync script20230807T144611Z

Merge remote-tracking branch 'origin/wip/dilfridge'

Closes: https://bugs.gentoo.org/911793
Signed-off-by: Robin H. Johnson <robbat2@gentoo.org>

1 files changed, 117 insertions, 0 deletions

diff --git a/sign-sync-binpackages.sh b/sign-sync-binpackages.sh
new file mode 100755
index 0000000..7db6611
--- /dev/null
+++ b/sign-sync-binpackages.sh
@@ -0,0 +1,117 @@
+#!/bin/bash
+# Copyright 2023 Gentoo Authors; Distributed under the GPL v2
+# might be earlier copyright, no history available
+
+# NOTE 1: This script is SLOW. It should run at most once per day.
+# NOTE 2: This script requires that the signing key has its ownertrust
+#         set to ultimate. Which makes sense anyway, since we have the
+#         secret key.
+# NOTE 3: This script has to run as gmirror user.
+
+# Keep this variable in sync
+_ARCHES="alpha amd64 arm64 arm hppa ia64 loong m68k mips ppc riscv s390    sparc x86"
+        #alpha amd64 arm64 arm hppa ia64 loong m68k mips ppc riscv s390 sh sparc x86
+ARCHES=${ARCHES:-${_ARCHES}}
+
+VERBOSE='0'
+
+INTREE=/release/weekly/binpackages
+STAGINGTREE=/release/binpackages-staging
+OUTTREE=/var/tmp/gmirror-releases/releases
+
+IN_RSYNC_OPTS=(
+	--no-motd
+	--archive
+	--delete
+	--delete-after
+	--ignore-missing-args
+	--update
+	--mkpath
+)
+
+OUT_RSYNC_OPTS=(
+	--no-motd
+	--archive
+	--ignore-errors
+	--delete
+	--delete-after
+	--ignore-missing-args
+	--mkpath
+)
+
+export BINPKG_GPG_SIGNING_GPG_HOME=/home/gmirror/.gnupg-releng
+export BINPKG_GPG_SIGNING_KEY=13EBBDBEDE7A12775DFDB1BABB572E0E2D182910
+export BINPKG_GPG_VERIFY_GPG_HOME=${BINPKG_GPG_SIGNING_GPG_HOME}
+
+# this script needs to be run as gmirror user
+[[ $(whoami) == "gmirror" ]] || exit 111
+
+# we make sure we're not running twice in parallel
+if [[ -f ${STAGINGTREE}/.running ]] ; then
+  echo sign-sync-binpackages.sh lockfile ${STAGINGTREE}/.running exists, aborting
+  exit 112
+fi
+touch ${STAGINGTREE}/.running || exit 110
+
+# make sure we have an updated gpg-agent
+gpgconf --kill all
+
+# prepare some handy variables
+_verbose_v=''
+[[ ${VERBOSE} == '1' ]] && _verbose_v='-v'
+
+
+# step 1: rsync from the dirs where the arches copy in
+# make sure to *not* overwrite existing newer files (obviously
+# the signature changed them)...
+
+for a in ${ARCHES} ; do
+  rsync ${_verbose_v} "${IN_RSYNC_OPTS[@]}" ${INTREE}/${a}/* ${STAGINGTREE}/${a}/
+done
+
+# now the set of files is frozen in the staging dir, and we dont care
+# if any arches start uploading in the meantime
+
+
+# step 2: iterate over all binary package trees, sign
+# all unsigned files
+# we assume the directory structure to be
+#   .../binpackages-staging/amd64/17.1/x86-64
+#   .../binpackages-staging/amd64/17.1/x86-64_musl
+#   .../binpackages-staging/mips/17.0/mipsel3_n32
+#   .../binpackages-staging/x86/17.0/x86_musl_hardened
+
+for t in ${STAGINGTREE}/*/*/* ; do
+  # find all unsigned packages as fast as possible
+  find "${t}" -name '*.gpkg.tar' -print0 | \
+    parallel -0 -n1 --will-cite -- "tar tf {} |grep -E -e '/metadata\.tar\..*\.sig$' -L --label={}" > ${STAGINGTREE}/.unsigned
+
+  if [[ ${VERBOSE} == '1' ]] ; then
+    echo "List of unsigned pacakges:"
+    cat ${STAGINGTREE}/.unsigned
+    echo ; echo
+  fi
+
+  # sign the packages
+  [[ ${VERBOSE} == '1' ]] && xargs -n1 --no-run-if-empty -- gpkg-sign < ${STAGINGTREE}/.unsigned || exit 113
+  [[ ${VERBOSE} == '1' ]] || xargs -n1 --no-run-if-empty -- gpkg-sign < ${STAGINGTREE}/.unsigned > /dev/null || exit 113
+
+  # regenerate the indices
+  [[ ${VERBOSE} == '1' ]] && PKGDIR=${t} emaint -f binhost || exit 114
+  [[ ${VERBOSE} == '1' ]] || PKGDIR=${t} emaint -f binhost > /dev/null || exit 114
+done
+# unfortunately these commands make much noise... let's hope we notice errors
+
+
+# step 3: sync the result into the mirror directories from where
+# the files are distributed
+
+for a in ${ARCHES}; do
+	[[ -d ${OUTTREE}/${a}/binpackages ]] || mkdir -p ${_verbose_v} ${OUTTREE}/${a}/binpackages
+	rsync ${_verbose_v} "${OUT_RSYNC_OPTS[@]}" ${STAGINGTREE}/${a}/* ${OUTTREE}/${a}/binpackages/
+	date -u > ${OUTTREE}/${a}/binpackages/.timestamp
+done
+
+
+# we're done so remove the "lockfile"
+rm ${STAGINGTREE}/.running