diff options
author | 2019-12-05 21:21:02 -0800 | |
---|---|---|
committer | 2019-12-05 21:21:02 -0800 | |
commit | 62687286e02f6f7862cfff39c0bb82862fa493da (patch) | |
tree | 2d9cdc6512eaaf4ca1be2e0dad1a1f8bf9fe9d66 /snapshots-create.sh | |
parent | Update copyrights (diff) | |
download | mastermirror-scripts-62687286e02f6f7862cfff39c0bb82862fa493da.tar.gz mastermirror-scripts-62687286e02f6f7862cfff39c0bb82862fa493da.tar.bz2 mastermirror-scripts-62687286e02f6f7862cfff39c0bb82862fa493da.zip |
snapshots-create: start to implement bug 574752
Source: https://bugs.gentoo.org/attachment.cgi?id=589026
Reference: https://bugs.gentoo.org/574752
Signed-off-by: Robin H. Johnson <robbat2@gentoo.org>
Diffstat (limited to 'snapshots-create.sh')
-rwxr-xr-x | snapshots-create.sh | 60 |
1 files changed, 59 insertions, 1 deletions
diff --git a/snapshots-create.sh b/snapshots-create.sh index f43990f..d90c829 100755 --- a/snapshots-create.sh +++ b/snapshots-create.sh @@ -33,6 +33,8 @@ DELTA_FILENAME="portage-${DELTA_BASE}.tar.bz2" YESTERDAY=`/bin/date -d yesterday +%Y%m%d` FILENAME="portage-${YESTERDAY}.tar.bz2" +FILENAME_NEW="gentoo-${YESTERDAY}.tar.xz" + # Parallel compressors can use a LOT of cpu, be nice about it NICE="nice -n 10" @@ -148,6 +150,62 @@ fi write_time_log "END TARBALL $(date -u)" # end 1) +# 1b) Also create new-style tarball +write_time_log "START TARBALL_NEW $(date -u)" +if [ ! -f "${FILENAME_NEW%.xz}" ]; then + # Build exclusion list + EXCLUSION_LIST="$(mktemp -p ${TEMP} snapshot-exclude.XXXXXXXXXX)" + /usr/local/bin/mastermirror/print-exclusion-list.sh "${MASTER}" >"${EXCLUSION_LIST}" + + TAR_OPTIONS=( + # Force a small block size + --blocking-factor=1 + --record-size=512 + # GNU tar format saves approximately 1K per file in the tarball over POSIX + # format. Multiply ~170k files, and the savings are large. + --format=gnu + # Sorting by name produces consistent ordering and helps compression of + # related content. Custom ordering might further improve ordering in future + # (eg all metadata.xml first) + --sort=name + # Force ownership of content: + --owner=portage + --group=portage + # Excluded content: + --no-wildcards + --exclude-from "${EXCLUSION_LIST}" + # Do not capture any xattr/acl info at all. + --no-acls + --no-xattrs + --no-selinux + # Include a volume ID for tracing + # volume header is not supported by: + # - Docker https://bugs.gentoo.org/631644 + # - tarsync https://bugs.gentoo.org/631616 + # -V "${FILENAME%.bz2}" + # do everything relative to the destination + -C "${MASTER}" + # The . needs to match the file argument + --transform="s,^\.,${FILENAME_NEW%%.*},g" + # The operation, destination, source arguments + --create + --file ${FILENAME_NEW%.xz} + . + ) + + tar "${TAR_OPTIONS[@]}" + rc=$? + if [ $rc -ne 0 ]; then + echo "Tar run failed!" + exit 1 + fi + rm -f "${EXCLUSION_LIST}" +fi +[ ! -f "${FILENAME_NEW}.umd5sum" ] && md5sum ${FILENAME_NEW%.xz} > ${FILENAME_NEW}.umd5sum +[ ! -f "${FILENAME_NEW}" ] && ${NICE} ${XZ_PROG} -9 -e <"${FILENAME_NEW%.*}" >"${FILENAME_NEW%.*}.xz" +write_time_log "END TARBALL_NEW $(date -u)" +# end 1b) + # 2) Sanity check the tarball size and bail out if it appears abnormal. write_time_log "START SIZE SANITY $(date -u)" current_size=$(stat -c '%s' "${FILENAME}") @@ -187,7 +245,7 @@ write_time_log "END XZ $(date -u)" # 4) sign write_time_log "START SIGN $(date -u)" -for f in "${FILENAME}" "${FILENAME%.*}".xz ; do +for f in "${FILENAME}" "${FILENAME%.*}".xz "${FILENAME_NEW}"; do if [ ! -f "${UPLOAD}${f}".umd5sum ]; then cp "${FILENAME}".umd5sum "${UPLOAD}${f}".umd5sum || exit $? md5sum "$f" > "$f".md5sum || exit $? |