snapshots-create: start to implement bug 574752

Source: https://bugs.gentoo.org/attachment.cgi?id=589026
Reference: https://bugs.gentoo.org/574752
Signed-off-by: Robin H. Johnson <robbat2@gentoo.org>

1 files changed, 59 insertions, 1 deletions

diff --git a/snapshots-create.sh b/snapshots-create.sh
index f43990f..d90c829 100755
--- a/snapshots-create.sh
+++ b/snapshots-create.sh
@@ -33,6 +33,8 @@ DELTA_FILENAME="portage-${DELTA_BASE}.tar.bz2"
 YESTERDAY=`/bin/date -d yesterday +%Y%m%d`
 FILENAME="portage-${YESTERDAY}.tar.bz2"
 
+FILENAME_NEW="gentoo-${YESTERDAY}.tar.xz"
+
 # Parallel compressors can use a LOT of cpu, be nice about it
 NICE="nice -n 10"
 
@@ -148,6 +150,62 @@ fi
 write_time_log "END TARBALL		$(date -u)"
 # end 1)
 
+# 1b) Also create new-style tarball
+write_time_log "START TARBALL_NEW		$(date -u)"
+if [ ! -f "${FILENAME_NEW%.xz}" ]; then
+	# Build exclusion list
+	EXCLUSION_LIST="$(mktemp -p ${TEMP} snapshot-exclude.XXXXXXXXXX)"
+	/usr/local/bin/mastermirror/print-exclusion-list.sh "${MASTER}" >"${EXCLUSION_LIST}"
+
+	TAR_OPTIONS=(
+	# Force a small block size
+		--blocking-factor=1
+		--record-size=512
+	# GNU tar format saves approximately 1K per file in the tarball over POSIX
+	# format. Multiply ~170k files, and the savings are large.
+		--format=gnu
+	# Sorting by name produces consistent ordering and helps compression of
+	# related content. Custom ordering might further improve ordering in future
+	# (eg all metadata.xml first)
+		--sort=name
+	# Force ownership of content:
+		--owner=portage
+		--group=portage
+	# Excluded content:
+		--no-wildcards
+		--exclude-from "${EXCLUSION_LIST}"
+	# Do not capture any xattr/acl info at all.
+		--no-acls
+		--no-xattrs
+		--no-selinux
+	# Include a volume ID for tracing
+	# volume header is not supported by:
+	# - Docker https://bugs.gentoo.org/631644
+	# - tarsync https://bugs.gentoo.org/631616
+	#	-V "${FILENAME%.bz2}"
+	# do everything relative to the destination
+		-C "${MASTER}"
+		# The . needs to match the file argument
+		--transform="s,^\.,${FILENAME_NEW%%.*},g"
+	# The operation, destination, source arguments
+		--create
+		--file ${FILENAME_NEW%.xz}
+		.
+	)
+
+	tar "${TAR_OPTIONS[@]}"
+	rc=$?
+	if [ $rc -ne 0 ]; then
+		echo "Tar run failed!"
+		exit 1
+	fi
+	rm -f "${EXCLUSION_LIST}"
+fi
+[ ! -f "${FILENAME_NEW}.umd5sum" ] && md5sum ${FILENAME_NEW%.xz} > ${FILENAME_NEW}.umd5sum
+[ ! -f "${FILENAME_NEW}" ] && ${NICE} ${XZ_PROG} -9 -e <"${FILENAME_NEW%.*}" >"${FILENAME_NEW%.*}.xz"
+write_time_log "END TARBALL_NEW		$(date -u)"
+# end 1b)
+
 # 2) Sanity check the tarball size and bail out if it appears abnormal.
 write_time_log "START SIZE SANITY	$(date -u)"
 current_size=$(stat -c '%s' "${FILENAME}")
@@ -187,7 +245,7 @@ write_time_log "END XZ			$(date -u)"
 
 # 4) sign
 write_time_log "START SIGN		$(date -u)"
-for f in "${FILENAME}" "${FILENAME%.*}".xz ; do
+for f in "${FILENAME}" "${FILENAME%.*}".xz "${FILENAME_NEW}"; do
 	if [ ! -f "${UPLOAD}${f}".umd5sum ]; then
 		cp "${FILENAME}".umd5sum "${UPLOAD}${f}".umd5sum || exit $?
 		md5sum "$f" > "$f".md5sum || exit $?