1 files changed, 97 insertions, 0 deletions

diff --git a/devuan_package_mirror_sync.sh b/devuan_package_mirror_sync.sh
new file mode 100755
index 0000000..23381a7
--- /dev/null
+++ b/devuan_package_mirror_sync.sh
@@ -0,0 +1,97 @@
+#!/bin/bash
+
+# I've not found a way to do this in one rsync call, so here's two.
+#
+# The first one adds any new actual package files, including source and such.
+# It excludes the metadata files.
+#
+# The second one doesn't exclude anything, which will pick up the
+# metadata files exclruded the first time, then delete anything that should
+# be deleted.
+#
+# In this way the amount of time where your mirror isn't fully valid is minimised.
+
+# NOTE - You will need to customize this script.  MIRROR_PATH is the path
+# to your mirror, BEHAVE is limits on the rsync commands, OPTIONS
+# includes "--chown mirrors:www-data" change that to the user and group
+# you want your files to be.  --exclude "/devuan-cd/" may need to be
+# changed or removed if you run a file / ISO mirror.
+
+# Stop on errors, and don't expand *
+set -ef
+
+MIRROR_PATH="/srv/mirrors/files.devuan.org"
+BEHAVE="ionice -c3   nice -n 19"
+
+# Bits from the rsync 3.2.7 man page.
+#       For remote transfers, a modern rsync uses ssh for its communications,
+# SORTED TRANSFER ORDER
+#    Rsync always sorts the specified filenames into its internal transfer list.  This handles the merging together of the contents of identically named directories,
+#       makes it easy to remove duplicate filenames. It can, however, confuse someone when the files are transferred in a different order than what was given on the command-line.
+#    If you need a particular file to be transferred prior to another, either separate the files into different rsync calls, or consider using --delay-updates
+#       (which doesn't affect the sorted transfer order, but does make the final file-updating phase happen much more rapidly).
+#                               NOTE - Seems we have to use the double rsync method, coz symlinks update in the wrong order, no matter what we do.
+
+OPTIONS="--delay-updates   -rlptSzhhv   --no-motd   --chown mirrors:www-data  -M--open-noatime"
+# --verbose, -v            increase verbosity
+# --no-motd                suppress daemon-mode MOTD
+# --archive, -a            archive mode is -rlptgoD (no -A,-X,-U,-N,-H)
+# --recursive, -r          recurse into directories
+#                               NOTE - --inc-recursive, --i-r is the default, but --delay-updates disables it.
+# --mkpath                 create destination's missing path components
+#                               "just as if mkdir -p $DEST_PATH had been run on the receiving side."
+#                               SIGH - other complications.
+# --links, -l              copy symlinks as symlinks
+#                                       NOTE - symlink handling is complex.
+# --hard-links, -H         preserve hard links
+# --perms, -p              preserve permissions
+# --acls, -A               preserve ACLs (implies --perms)
+# --xattrs, -X             preserve extended attributes
+# --owner, -o              preserve owner (super-user only)
+# --group, -g              preserve group
+# --devices                preserve device files (super-user only)
+# --specials               preserve special files
+# -D                       same as --devices --specials
+# --times, -t              preserve modification times
+# --atimes, -U             preserve access (use) times
+# --open-noatime           avoid changing the atime on opened files
+#                               Not in 3.1.2 version.
+# --crtimes, -N            preserve create times (newness)
+#                               Not in 3.1.2 version.
+# --sparse, -S             turn sequences of nulls into sparse blocks
+# --delete-delay           find deletions during, delete after
+# --delay-updates          put all updated files into place at end
+#                               "See also the "atomic-rsync" python script in the "support" subdir for an update algorithm that is even more atomic (it uses --link-dest and a parallel hierarchy of files)."
+#                                       NOTE - worth a look later.
+# --chown=USER:GROUP       simple username/groupname mapping
+# --compress, -z           compress file data during the transfer
+# --exclude=PATTERN        exclude files matching PATTERN
+# --copy-as=USER[:GROUP]   specify user & optional group for the copy
+# --human-readable, -h     output numbers in a human-readable format
+# --remote-option=OPT, -M  send OPTION to the remote side only
+
+# Exclude metadata files.
+EXCLUDES="--exclude Packages* \
+          --exclude Sources* \
+          --exclude Release* \
+          --exclude InRelease \
+          --exclude Contents-* \
+          --exclude Translation-* \
+          --exclude ls-lR* \
+          --exclude .~tmp~"
+
+mkdir -p ${MIRROR_PATH}
+# The --exclude "/devuan-cd/" is so we don't wipe out the ISO mirror.
+time $BEHAVE flock -n ${MIRROR_PATH}/devuan   rsync ${OPTIONS}                    vesta@pkgmaster.devuan.org:~/devuan/ ${MIRROR_PATH}   --exclude "/devuan-cd/" $EXCLUDES \
+    | grep -Ev '/$|\.svg|\.txt$|\.txt.old$|versionlog\.state|log/sources|aintainers$' | head -n -1 | tail -n +2
+echo ""
+time $BEHAVE flock -n ${MIRROR_PATH}/devuan   rsync ${OPTIONS}   --delete-delay   vesta@pkgmaster.devuan.org:~/devuan/ ${MIRROR_PATH}   --exclude "/devuan-cd/" \
+    | grep -Ev '/$|\.svg|\.txt$|\.txt.old$|versionlog\.state|log/sources|aintainers$' | head -n -1 | tail -n +2
+
+
+# These helped - 
+# https://git.devuan.org/devuan/amprolla3/src/branch/test/deployment/orchestrate.sh#L84
+# https://chrisgilmerproj.github.io/debian/mirror/rsync/2013/08/29/mirror-debian.html
+# https://pkgmaster.devuan.org/devuan_mirror_walkthrough.txt
+
+# Investigated rsync batch mode, seems crap for our use.