#!/usr/bin/env bash

# This script syncs a Devuan Linux package mirror.

# I've not found a way to do this in one rsync call, so here's two.
#
# The first one adds any new actual package files, including source and such.
# It excludes the metadata files.
#
# The second one doesn't exclude anything, which will pick up the
# metadata files excluded the first time, then delete anything that should
# be deleted at the end.
#
# In this way the amount of time where your mirror isn't fully valid is minimised.

# NOTE - You will need to customize this script.  MIRROR_PATH is the path
# to your mirror, BEHAVE is limits on the rsync commands, OPTIONS
# includes "--chown mirrors:www-data" change that to the user and group
# you want your files to be.  --exclude "/devuan-cd/" may need to be
# changed or removed if you run a file / ISO mirror or not.  The numbers used for
# deciding to print stuff is just a rough guess to try to keep the emails down,
# adjust to suit.
#
# In general the last section could be tweaked or modified to suit the sorts of
# reports you need.  For me reporting which Devuan packages got updated could be
# done simply, and I'm trying to keep this simple.  Reporting which Debian packages
# changed is too complex, and I don't need it.


# Stop on errors, and don't expand *
set -ef

MIRROR_PATH="/srv/mirrors/files.devuan.org"
BEHAVE="ionice -c3   nice -n 19   flock -n ${MIRROR_PATH}/devuan"

# Bits from the rsync 3.2.7 man page.
# 	For remote transfers, a modern rsync uses ssh for its communications,
# SORTED TRANSFER ORDER
#    Rsync always sorts the specified filenames into its internal transfer list.  This handles the merging together of the contents of identically named directories,
#       makes it easy to remove duplicate filenames. It can, however, confuse someone when the files are transferred in a different order than what was given on the command-line.
#    If you need a particular file to be transferred prior to another, either separate the files into different rsync calls, or consider using --delay-updates
#       (which doesn't affect the sorted transfer order, but does make the final file-updating phase happen much more rapidly).
#				NOTE - Seems we have to use the double rsync method, coz symlinks update in the wrong order, no matter what we do.

OPTIONS="--delay-updates   -rptSzhhv   --no-motd   --chown mirrors:www-data  -M--open-noatime"
# --verbose, -v            increase verbosity
# --no-motd                suppress daemon-mode MOTD
# --archive, -a            archive mode is -rlptgoD (no -A,-X,-U,-N,-H)
# --recursive, -r          recurse into directories
# 				NOTE - --inc-recursive, --i-r is the default, but --delay-updates disables it.
# --mkpath                 create destination's missing path components
#				"just as if mkdir -p $DEST_PATH had been run on the receiving side."
#				SIGH - other complications.
# --links, -l              copy symlinks as symlinks
#					NOTE - symlink handling is complex.
# --hard-links, -H         preserve hard links
# --perms, -p              preserve permissions
# --acls, -A               preserve ACLs (implies --perms)
# --xattrs, -X             preserve extended attributes
# --owner, -o              preserve owner (super-user only)
# --group, -g              preserve group
# --devices                preserve device files (super-user only)
# --specials               preserve special files
# -D                       same as --devices --specials
# --times, -t              preserve modification times
# --atimes, -U             preserve access (use) times
# --open-noatime           avoid changing the atime on opened files
#				Not in 3.1.2 version.
# --crtimes, -N            preserve create times (newness)
#				Not in 3.1.2 version.
# --sparse, -S             turn sequences of nulls into sparse blocks
# --delete-delay           find deletions during, delete after
# --delay-updates          put all updated files into place at end
#				"See also the "atomic-rsync" python script in the "support" subdir for an update algorithm that is even more atomic (it uses --link-dest and a parallel hierarchy of files)."
#					NOTE - worth a look later.
# --chown=USER:GROUP       simple username/groupname mapping
# --compress, -z           compress file data during the transfer
# --exclude=PATTERN        exclude files matching PATTERN
# --copy-as=USER[:GROUP]   specify user & optional group for the copy
# --human-readable, -h     output numbers in a human-readable format
# --remote-option=OPT, -M  send OPTION to the remote side only


# These helped - 
# https://git.devuan.org/devuan/amprolla3/src/branch/test/deployment/orchestrate.sh#L84
# https://chrisgilmerproj.github.io/debian/mirror/rsync/2013/08/29/mirror-debian.html
# https://pkgmaster.devuan.org/devuan_mirror_walkthrough.txt

# Investigated rsync batch mode, seems crap for our use.


mkdir -p ${MIRROR_PATH}


# The rsync calls.
# The --exclude "/devuan-cd/" is so we don't wipe out the ISO mirror.
startTime=$(date +%s%3N)
resultM=$( ${BEHAVE}   rsync ${OPTIONS}                         vesta@pkgmaster.devuan.org:~/devuan/ ${MIRROR_PATH}   --exclude "/devuan-cd/" \
    --exclude "Packages*" --exclude "Sources*" --exclude "Release*" --exclude "InRelease" --exclude "Contents-*" --exclude "Translation-*"   --exclude "ls-lR*" --exclude "current" --exclude ".~tmp~" )
endTime=$(date +%s%3N) ; durationM=$((endTime - startTime))

startTime=$(date +%s%3N)
resultC=$( ${BEHAVE}   rsync ${OPTIONS}   -l   --delete-delay   vesta@pkgmaster.devuan.org:~/devuan/ ${MIRROR_PATH}   --exclude "/devuan-cd/" )
endTime=$(date +%s%3N) ; durationC=$((endTime - startTime))


# Report the results.
resultM=$( echo "$resultM" | grep -Ev '/$|\.svg|\.txt$|\.txt.old$|versionlog\.state|log/sources|aintainers$|/by-hash/|skipping non-regular file' | head -n -4 | tail -n +2 )
resultC=$( echo "$resultC" | grep -Ev '/$|\.svg|\.txt$|\.txt.old$|versionlog\.state|log/sources|aintainers$|/by-hash/'                           | head -n -3 | tail -n +2 )
if [ -n "$resultM" ] ; then echo -e "$resultM\n"; fi
if [ $durationM -gt 4500 ] || [ -n "$resultM" ] ; then   echo -e "Main rsync time: $durationM ms.\n"; fi
if [ -n "$resultC" ] ; then echo -e "$resultC\n"; fi
if [ $durationC -gt 3000 ] || [ -n "$resultC" ] ; then   echo -e "Clean up rsync time: $durationC ms.\n"; fi
