#!/bin/bash

# I've not found a way to do this in one rsync call, so here's two.
#
# The first one adds any new actual package files, including source and such.
# It excludes the metadata files.
#
# The second one doesn't exclude anything, which will pick up the
# metadata files excluded the first time, then delete anything that should
# be deleted.
#
# In this way the amount of time where your mirror isn't fully valid is minimised.

# NOTE - You will need to customize this script.  MIRROR_PATH is the path
# to your mirror, BEHAVE is limits on the rsync commands, OPTIONS
# includes "--chown mirrors:www-data" change that to the user and group
# you want your files to be.  --exclude "/devuan-cd/" may need to be
# changed or removed if you run a file / ISO mirror.

# Stop on errors, and don't expand *
set -ef

MIRROR_PATH="/srv/mirrors/files.devuan.org"
BEHAVE="ionice -c3   nice -n 19"

# Bits from the rsync 3.2.7 man page.
# 	For remote transfers, a modern rsync uses ssh for its communications,
# SORTED TRANSFER ORDER
#    Rsync always sorts the specified filenames into its internal transfer list.  This handles the merging together of the contents of identically named directories,
#       makes it easy to remove duplicate filenames. It can, however, confuse someone when the files are transferred in a different order than what was given on the command-line.
#    If you need a particular file to be transferred prior to another, either separate the files into different rsync calls, or consider using --delay-updates
#       (which doesn't affect the sorted transfer order, but does make the final file-updating phase happen much more rapidly).
#				NOTE - Seems we have to use the double rsync method, coz symlinks update in the wrong order, no matter what we do.

OPTIONS="--delay-updates   -rlptSzhhv   --no-motd   --chown mirrors:www-data  -M--open-noatime"
# --verbose, -v            increase verbosity
# --no-motd                suppress daemon-mode MOTD
# --archive, -a            archive mode is -rlptgoD (no -A,-X,-U,-N,-H)
# --recursive, -r          recurse into directories
# 				NOTE - --inc-recursive, --i-r is the default, but --delay-updates disables it.
# --mkpath                 create destination's missing path components
#				"just as if mkdir -p $DEST_PATH had been run on the receiving side."
#				SIGH - other complications.
# --links, -l              copy symlinks as symlinks
#					NOTE - symlink handling is complex.
# --hard-links, -H         preserve hard links
# --perms, -p              preserve permissions
# --acls, -A               preserve ACLs (implies --perms)
# --xattrs, -X             preserve extended attributes
# --owner, -o              preserve owner (super-user only)
# --group, -g              preserve group
# --devices                preserve device files (super-user only)
# --specials               preserve special files
# -D                       same as --devices --specials
# --times, -t              preserve modification times
# --atimes, -U             preserve access (use) times
# --open-noatime           avoid changing the atime on opened files
#				Not in 3.1.2 version.
# --crtimes, -N            preserve create times (newness)
#				Not in 3.1.2 version.
# --sparse, -S             turn sequences of nulls into sparse blocks
# --delete-delay           find deletions during, delete after
# --delay-updates          put all updated files into place at end
#				"See also the "atomic-rsync" python script in the "support" subdir for an update algorithm that is even more atomic (it uses --link-dest and a parallel hierarchy of files)."
#					NOTE - worth a look later.
# --chown=USER:GROUP       simple username/groupname mapping
# --compress, -z           compress file data during the transfer
# --exclude=PATTERN        exclude files matching PATTERN
# --copy-as=USER[:GROUP]   specify user & optional group for the copy
# --human-readable, -h     output numbers in a human-readable format
# --remote-option=OPT, -M  send OPTION to the remote side only

# Exclude metadata files.
EXCLUDES="--exclude Packages* \
          --exclude Sources* \
          --exclude Release* \
          --exclude InRelease \
          --exclude Contents-* \
          --exclude Translation-* \
          --exclude ls-lR* \
          --exclude .~tmp~"

mkdir -p ${MIRROR_PATH}
# The --exclude "/devuan-cd/" is so we don't wipe out the ISO mirror.
time $BEHAVE flock -n ${MIRROR_PATH}/devuan   rsync ${OPTIONS}                    vesta@pkgmaster.devuan.org:~/devuan/ ${MIRROR_PATH}   --exclude "/devuan-cd/" $EXCLUDES \
    | grep -Ev '/$|\.svg|\.txt$|\.txt.old$|versionlog\.state|log/sources|aintainers$' | head -n -1 | tail -n +2
echo ""
time $BEHAVE flock -n ${MIRROR_PATH}/devuan   rsync ${OPTIONS}   --delete-delay   vesta@pkgmaster.devuan.org:~/devuan/ ${MIRROR_PATH}   --exclude "/devuan-cd/" \
    | grep -Ev '/$|\.svg|\.txt$|\.txt.old$|versionlog\.state|log/sources|aintainers$' | head -n -1 | tail -n +2


# These helped - 
# https://git.devuan.org/devuan/amprolla3/src/branch/test/deployment/orchestrate.sh#L84
# https://chrisgilmerproj.github.io/debian/mirror/rsync/2013/08/29/mirror-debian.html
# https://pkgmaster.devuan.org/devuan_mirror_walkthrough.txt

# Investigated rsync batch mode, seems crap for our use.