1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
|
#!/usr/bin/env bash
# I've not found a way to do this in one rsync call, so here's two.
#
# The first one adds any new actual package files, including source and such.
# It excludes the metadata files.
#
# The second one doesn't exclude anything, which will pick up the
# metadata files excluded the first time, then delete anything that should
# be deleted.
#
# In this way the amount of time where your mirror isn't fully valid is minimised.
# NOTE - You will need to customize this script. MIRROR_PATH is the path
# to your mirror, BEHAVE is limits on the rsync commands, OPTIONS
# includes "--chown mirrors:www-data" change that to the user and group
# you want your files to be. --exclude "/devuan-cd/" may need to be
# changed or removed if you run a file / ISO mirror.
# Stop on errors, and don't expand *
set -ef
MIRROR_PATH="/srv/mirrors/files.devuan.org"
BEHAVE="ionice -c3 nice -n 19 flock -n ${MIRROR_PATH}/devuan"
# Bits from the rsync 3.2.7 man page.
# For remote transfers, a modern rsync uses ssh for its communications,
# SORTED TRANSFER ORDER
# Rsync always sorts the specified filenames into its internal transfer list. This handles the merging together of the contents of identically named directories,
# makes it easy to remove duplicate filenames. It can, however, confuse someone when the files are transferred in a different order than what was given on the command-line.
# If you need a particular file to be transferred prior to another, either separate the files into different rsync calls, or consider using --delay-updates
# (which doesn't affect the sorted transfer order, but does make the final file-updating phase happen much more rapidly).
# NOTE - Seems we have to use the double rsync method, coz symlinks update in the wrong order, no matter what we do.
OPTIONS="--delay-updates -rptSzhhv --no-motd --chown mirrors:www-data -M--open-noatime"
# --verbose, -v increase verbosity
# --no-motd suppress daemon-mode MOTD
# --archive, -a archive mode is -rlptgoD (no -A,-X,-U,-N,-H)
# --recursive, -r recurse into directories
# NOTE - --inc-recursive, --i-r is the default, but --delay-updates disables it.
# --mkpath create destination's missing path components
# "just as if mkdir -p $DEST_PATH had been run on the receiving side."
# SIGH - other complications.
# --links, -l copy symlinks as symlinks
# NOTE - symlink handling is complex.
# --hard-links, -H preserve hard links
# --perms, -p preserve permissions
# --acls, -A preserve ACLs (implies --perms)
# --xattrs, -X preserve extended attributes
# --owner, -o preserve owner (super-user only)
# --group, -g preserve group
# --devices preserve device files (super-user only)
# --specials preserve special files
# -D same as --devices --specials
# --times, -t preserve modification times
# --atimes, -U preserve access (use) times
# --open-noatime avoid changing the atime on opened files
# Not in 3.1.2 version.
# --crtimes, -N preserve create times (newness)
# Not in 3.1.2 version.
# --sparse, -S turn sequences of nulls into sparse blocks
# --delete-delay find deletions during, delete after
# --delay-updates put all updated files into place at end
# "See also the "atomic-rsync" python script in the "support" subdir for an update algorithm that is even more atomic (it uses --link-dest and a parallel hierarchy of files)."
# NOTE - worth a look later.
# --chown=USER:GROUP simple username/groupname mapping
# --compress, -z compress file data during the transfer
# --exclude=PATTERN exclude files matching PATTERN
# --copy-as=USER[:GROUP] specify user & optional group for the copy
# --human-readable, -h output numbers in a human-readable format
# --remote-option=OPT, -M send OPTION to the remote side only
mkdir -p ${MIRROR_PATH}
# The --exclude "/devuan-cd/" is so we don't wipe out the ISO mirror.
start_time=$(date +%s%3N)
rslt=$( ${BEHAVE} rsync ${OPTIONS} vesta@pkgmaster.devuan.org:~/devuan/ ${MIRROR_PATH} --exclude "/devuan-cd/" \
--exclude "Packages*" --exclude "Sources*" --exclude "Release*" --exclude "InRelease" --exclude "Contents-*" --exclude "Translation-*" --exclude "ls-lR*" --exclude "current" --exclude ".~tmp~" \
| grep -Ev '/$|\.svg|\.txt$|\.txt.old$|versionlog\.state|log/sources|aintainers$|/by-hash/|skipping non-regular file' | head -n -4 | tail -n +2 )
end_time=$(date +%s%3N) ; duration_ms=$((end_time - start_time))
echo -n "$rslt"; if [ -n "$rslt" ] ; then echo ""; fi
if [ $duration_ms -gt 2000 ] ; then echo -e "Main rsync time in ms: $duration_ms\n"; fi
start_time=$(date +%s%3N)
rslt=$( ${BEHAVE} rsync ${OPTIONS} -l --delete-delay vesta@pkgmaster.devuan.org:~/devuan/ ${MIRROR_PATH} --exclude "/devuan-cd/" \
| grep -Ev '/$|\.svg|\.txt$|\.txt.old$|versionlog\.state|log/sources|aintainers$|/by-hash/' | head -n -3 | tail -n +2 )
end_time=$(date +%s%3N) ; duration_ms=$((end_time - start_time))
echo -n "$rslt"; if [ -n "$rslt" ] ; then echo ""; fi
if [ $duration_ms -gt 1000 ] ; then echo "Clean up rsync time in ms: $duration_ms"; fi
# These helped -
# https://git.devuan.org/devuan/amprolla3/src/branch/test/deployment/orchestrate.sh#L84
# https://chrisgilmerproj.github.io/debian/mirror/rsync/2013/08/29/mirror-debian.html
# https://pkgmaster.devuan.org/devuan_mirror_walkthrough.txt
# Investigated rsync batch mode, seems crap for our use.
|
