1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
|
#!/bin/bash
# I've not found a way to do this in one rsync call, so here's two.
#
# The first one adds any new actual package files, including source and such.
# It excludes the metadata files.
#
# The second one doesn't exclude anything, which will pick up the
# metadata files excluded the first time, then delete anything that should
# be deleted.
#
# In this way the amount of time where your mirror isn't fully valid is minimised.
# NOTE - You will need to customize this script. MIRROR_PATH is the path
# to your mirror, BEHAVE is limits on the rsync commands, OPTIONS
# includes "--chown mirrors:www-data" change that to the user and group
# you want your files to be. --exclude "/devuan-cd/" may need to be
# changed or removed if you run a file / ISO mirror.
# Stop on errors, and don't expand *
set -ef
MIRROR_PATH="/srv/mirrors/files.devuan.org"
BEHAVE="ionice -c3 nice -n 19"
# Bits from the rsync 3.2.7 man page.
# For remote transfers, a modern rsync uses ssh for its communications,
# SORTED TRANSFER ORDER
# Rsync always sorts the specified filenames into its internal transfer list. This handles the merging together of the contents of identically named directories,
# makes it easy to remove duplicate filenames. It can, however, confuse someone when the files are transferred in a different order than what was given on the command-line.
# If you need a particular file to be transferred prior to another, either separate the files into different rsync calls, or consider using --delay-updates
# (which doesn't affect the sorted transfer order, but does make the final file-updating phase happen much more rapidly).
# NOTE - Seems we have to use the double rsync method, coz symlinks update in the wrong order, no matter what we do.
OPTIONS="--delay-updates -rlptSzhhv --no-motd --chown mirrors:www-data -M--open-noatime"
# --verbose, -v increase verbosity
# --no-motd suppress daemon-mode MOTD
# --archive, -a archive mode is -rlptgoD (no -A,-X,-U,-N,-H)
# --recursive, -r recurse into directories
# NOTE - --inc-recursive, --i-r is the default, but --delay-updates disables it.
# --mkpath create destination's missing path components
# "just as if mkdir -p $DEST_PATH had been run on the receiving side."
# SIGH - other complications.
# --links, -l copy symlinks as symlinks
# NOTE - symlink handling is complex.
# --hard-links, -H preserve hard links
# --perms, -p preserve permissions
# --acls, -A preserve ACLs (implies --perms)
# --xattrs, -X preserve extended attributes
# --owner, -o preserve owner (super-user only)
# --group, -g preserve group
# --devices preserve device files (super-user only)
# --specials preserve special files
# -D same as --devices --specials
# --times, -t preserve modification times
# --atimes, -U preserve access (use) times
# --open-noatime avoid changing the atime on opened files
# Not in 3.1.2 version.
# --crtimes, -N preserve create times (newness)
# Not in 3.1.2 version.
# --sparse, -S turn sequences of nulls into sparse blocks
# --delete-delay find deletions during, delete after
# --delay-updates put all updated files into place at end
# "See also the "atomic-rsync" python script in the "support" subdir for an update algorithm that is even more atomic (it uses --link-dest and a parallel hierarchy of files)."
# NOTE - worth a look later.
# --chown=USER:GROUP simple username/groupname mapping
# --compress, -z compress file data during the transfer
# --exclude=PATTERN exclude files matching PATTERN
# --copy-as=USER[:GROUP] specify user & optional group for the copy
# --human-readable, -h output numbers in a human-readable format
# --remote-option=OPT, -M send OPTION to the remote side only
# Exclude metadata files.
EXCLUDES="--exclude Packages* \
--exclude Sources* \
--exclude Release* \
--exclude InRelease \
--exclude Contents-* \
--exclude Translation-* \
--exclude ls-lR* \
--exclude .~tmp~"
mkdir -p ${MIRROR_PATH}
# The --exclude "/devuan-cd/" is so we don't wipe out the ISO mirror.
time $BEHAVE flock -n ${MIRROR_PATH}/devuan rsync ${OPTIONS} vesta@pkgmaster.devuan.org:~/devuan/ ${MIRROR_PATH} --exclude "/devuan-cd/" $EXCLUDES \
| grep -Ev '/$|\.svg|\.txt$|\.txt.old$|versionlog\.state|log/sources|aintainers$' | head -n -1 | tail -n +2
echo ""
time $BEHAVE flock -n ${MIRROR_PATH}/devuan rsync ${OPTIONS} --delete-delay vesta@pkgmaster.devuan.org:~/devuan/ ${MIRROR_PATH} --exclude "/devuan-cd/" \
| grep -Ev '/$|\.svg|\.txt$|\.txt.old$|versionlog\.state|log/sources|aintainers$' | head -n -1 | tail -n +2
# These helped -
# https://git.devuan.org/devuan/amprolla3/src/branch/test/deployment/orchestrate.sh#L84
# https://chrisgilmerproj.github.io/debian/mirror/rsync/2013/08/29/mirror-debian.html
# https://pkgmaster.devuan.org/devuan_mirror_walkthrough.txt
# Investigated rsync batch mode, seems crap for our use.
|
