Added util/renew-opendmarc-tlds.sh script to renew PSL
[exim.git] / src / util / renew-opendmarc-tlds.sh
1 #!/bin/sh -eu
2 #
3 # Short version of this script:
4 # curl -f -o /var/cache/exim/opendmarc.tlds https://publicsuffix.org/list/public_suffix_list.dat
5 # but run as Exim runtime user, writing to a place it can write to, and with
6 # sanity checks and atomic replacement.
7 #
8 # For now, we deliberately leave the invalid file around for analysis
9 # with .<pid> suffix.
10 #
11 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~8< cut here >8~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
12 #
13 # Create a cron-job as the Exim run-time user to invoke this daily, with a
14 # single parameter, 'cron'. Eg:
15 #
16 # 3 4 * * * /usr/local/sbin/renew-opendmarc-tlds.sh cron
17 #
18 # That will, at 3 minutes past the 4th hour (in whatever timezone cron is
19 # running it) invoke this script with 'cron'; we will then sleep between 10 and
20 # 50 seconds, before continuing.
21 #
22 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~8< cut here >8~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
23 #
24 # This should be "pretty portable"; the only things it depends upon are:
25 # * a POSIX shell which additionally implements 'local' (dash works)
26 # * the 'curl' command; change the fetch_candidate() function to replace that
27 # * the 'stat' command, to get the size of a file; change size_of() if need be
28 # * the 'hexdump' command and /dev/urandom existing
29 # + used when invoked with 'cron', to avoid retrieving on a minute boundary
30 # and contending with many other automated systems.
31 # + with bash/zsh, can replace with: $(( 10 + ( RANDOM % 40 ) ))
32 # + on Debian/Ubuntu systems, hexdump is in the 'bsdmainutils' package.
33
34 # Consider putting an email address inside the parentheses, something like
35 # noc@example.org or other reachable address, so that if something goes wrong
36 # and the server operators need to step in, they can see from logs who to
37 # contact instead of just blocking your IP:
38 readonly CurlUserAgent='renew-opendmarc-tlds/0.1 (distributed with Exim)'
39
40 # change this to your Exim run-time user (exim -n -bP exim_user) :
41 readonly RuntimeUser='_exim'
42
43 # Do not make this a directory which untrusted users can write to:
44 readonly StateDir='/var/cache/exim'
45
46 readonly URL='https://publicsuffix.org/list/public_suffix_list.dat'
47
48 readonly TargetShortFile='opendmarc.tlds'
49
50 # When replacing, new file must be at least this percentage the size of
51 # the old one or it's an error:
52 readonly MinNewSizeRation=90
53
54 # Each of these regexps must be matched by the file, or it's an error:
55 readonly MustExistRegexps='
56 ^ac\.uk$
57 ^org$
58 ^tech$
59 '
60
61 # =======================8< end of configuration >8=======================
62
63 set -eu
64
65 readonly FullTargetPath="${StateDir}/${TargetShortFile}"
66 readonly WorkingFile="${FullTargetPath}.$$"
67
68 progname="$(basename "$0")"
69 note() { printf >&2 '%s: %s\n' "$progname" "$*"; }
70 die() { note "$@"; exit 1; }
71
72 # guard against stomping on file-permissions
73 [ ".$(id -un)" = ".${RuntimeUser:?}" ] || \
74 die "must be invoked as ${RuntimeUser}"
75
76 fetch_candidate() {
77 curl --user-agent "$CurlUserAgent" -fSs -o "${WorkingFile}" "${URL}"
78 }
79
80 size_of() {
81 stat -c %s "$1"
82 }
83
84 sanity_check_candidate() {
85 local new_size prev_size re
86 new_size="$(size_of "$WorkingFile")"
87
88 for re in $MustExistRegexps; do
89 grep -qs "$re" -- "$WorkingFile" || \
90 die "regexp $re not found in $WorkingFile"
91 done
92
93 if ! prev_size="$(size_of "$FullTargetPath")"; then
94 note "missing previous file, can't size-compare: $FullTargetPath"
95 # We're sane by definition, probably initial fetch, and the
96 # stat failure and this note will be printed. That's fine; if
97 # a cron invocation is missing the file then something has gone
98 # badly wrong.
99 return 0
100 fi
101 local ratio
102 ratio=$(expr $new_size \* 100 / $prev_size)
103 if [ $ratio -lt $MinNewSizeRation ]; then
104 die "New $TargetShortFile candidate only ${ratio}% size of old; $new_size vs $prev_size"
105 fi
106 }
107
108 if [ "${1:-.}" = "cron" ]; then
109 shift
110 # Don't pull on-the-minute, wait for off-cycle-peak
111 sleep $(( ($(dd if=/dev/urandom bs=1 count=1 2>/dev/null | hexdump -e '1/1 "%u"') % 40) + 10))
112 fi
113
114 umask 022
115 fetch_candidate
116 sanity_check_candidate
117 mv -- "$WorkingFile" "$FullTargetPath"