Downgrade docker min version
[discourse_docker.git] / launcher
1 #!/bin/bash
2
3 usage () {
4 echo "Usage: launcher COMMAND CONFIG [--skip-prereqs] [--docker-args STRING]"
5 echo "Commands:"
6 echo " start: Start/initialize a container"
7 echo " stop: Stop a running container"
8 echo " restart: Restart a container"
9 echo " destroy: Stop and remove a container"
10 echo " enter: Open a shell to run commands inside the container"
11 echo " logs: View the Docker logs for a container"
12 echo " bootstrap: Bootstrap a container for the config based on a template"
13 echo " rebuild: Rebuild a container (destroy old, bootstrap, start new)"
14 echo " cleanup: Remove all containers that have stopped for > 24 hours"
15 echo
16 echo "Options:"
17 echo " --skip-prereqs Don't check launcher prerequisites"
18 echo " --docker-args Extra arguments to pass when running docker"
19 echo " --skip-mac-address Don't assign a mac address"
20 exit 1
21 }
22
23 command=$1
24 config=$2
25 user_args=""
26
27 while [ ${#} -gt 0 ]; do
28 case "${1}" in
29 --debug)
30 DEBUG="1"
31 ;;
32 --skip-prereqs)
33 SKIP_PREREQS="1"
34 ;;
35 --skip-mac-address)
36 SKIP_MAC_ADDRESS="1"
37 ;;
38 --docker-args)
39 user_args="$2"
40 shift
41 ;;
42 esac
43
44 shift 1
45 done
46
47 # Docker doesn't like uppercase characters, spaces or special characters, catch it now before we build everything out and then find out
48 re='[A-Z/ !@#$%^&*()+~`=]'
49 if [[ $config =~ $re ]];
50 then
51 echo
52 echo "ERROR: Config name must not contain upper case characters, spaces or special characters. Correct config name and rerun $0."
53 echo
54 exit 1
55 fi
56
57 cd "$(dirname "$0")"
58
59 docker_min_version='17.03.1'
60 docker_rec_version='17.06.2'
61 git_min_version='1.8.0'
62 git_rec_version='1.8.0'
63
64 config_file=containers/"$config".yml
65 cidbootstrap=cids/"$config"_bootstrap.cid
66 local_discourse=local_discourse
67 image=discourse/base:2.0.20171008
68 docker_path=`which docker.io || which docker`
69 git_path=`which git`
70
71 if [ "${SUPERVISED}" = "true" ]; then
72 restart_policy="--restart=no"
73 attach_on_start="-a"
74 attach_on_run="-a stdout -a stderr"
75 else
76 attach_on_run="-d"
77 fi
78
79 if [ -n "$DOCKER_HOST" ]; then
80 docker_ip=`sed -e 's/^tcp:\/\/\(.*\):.*$/\1/' <<< "$DOCKER_HOST"`
81 elif [ -x "$(which ip 2>/dev/null)" ]; then
82 docker_ip=`ip addr show docker0 | \
83 grep 'inet ' | \
84 awk '{ split($2,a,"/"); print a[1] }';`
85 else
86 docker_ip=`ifconfig | \
87 grep -B1 "inet addr" | \
88 awk '{ if ( $1 == "inet" ) { print $2 } else if ( $2 == "Link" ) { printf "%s:" ,$1 } }' | \
89 grep docker0 | \
90 awk -F: '{ print $3 }';`
91 fi
92
93 # From https://stackoverflow.com/a/44660519/702738
94 compare_version() {
95 if [[ $1 == $2 ]]; then
96 return 1
97 fi
98 local IFS=.
99 local i a=(${1%%[^0-9.]*}) b=(${2%%[^0-9.]*})
100 local arem=${1#${1%%[^0-9.]*}} brem=${2#${2%%[^0-9.]*}}
101 for ((i=0; i<${#a[@]} || i<${#b[@]}; i++)); do
102 if ((10#${a[i]:-0} < 10#${b[i]:-0})); then
103 return 1
104 elif ((10#${a[i]:-0} > 10#${b[i]:-0})); then
105 return 0
106 fi
107 done
108 if [ "$arem" '<' "$brem" ]; then
109 return 1
110 elif [ "$arem" '>' "$brem" ]; then
111 return 0
112 fi
113 return 1
114 }
115
116
117 install_docker() {
118 echo "Docker is not installed, you will need to install Docker in order to run Launcher"
119 echo "See https://docs.docker.com/installation/"
120 exit 1
121 }
122
123 check_prereqs() {
124
125 if [ -z $docker_path ]; then
126 install_docker
127 fi
128
129 # 1. docker daemon running?
130 # we send stderr to /dev/null cause we don't care about warnings,
131 # it usually complains about swap which does not matter
132 test=`$docker_path info 2> /dev/null`
133 if [[ $? -ne 0 ]] ; then
134 echo "Cannot connect to the docker daemon - verify it is running and you have access"
135 exit 1
136 fi
137
138 # 2. running an approved storage driver?
139 if ! $docker_path info 2> /dev/null | egrep -q '^Storage Driver: (aufs|btrfs|zfs|overlay|overlay2)$'; then
140 echo "Your Docker installation is not using a supported storage driver. If we were to proceed you may have a broken install."
141 echo "aufs is the recommended storage driver, although zfs/btrfs/overlay and overlay2 may work as well."
142 echo "Other storage drivers are known to be problematic."
143 echo "You can tell what filesystem you are using by running \"docker info\" and looking at the 'Storage Driver' line."
144 echo
145 echo "If you wish to continue anyway using your existing unsupported storage driver,"
146 echo "read the source code of launcher and figure out how to bypass this check."
147 exit 1
148 fi
149
150 # 3. running recommended docker version
151 test=($($docker_path --version)) # Get docker version string
152 test=${test[2]//,/} # Get version alone and strip comma if exists
153
154 # At least minimum docker version
155 if compare_version "${docker_min_version}" "${test}"; then
156 echo "ERROR: Docker version ${test} not supported, please upgrade to at least ${docker_min_version}, or recommended ${docker_rec_version}"
157 exit 1
158 fi
159
160 # Recommend newer docker version
161 if compare_version "${docker_rec_version}" "${test}"; then
162 echo "WARNING: Docker version ${test} deprecated, recommend upgrade to ${docker_rec_version} or newer."
163 fi
164
165 # 4. discourse docker image is downloaded
166 test=`$docker_path images | awk '{print $1 ":" $2 }' | grep "$image"`
167
168 if [ -z "$test" ]; then
169 echo
170 echo "WARNING: We are about to start downloading the Discourse base image"
171 echo "This process may take anywhere between a few minutes to an hour, depending on your network speed"
172 echo
173 echo "Please be patient"
174 echo
175 fi
176
177 # 5. running recommended git version
178 test=($($git_path --version)) # Get git version string
179 test=${test[2]//,/} # Get version alone and strip comma if exists
180
181 # At least minimum version
182 if compare_version "${git_min_version}" "${test}"; then
183 echo "ERROR: Git version ${test} not supported, please upgrade to at least ${git_min_version}, or recommended ${git_rec_version}"
184 exit 1
185 fi
186
187 # Recommend best version
188 if compare_version "${git_rec_version}" "${test}"; then
189 echo "WARNING: Git version ${test} deprecated, recommend upgrade to ${git_rec_version} or newer."
190 fi
191
192 # 6. able to attach stderr / out / tty
193 test=`$docker_path run $user_args -i --rm -a stdout -a stderr $image echo working`
194 if [[ "$test" =~ "working" ]] ; then : ; else
195 echo "Your Docker installation is not working correctly"
196 echo
197 echo "See: https://meta.discourse.org/t/docker-error-on-bootstrap/13657/18?u=sam"
198 exit 1
199 fi
200
201 # 7. enough space for the bootstrap on docker folder
202 folder=`$docker_path info --format '{{.DockerRootDir}}'`
203 safe_folder=${folder:-/var/lib/docker}
204 test=$(($(stat -f --format="%a*%S" $safe_folder)/1024**3 < 5))
205 if [[ $test -ne 0 ]] ; then
206 echo "You have less than 5GB of free space on the disk where $safe_folder is located. You will need more space to continue"
207 df -h $safe_folder
208 echo
209 read -p "Would you like to attempt to recover space by cleaning docker images and containers in the system?(y/N)" -n 1 -r
210 echo
211 if [[ $REPLY =~ ^[Yy]$ ]]
212 then
213 docker system prune
214 echo "If the cleanup was successful, you may try again now"
215 fi
216 exit 1
217 fi
218 }
219
220
221 if [ -z "$SKIP_PREREQS" ] && [ "$command" != "cleanup" ]; then
222 check_prereqs
223 fi
224
225 host_run() {
226 read -r -d '' env_ruby << 'RUBY'
227 require 'yaml'
228
229 input = STDIN.readlines.join
230 yaml = YAML.load(input)
231
232 if host_run = yaml['host_run']
233 params = yaml['params'] || {}
234 host_run.each do |run|
235 params.each do |k,v|
236 run = run.gsub("$#{k}", v)
237 end
238 STDOUT.write "#{run}--SEP--"
239 end
240 end
241 RUBY
242
243 host_run=`cat $config_file | $docker_path run $user_args --rm -i -a stdout -a stdin $image ruby -e "$env_ruby"`
244
245 while [ "$host_run" ] ; do
246 iter=${host_run%%--SEP--*}
247 echo
248 echo "Host run: $iter"
249 $iter || exit 1
250 echo
251 host_run="${host_run#*--SEP--}"
252 done
253 }
254
255
256 set_volumes() {
257 volumes=`cat $config_file | $docker_path run $user_args --rm -i -a stdout -a stdin $image ruby -e \
258 "require 'yaml'; puts YAML.load(STDIN.readlines.join)['volumes'].map{|v| '-v ' << v['volume']['host'] << ':' << v['volume']['guest'] << ' '}.join"`
259 }
260
261 set_links() {
262 links=`cat $config_file | $docker_path run $user_args --rm -i -a stdout -a stdin $image ruby -e \
263 "require 'yaml'; puts YAML.load(STDIN.readlines.join)['links'].map{|l| '--link ' << l['link']['name'] << ':' << l['link']['alias'] << ' '}.join"`
264 }
265
266 find_templates() {
267 local templates=`cat $1 | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \
268 "require 'yaml'; puts YAML.load(STDIN.readlines.join)['templates']"`
269
270 local arrTemplates=${templates// / }
271
272 if [ ! -z "$templates" ]; then
273 for template in "${arrTemplates[@]}"
274 do
275 local nested_templates=$(find_templates $template)
276
277 if [ ! -z "$nested_templates" ]; then
278 templates="$templates $nested_templates"
279 fi
280 done
281
282 echo $templates
283 else
284 echo ""
285 fi
286 }
287
288 set_template_info() {
289 templates=$(find_templates $config_file)
290
291 arrTemplates=(${templates// / })
292 config_data=$(cat $config_file)
293
294 input="hack: true"
295
296 for template in "${arrTemplates[@]}"
297 do
298 [ ! -z $template ] && {
299 input="$input _FILE_SEPERATOR_ $(cat $template)"
300 }
301 done
302
303 # we always want our config file last so it takes priority
304 input="$input _FILE_SEPERATOR_ $config_data"
305
306 read -r -d '' env_ruby << 'RUBY'
307 require 'yaml'
308
309 input=STDIN.readlines.join
310 # default to UTF-8 for the dbs sake
311 env = {'LANG' => 'en_US.UTF-8'}
312 input.split('_FILE_SEPERATOR_').each do |yml|
313 yml.strip!
314 begin
315 env.merge!(YAML.load(yml)['env'] || {})
316 rescue Psych::SyntaxError => e
317 puts e
318 puts "*ERROR."
319 rescue => e
320 puts yml
321 p e
322 end
323 end
324 puts env.map{|k,v| "-e\n#{k}=#{v}" }.join("\n")
325 RUBY
326
327 raw=`exec echo "$input" | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e "$env_ruby"`
328
329 env=()
330 ok=1
331 while read i; do
332 if [ "$i" == "*ERROR." ]; then
333 ok=0
334 elif [ -n "$i" ]; then
335 env[${#env[@]}]=$i
336 fi
337 done <<< "$raw"
338
339 if [ "$ok" -ne 1 ]; then
340 echo "${env[@]}"
341 echo "YAML syntax error. Please check your containers/*.yml config files."
342 exit 1
343 fi
344
345 read -r -d '' labels_ruby << 'RUBY'
346 require 'yaml'
347
348 input=STDIN.readlines.join
349 # default to UTF-8 for the dbs sake
350 labels = {}
351 input.split('_FILE_SEPERATOR_').each do |yml|
352 yml.strip!
353 begin
354 labels.merge!(YAML.load(yml)['labels'] || {})
355 rescue Psych::SyntaxError => e
356 puts e
357 puts "*ERROR."
358 rescue => e
359 puts yml
360 p e
361 end
362 end
363 puts labels.map{|k,v| "-l\n#{k}=#{v}" }.join("\n")
364 RUBY
365
366 raw=`exec echo "$input" | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e "$labels_ruby"`
367
368 labels=()
369 ok=1
370 while read i; do
371 if [ "$i" == "*ERROR." ]; then
372 ok=0
373 elif [ -n "$i" ]; then
374 labels[${#labels[@]}]=$i
375 fi
376 done <<< "$raw"
377
378 if [ "$ok" -ne 1 ]; then
379 echo "${labels[@]}"
380 echo "YAML syntax error. Please check your containers/*.yml config files."
381 exit 1
382 fi
383 }
384
385 if [ -z $docker_path ]; then
386 install_docker
387 fi
388
389 [ "$command" == "cleanup" ] && {
390 echo
391 echo "The following command will"
392 echo "- Delete all docker images for old containers"
393 echo "- Delete all stopped and orphan containers"
394 echo
395 read -p "Are you sure (Y/n): " -n 1 -r && echo
396 if [[ $REPLY =~ ^[Yy]$ || ! $REPLY ]]
397 then
398 space=$(df /var/lib/docker | awk '{ print $4 }' | grep -v Available)
399 echo "Starting Cleanup (bytes free $space)"
400
401 STATE_DIR=./.gc-state scripts/docker-gc
402
403 space=$(df /var/lib/docker | awk '{ print $4 }' | grep -v Available)
404 echo "Finished Cleanup (bytes free $space)"
405
406 else
407 exit 1
408 fi
409 exit 0
410 }
411
412 if [ -z "$command" -a -z "$config" ]; then
413 usage
414 fi
415
416 if [ ! "$command" == "setup" ]; then
417 if [[ ! -e $config_file ]]; then
418 echo "Config file was not found, ensure $config_file exists"
419 echo
420 echo "Available configs ( `cd containers && ls -dm *.yml | tr -s '\n' ' ' | awk '{ gsub(/\.yml/, ""); print }'`)"
421 exit 1
422 fi
423 fi
424
425 docker_version=($($docker_path --version))
426 docker_version=${test[2]//,/}
427 restart_policy=${restart_policy:---restart=always}
428
429 set_existing_container(){
430 existing=`$docker_path ps -a | awk '{ print $1, $(NF) }' | grep " $config$" | awk '{ print $1 }'`
431 }
432
433 run_stop() {
434
435 set_existing_container
436
437 if [ ! -z $existing ]
438 then
439 (
440 set -x
441 $docker_path stop -t 10 $config
442 )
443 else
444 echo "$config was not started !"
445 exit 1
446 fi
447 }
448
449 set_run_image() {
450 run_image=`cat $config_file | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \
451 "require 'yaml'; puts YAML.load(STDIN.readlines.join)['run_image']"`
452
453 if [ -z "$run_image" ]; then
454 run_image="$local_discourse/$config"
455 fi
456 }
457
458 set_boot_command() {
459 boot_command=`cat $config_file | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \
460 "require 'yaml'; puts YAML.load(STDIN.readlines.join)['boot_command']"`
461
462 if [ -z "$boot_command" ]; then
463
464 no_boot_command=`cat $config_file | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \
465 "require 'yaml'; puts YAML.load(STDIN.readlines.join)['no_boot_command']"`
466
467 if [ -z "$no_boot_command" ]; then
468 boot_command="/sbin/boot"
469 fi
470 fi
471 }
472
473 run_start() {
474
475 existing=`$docker_path ps | awk '{ print $1, $(NF) }' | grep " $config$" | awk '{ print $1 }'`
476 echo $existing
477 if [ ! -z $existing ]
478 then
479 echo "Nothing to do, your container has already started!"
480 exit 0
481 fi
482
483 existing=`$docker_path ps -a | awk '{ print $1, $(NF) }' | grep " $config$" | awk '{ print $1 }'`
484 if [ ! -z $existing ]
485 then
486 echo "starting up existing container"
487 (
488 set -x
489 $docker_path start $config
490 )
491 exit 0
492 fi
493
494 host_run
495
496 ports=`cat $config_file | $docker_path run $user_args --rm -i -a stdout -a stdin $image ruby -e \
497 "require 'yaml'; puts YAML.load(STDIN.readlines.join)['expose'].map{|p| \"-p #{p}\"}.join(' ')"`
498
499 docker_args=`cat $config_file | $docker_path run $user_args --rm -i -a stdout -a stdin $image ruby -e \
500 "require 'yaml'; puts YAML.load(STDIN.readlines.join)['docker_args']"`
501
502 set_template_info
503 set_volumes
504 set_links
505 set_run_image
506 set_boot_command
507
508 # get hostname and settings from container configuration
509 for envar in "${env[@]}"
510 do
511 if [[ $envar == DOCKER_USE_HOSTNAME* ]] || [[ $envar == DISCOURSE_HOSTNAME* ]]
512 then
513 # use as environment variable
514 eval $envar
515 fi
516 done
517
518 (
519 hostname=`hostname -s`
520 # overwrite hostname
521 if [ "$DOCKER_USE_HOSTNAME" = "true" ]
522 then
523 hostname=$DISCOURSE_HOSTNAME
524 else
525 hostname=$hostname-$config
526 fi
527
528 # we got to normalize so we only have allowed strings, this is more comprehensive but lets see how bash does first
529 # hostname=`$docker_path run $user_args --rm $image ruby -e 'print ARGV[0].gsub(/[^a-zA-Z-]/, "-")' $hostname`
530 # docker added more hostname rules
531 hostname=${hostname//_/-}
532
533
534 if [ -z "$SKIP_MAC_ADDRESS" ] ; then
535 mac_address="--mac-address $($docker_path run $user_args -i --rm -a stdout -a stderr $image /bin/sh -c "echo $hostname | md5sum | sed 's/^\(..\)\(..\)\(..\)\(..\)\(..\).*$/02:\1:\2:\3:\4:\5/'")"
536 fi
537
538 set -x
539 $docker_path run $links $attach_on_run $restart_policy "${env[@]}" "${labels[@]}" -h "$hostname" \
540 -e DOCKER_HOST_IP="$docker_ip" --name $config -t $ports $volumes $mac_address $docker_args $user_args \
541 $run_image $boot_command
542
543 )
544 exit 0
545
546 }
547
548
549 run_bootstrap() {
550
551 # I got no frigging clue what this does, ask Sam Saffron. It RUNS STUFF ON THE HOST I GUESS?
552 host_run
553
554 # Is the image available?
555 # If not, pull it here so the user is aware what's happening.
556 $docker_path history $image >/dev/null 2>&1 || $docker_path pull $image
557
558 set_template_info
559
560 base_image=`cat $config_file | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \
561 "require 'yaml'; puts YAML.load(STDIN.readlines.join)['base_image']"`
562
563 update_pups=`cat $config_file | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \
564 "require 'yaml'; puts YAML.load(STDIN.readlines.join)['update_pups']"`
565
566 if [[ ! X"" = X"$base_image" ]]; then
567 image=$base_image
568 fi
569
570 set_volumes
571 set_links
572
573 rm -f $cidbootstrap
574
575 run_command="cd /pups &&"
576 if [[ ! "false" = $update_pups ]]; then
577 run_command="$run_command git pull &&"
578 fi
579 run_command="$run_command /pups/bin/pups --stdin"
580
581 echo $run_command
582
583 unset ERR
584 (exec echo "$input" | $docker_path run $user_args $links "${env[@]}" -e DOCKER_HOST_IP="$docker_ip" --cidfile $cidbootstrap -i -a stdin -a stdout -a stderr $volumes $image \
585 /bin/bash -c "$run_command") || ERR=$?
586
587 unset FAILED
588 # magic exit code that indicates a retry
589 if [[ "$ERR" == 77 ]]; then
590 $docker_path rm `cat $cidbootstrap`
591 rm $cidbootstrap
592 exit 77
593 elif [[ "$ERR" > 0 ]]; then
594 FAILED=TRUE
595 fi
596
597 if [[ $FAILED = "TRUE" ]]; then
598 if [[ ! -z "$DEBUG" ]]; then
599 $docker_path commit `cat $cidbootstrap` $local_discourse/$config-debug || echo 'FAILED TO COMMIT'
600 echo "** DEBUG ** Maintaining image for diagnostics $local_discourse/$config-debug"
601 fi
602
603 $docker_path rm `cat $cidbootstrap`
604 rm $cidbootstrap
605 echo "** FAILED TO BOOTSTRAP ** please scroll up and look for earlier error messages, there may be more than one"
606 exit 1
607 fi
608
609 sleep 5
610
611 $docker_path commit `cat $cidbootstrap` $local_discourse/$config || echo 'FAILED TO COMMIT'
612 $docker_path rm `cat $cidbootstrap` && rm $cidbootstrap
613 }
614
615
616
617 case "$command" in
618 bootstrap)
619 run_bootstrap
620 echo "Successfully bootstrapped, to startup use ./launcher start $config"
621 exit 0
622 ;;
623
624 enter)
625 exec $docker_path exec -it $config /bin/bash --login
626 ;;
627
628 stop)
629 run_stop
630 exit 0
631 ;;
632
633 logs)
634
635 $docker_path logs $config
636 exit 0
637 ;;
638
639 restart)
640 run_stop
641 run_start
642 exit 0
643 ;;
644
645 start)
646 run_start
647 exit 0
648 ;;
649
650 rebuild)
651 if [ "$(git symbolic-ref --short HEAD)" == "master" ]; then
652 echo "Ensuring launcher is up to date"
653
654 git remote update
655
656 LOCAL=$(git rev-parse @)
657 REMOTE=$(git rev-parse @{u})
658 BASE=$(git merge-base @ @{u})
659
660 if [ $LOCAL = $REMOTE ]; then
661 echo "Launcher is up-to-date"
662
663 elif [ $LOCAL = $BASE ]; then
664 echo "Updating Launcher"
665 git pull || (echo 'failed to update' && exit 1)
666
667 for (( i=${#BASH_ARGV[@]}-1,j=0; i>=0,j<${#BASH_ARGV[@]}; i--,j++ ))
668 do
669 args[$j]=${BASH_ARGV[$i]}
670 done
671 exec /bin/bash $0 "${args[@]}" # $@ is empty, because of shift at the beginning. Use BASH_ARGV instead.
672
673 elif [ $REMOTE = $BASE ]; then
674 echo "Your version of Launcher is ahead of origin"
675
676 else
677 echo "Launcher has diverged source, this is only expected in Dev mode"
678 fi
679
680 fi
681
682 set_existing_container
683
684 if [ ! -z $existing ]
685 then
686 echo "Stopping old container"
687 (
688 set -x
689 $docker_path stop -t 10 $config
690 )
691 fi
692
693 run_bootstrap
694
695 if [ ! -z $existing ]
696 then
697 echo "Removing old container"
698 (
699 set -x
700 $docker_path rm $config
701 )
702 fi
703
704 run_start
705 exit 0
706 ;;
707
708
709 destroy)
710 (set -x; $docker_path stop -t 10 $config && $docker_path rm $config) || (echo "$config was not found" && exit 0)
711 exit 0
712 ;;
713 esac
714
715 usage