FEATURE: Bump to latest image
[discourse_docker.git] / launcher
1 #!/bin/bash
2
3 usage () {
4 echo "Usage: launcher COMMAND CONFIG [--skip-prereqs] [--docker-args STRING]"
5 echo "Commands:"
6 echo " start: Start/initialize a container"
7 echo " stop: Stop a running container"
8 echo " restart: Restart a container"
9 echo " destroy: Stop and remove a container"
10 echo " enter: Open a shell to run commands inside the container"
11 echo " logs: View the Docker logs for a container"
12 echo " bootstrap: Bootstrap a container for the config based on a template"
13 echo " rebuild: Rebuild a container (destroy old, bootstrap, start new)"
14 echo " cleanup: Remove all containers that have stopped for > 24 hours"
15 echo
16 echo "Options:"
17 echo " --skip-prereqs Don't check launcher prerequisites"
18 echo " --docker-args Extra arguments to pass when running docker"
19 echo " --skip-mac-address Don't assign a mac address"
20 exit 1
21 }
22
23 command=$1
24 config=$2
25 user_args=""
26
27 while [ ${#} -gt 0 ]; do
28 case "${1}" in
29 --debug)
30 DEBUG="1"
31 ;;
32 --skip-prereqs)
33 SKIP_PREREQS="1"
34 ;;
35 --skip-mac-address)
36 SKIP_MAC_ADDRESS="1"
37 ;;
38 --docker-args)
39 user_args="$2"
40 shift
41 ;;
42 esac
43
44 shift 1
45 done
46
47 # Docker doesn't like uppercase characters, spaces or special characters, catch it now before we build everything out and then find out
48 re='[A-Z/ !@#$%^&*()+~`=]'
49 if [[ $config =~ $re ]];
50 then
51 echo
52 echo "ERROR: Config name must not contain upper case characters, spaces or special characters. Correct config name and rerun $0."
53 echo
54 exit 1
55 fi
56
57 cd "$(dirname "$0")"
58
59 docker_min_version='1.8.0'
60 docker_rec_version='1.8.0'
61 git_min_version='1.8.0'
62 git_rec_version='1.8.0'
63
64 config_file=containers/"$config".yml
65 cidbootstrap=cids/"$config"_bootstrap.cid
66 local_discourse=local_discourse
67 image=discourse/base:2.0.20170728
68 docker_path=`which docker.io || which docker`
69 git_path=`which git`
70
71 if [ "${SUPERVISED}" = "true" ]; then
72 restart_policy="--restart=no"
73 attach_on_start="-a"
74 attach_on_run="-a stdout -a stderr"
75 else
76 attach_on_run="-d"
77 fi
78
79 if [ -n "$DOCKER_HOST" ]; then
80 docker_ip=`sed -e 's/^tcp:\/\/\(.*\):.*$/\1/' <<< "$DOCKER_HOST"`
81 elif [ -x "$(which ip 2>/dev/null)" ]; then
82 docker_ip=`ip addr show docker0 | \
83 grep 'inet ' | \
84 awk '{ split($2,a,"/"); print a[1] }';`
85 else
86 docker_ip=`ifconfig | \
87 grep -B1 "inet addr" | \
88 awk '{ if ( $1 == "inet" ) { print $2 } else if ( $2 == "Link" ) { printf "%s:" ,$1 } }' | \
89 grep docker0 | \
90 awk -F: '{ print $3 }';`
91 fi
92
93 compare_version() {
94 declare -a ver_a
95 declare -a ver_b
96 IFS=. read -a ver_a <<< "$1"
97 IFS=. read -a ver_b <<< "$2"
98
99 while [[ -n $ver_a ]]; do
100 if (( ver_a > ver_b )); then
101 return 0
102 elif (( ver_b > ver_a )); then
103 return 1
104 else
105 unset ver_a[0]
106 ver_a=("${ver_a[@]}")
107 unset ver_b[0]
108 ver_b=("${ver_b[@]}")
109 fi
110 done
111 return 1 # They are equal
112 }
113
114
115 install_docker() {
116 echo "Docker is not installed, you will need to install Docker in order to run Launcher"
117 echo "See https://docs.docker.com/installation/"
118 exit 1
119 }
120
121 check_prereqs() {
122
123 if [ -z $docker_path ]; then
124 install_docker
125 fi
126
127 # 1. docker daemon running?
128 # we send stderr to /dev/null cause we don't care about warnings,
129 # it usually complains about swap which does not matter
130 test=`$docker_path info 2> /dev/null`
131 if [[ $? -ne 0 ]] ; then
132 echo "Cannot connect to the docker daemon - verify it is running and you have access"
133 exit 1
134 fi
135
136 # 2. running an approved storage driver?
137 if ! $docker_path info 2> /dev/null | egrep -q '^Storage Driver: (aufs|btrfs|zfs|overlay|overlay2)$'; then
138 echo "Your Docker installation is not using a supported storage driver. If we were to proceed you may have a broken install."
139 echo "aufs is the recommended storage driver, although zfs/btrfs/overlay and overlay2 may work as well."
140 echo "Other storage drivers are known to be problematic."
141 echo "You can tell what filesystem you are using by running \"docker info\" and looking at the 'Storage Driver' line."
142 echo
143 echo "If you wish to continue anyway using your existing unsupported storage driver,"
144 echo "read the source code of launcher and figure out how to bypass this check."
145 exit 1
146 fi
147
148 # 3. running recommended docker version
149 test=($($docker_path --version)) # Get docker version string
150 test=${test[2]//,/} # Get version alone and strip comma if exists
151
152 # At least minimum docker version
153 if compare_version "${docker_min_version}" "${test}"; then
154 echo "ERROR: Docker version ${test} not supported, please upgrade to at least ${docker_min_version}, or recommended ${docker_rec_version}"
155 exit 1
156 fi
157
158 # Recommend newer docker version
159 if compare_version "${docker_rec_version}" "${test}"; then
160 echo "WARNING: Docker version ${test} deprecated, recommend upgrade to ${docker_rec_version} or newer."
161 fi
162
163 # 4. discourse docker image is downloaded
164 test=`$docker_path images | awk '{print $1 ":" $2 }' | grep "$image"`
165
166 if [ -z "$test" ]; then
167 echo
168 echo "WARNING: We are about to start downloading the Discourse base image"
169 echo "This process may take anywhere between a few minutes to an hour, depending on your network speed"
170 echo
171 echo "Please be patient"
172 echo
173 fi
174
175 # 5. running recommended git version
176 test=($($git_path --version)) # Get git version string
177 test=${test[2]//,/} # Get version alone and strip comma if exists
178
179 # At least minimum version
180 if compare_version "${git_min_version}" "${test}"; then
181 echo "ERROR: Git version ${test} not supported, please upgrade to at least ${git_min_version}, or recommended ${git_rec_version}"
182 exit 1
183 fi
184
185 # Recommend best version
186 if compare_version "${git_rec_version}" "${test}"; then
187 echo "WARNING: Git version ${test} deprecated, recommend upgrade to ${git_rec_version} or newer."
188 fi
189
190 # 6. able to attach stderr / out / tty
191 test=`$docker_path run $user_args -i --rm -a stdout -a stderr $image echo working`
192 if [[ "$test" =~ "working" ]] ; then : ; else
193 echo "Your Docker installation is not working correctly"
194 echo
195 echo "See: https://meta.discourse.org/t/docker-error-on-bootstrap/13657/18?u=sam"
196 exit 1
197 fi
198
199 # 7. enough space for the bootstrap on docker folder
200 folder=`$docker_path info --format '{{.DockerRootDir}}'`
201 safe_folder=${folder:-/var/lib/docker}
202 test=$(($(stat -f --format="%a*%S" $safe_folder)/1024**3 < 5))
203 if [[ $test -ne 0 ]] ; then
204 echo "You have less than 5GB of free space on the disk. You will need more space to continue"
205 echo
206 read -p "Would you like to attempt to recover space by cleaning docker images and containers in the system?(y/N)" -n 1 -r
207 echo
208 if [[ $REPLY =~ ^[Yy]$ ]]
209 then
210 docker system prune
211 echo "If the cleanup was successful, you may try again now"
212 fi
213 exit 1
214 fi
215 }
216
217
218 if [ -z "$SKIP_PREREQS" ] && [ "$command" != "cleanup" ]; then
219 check_prereqs
220 fi
221
222 host_run() {
223 read -r -d '' env_ruby << 'RUBY'
224 require 'yaml'
225
226 input = STDIN.readlines.join
227 yaml = YAML.load(input)
228
229 if host_run = yaml['host_run']
230 params = yaml['params'] || {}
231 host_run.each do |run|
232 params.each do |k,v|
233 run = run.gsub("$#{k}", v)
234 end
235 STDOUT.write "#{run}--SEP--"
236 end
237 end
238 RUBY
239
240 host_run=`cat $config_file | $docker_path run $user_args --rm -i -a stdout -a stdin $image ruby -e "$env_ruby"`
241
242 while [ "$host_run" ] ; do
243 iter=${host_run%%--SEP--*}
244 echo
245 echo "Host run: $iter"
246 $iter || exit 1
247 echo
248 host_run="${host_run#*--SEP--}"
249 done
250 }
251
252
253 set_volumes() {
254 volumes=`cat $config_file | $docker_path run $user_args --rm -i -a stdout -a stdin $image ruby -e \
255 "require 'yaml'; puts YAML.load(STDIN.readlines.join)['volumes'].map{|v| '-v ' << v['volume']['host'] << ':' << v['volume']['guest'] << ' '}.join"`
256 }
257
258 set_links() {
259 links=`cat $config_file | $docker_path run $user_args --rm -i -a stdout -a stdin $image ruby -e \
260 "require 'yaml'; puts YAML.load(STDIN.readlines.join)['links'].map{|l| '--link ' << l['link']['name'] << ':' << l['link']['alias'] << ' '}.join"`
261 }
262
263 find_templates() {
264 local templates=`cat $1 | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \
265 "require 'yaml'; puts YAML.load(STDIN.readlines.join)['templates']"`
266
267 local arrTemplates=${templates// / }
268
269 if [ ! -z "$templates" ]; then
270 for template in "${arrTemplates[@]}"
271 do
272 local nested_templates=$(find_templates $template)
273
274 if [ ! -z "$nested_templates" ]; then
275 templates="$templates $nested_templates"
276 fi
277 done
278
279 echo $templates
280 else
281 echo ""
282 fi
283 }
284
285 set_template_info() {
286 templates=$(find_templates $config_file)
287
288 arrTemplates=(${templates// / })
289 config_data=$(cat $config_file)
290
291 input="hack: true"
292
293 for template in "${arrTemplates[@]}"
294 do
295 [ ! -z $template ] && {
296 input="$input _FILE_SEPERATOR_ $(cat $template)"
297 }
298 done
299
300 # we always want our config file last so it takes priority
301 input="$input _FILE_SEPERATOR_ $config_data"
302
303 read -r -d '' env_ruby << 'RUBY'
304 require 'yaml'
305
306 input=STDIN.readlines.join
307 # default to UTF-8 for the dbs sake
308 env = {'LANG' => 'en_US.UTF-8'}
309 input.split('_FILE_SEPERATOR_').each do |yml|
310 yml.strip!
311 begin
312 env.merge!(YAML.load(yml)['env'] || {})
313 rescue Psych::SyntaxError => e
314 puts e
315 puts "*ERROR."
316 rescue => e
317 puts yml
318 p e
319 end
320 end
321 puts env.map{|k,v| "-e\n#{k}=#{v}" }.join("\n")
322 RUBY
323
324 raw=`exec echo "$input" | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e "$env_ruby"`
325
326 env=()
327 ok=1
328 while read i; do
329 if [ "$i" == "*ERROR." ]; then
330 ok=0
331 elif [ -n "$i" ]; then
332 env[${#env[@]}]=$i
333 fi
334 done <<< "$raw"
335
336 if [ "$ok" -ne 1 ]; then
337 echo "${env[@]}"
338 echo "YAML syntax error. Please check your containers/*.yml config files."
339 exit 1
340 fi
341
342 read -r -d '' labels_ruby << 'RUBY'
343 require 'yaml'
344
345 input=STDIN.readlines.join
346 # default to UTF-8 for the dbs sake
347 labels = {}
348 input.split('_FILE_SEPERATOR_').each do |yml|
349 yml.strip!
350 begin
351 labels.merge!(YAML.load(yml)['labels'] || {})
352 rescue Psych::SyntaxError => e
353 puts e
354 puts "*ERROR."
355 rescue => e
356 puts yml
357 p e
358 end
359 end
360 puts labels.map{|k,v| "-l\n#{k}=#{v}" }.join("\n")
361 RUBY
362
363 raw=`exec echo "$input" | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e "$labels_ruby"`
364
365 labels=()
366 ok=1
367 while read i; do
368 if [ "$i" == "*ERROR." ]; then
369 ok=0
370 elif [ -n "$i" ]; then
371 labels[${#labels[@]}]=$i
372 fi
373 done <<< "$raw"
374
375 if [ "$ok" -ne 1 ]; then
376 echo "${labels[@]}"
377 echo "YAML syntax error. Please check your containers/*.yml config files."
378 exit 1
379 fi
380 }
381
382 if [ -z $docker_path ]; then
383 install_docker
384 fi
385
386 [ "$command" == "cleanup" ] && {
387 echo
388 echo "The following command will"
389 echo "- Delete all docker images for old containers"
390 echo "- Delete all stopped and orphan containers"
391 echo
392 read -p "Are you sure (Y/n): " -n 1 -r && echo
393 if [[ $REPLY =~ ^[Yy]$ || ! $REPLY ]]
394 then
395 space=$(df /var/lib/docker | awk '{ print $4 }' | grep -v Available)
396 echo "Starting Cleanup (bytes free $space)"
397
398 STATE_DIR=./.gc-state scripts/docker-gc
399
400 space=$(df /var/lib/docker | awk '{ print $4 }' | grep -v Available)
401 echo "Finished Cleanup (bytes free $space)"
402
403 else
404 exit 1
405 fi
406 exit 0
407 }
408
409 if [ -z "$command" -a -z "$config" ]; then
410 usage
411 fi
412
413 if [ ! "$command" == "setup" ]; then
414 if [[ ! -e $config_file ]]; then
415 echo "Config file was not found, ensure $config_file exists"
416 echo
417 echo "Available configs ( `cd containers && ls -dm *.yml | tr -s '\n' ' ' | awk '{ gsub(/\.yml/, ""); print }'`)"
418 exit 1
419 fi
420 fi
421
422 docker_version=($($docker_path --version))
423 docker_version=${test[2]//,/}
424 restart_policy=${restart_policy:---restart=always}
425
426 set_existing_container(){
427 existing=`$docker_path ps -a | awk '{ print $1, $(NF) }' | grep " $config$" | awk '{ print $1 }'`
428 }
429
430 run_stop() {
431
432 set_existing_container
433
434 if [ ! -z $existing ]
435 then
436 (
437 set -x
438 $docker_path stop -t 10 $config
439 )
440 else
441 echo "$config was not started !"
442 exit 1
443 fi
444 }
445
446 set_run_image() {
447 run_image=`cat $config_file | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \
448 "require 'yaml'; puts YAML.load(STDIN.readlines.join)['run_image']"`
449
450 if [ -z "$run_image" ]; then
451 run_image="$local_discourse/$config"
452 fi
453 }
454
455 set_boot_command() {
456 boot_command=`cat $config_file | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \
457 "require 'yaml'; puts YAML.load(STDIN.readlines.join)['boot_command']"`
458
459 if [ -z "$boot_command" ]; then
460
461 no_boot_command=`cat $config_file | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \
462 "require 'yaml'; puts YAML.load(STDIN.readlines.join)['no_boot_command']"`
463
464 if [ -z "$no_boot_command" ]; then
465 boot_command="/sbin/boot"
466 fi
467 fi
468 }
469
470 run_start() {
471
472 existing=`$docker_path ps | awk '{ print $1, $(NF) }' | grep " $config$" | awk '{ print $1 }'`
473 echo $existing
474 if [ ! -z $existing ]
475 then
476 echo "Nothing to do, your container has already started!"
477 exit 0
478 fi
479
480 existing=`$docker_path ps -a | awk '{ print $1, $(NF) }' | grep " $config$" | awk '{ print $1 }'`
481 if [ ! -z $existing ]
482 then
483 echo "starting up existing container"
484 (
485 set -x
486 $docker_path start $config
487 )
488 exit 0
489 fi
490
491 host_run
492
493 ports=`cat $config_file | $docker_path run $user_args --rm -i -a stdout -a stdin $image ruby -e \
494 "require 'yaml'; puts YAML.load(STDIN.readlines.join)['expose'].map{|p| \"-p #{p}\"}.join(' ')"`
495
496 docker_args=`cat $config_file | $docker_path run $user_args --rm -i -a stdout -a stdin $image ruby -e \
497 "require 'yaml'; puts YAML.load(STDIN.readlines.join)['docker_args']"`
498
499 set_template_info
500 set_volumes
501 set_links
502 set_run_image
503 set_boot_command
504
505 # get hostname and settings from container configuration
506 for envar in "${env[@]}"
507 do
508 if [[ $envar == DOCKER_USE_HOSTNAME* ]] || [[ $envar == DISCOURSE_HOSTNAME* ]]
509 then
510 # use as environment variable
511 eval $envar
512 fi
513 done
514
515 (
516 hostname=`hostname -s`
517 # overwrite hostname
518 if [ "$DOCKER_USE_HOSTNAME" = "true" ]
519 then
520 hostname=$DISCOURSE_HOSTNAME
521 else
522 hostname=$hostname-$config
523 fi
524
525 # we got to normalize so we only have allowed strings, this is more comprehensive but lets see how bash does first
526 # hostname=`$docker_path run $user_args --rm $image ruby -e 'print ARGV[0].gsub(/[^a-zA-Z-]/, "-")' $hostname`
527 # docker added more hostname rules
528 hostname=${hostname//_/-}
529
530
531 if [ -z "$SKIP_MAC_ADDRESS" ] ; then
532 mac_address="--mac-address $($docker_path run $user_args -i --rm -a stdout -a stderr $image /bin/sh -c "echo $hostname | md5sum | sed 's/^\(..\)\(..\)\(..\)\(..\)\(..\).*$/02:\1:\2:\3:\4:\5/'")"
533 fi
534
535 set -x
536 $docker_path run $links $attach_on_run $restart_policy "${env[@]}" "${labels[@]}" -h "$hostname" \
537 -e DOCKER_HOST_IP="$docker_ip" --name $config -t $ports $volumes $mac_address $docker_args $user_args \
538 $run_image $boot_command
539
540 )
541 exit 0
542
543 }
544
545
546 run_bootstrap() {
547
548 # I got no frigging clue what this does, ask Sam Saffron. It RUNS STUFF ON THE HOST I GUESS?
549 host_run
550
551 # Is the image available?
552 # If not, pull it here so the user is aware what's happening.
553 $docker_path history $image >/dev/null 2>&1 || $docker_path pull $image
554
555 set_template_info
556
557 base_image=`cat $config_file | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \
558 "require 'yaml'; puts YAML.load(STDIN.readlines.join)['base_image']"`
559
560 update_pups=`cat $config_file | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \
561 "require 'yaml'; puts YAML.load(STDIN.readlines.join)['update_pups']"`
562
563 if [[ ! X"" = X"$base_image" ]]; then
564 image=$base_image
565 fi
566
567 set_volumes
568 set_links
569
570 rm -f $cidbootstrap
571
572 run_command="cd /pups &&"
573 if [[ ! "false" = $update_pups ]]; then
574 run_command="$run_command git pull &&"
575 fi
576 run_command="$run_command /pups/bin/pups --stdin"
577
578 echo $run_command
579
580 unset ERR
581 (exec echo "$input" | $docker_path run $user_args $links "${env[@]}" -e DOCKER_HOST_IP="$docker_ip" --cidfile $cidbootstrap -i -a stdin -a stdout -a stderr $volumes $image \
582 /bin/bash -c "$run_command") || ERR=$?
583
584 unset FAILED
585 # magic exit code that indicates a retry
586 if [[ "$ERR" == 77 ]]; then
587 $docker_path rm `cat $cidbootstrap`
588 rm $cidbootstrap
589 exit 77
590 elif [[ "$ERR" > 0 ]]; then
591 FAILED=TRUE
592 fi
593
594 if [[ $FAILED = "TRUE" ]]; then
595 if [[ ! -z "$DEBUG" ]]; then
596 $docker_path commit `cat $cidbootstrap` $local_discourse/$config-debug || echo 'FAILED TO COMMIT'
597 echo "** DEBUG ** Maintaining image for diagnostics $local_discourse/$config-debug"
598 fi
599
600 $docker_path rm `cat $cidbootstrap`
601 rm $cidbootstrap
602 echo "** FAILED TO BOOTSTRAP ** please scroll up and look for earlier error messages, there may be more than one"
603 exit 1
604 fi
605
606 sleep 5
607
608 $docker_path commit `cat $cidbootstrap` $local_discourse/$config || echo 'FAILED TO COMMIT'
609 $docker_path rm `cat $cidbootstrap` && rm $cidbootstrap
610 }
611
612
613
614 case "$command" in
615 bootstrap)
616 run_bootstrap
617 echo "Successfully bootstrapped, to startup use ./launcher start $config"
618 exit 0
619 ;;
620
621 enter)
622 exec $docker_path exec -it $config /bin/bash --login
623 ;;
624
625 stop)
626 run_stop
627 exit 0
628 ;;
629
630 logs)
631
632 $docker_path logs $config
633 exit 0
634 ;;
635
636 restart)
637 run_stop
638 run_start
639 exit 0
640 ;;
641
642 start)
643 run_start
644 exit 0
645 ;;
646
647 rebuild)
648 if [ "$(git symbolic-ref --short HEAD)" == "master" ]; then
649 echo "Ensuring launcher is up to date"
650
651 git remote update
652
653 LOCAL=$(git rev-parse @)
654 REMOTE=$(git rev-parse @{u})
655 BASE=$(git merge-base @ @{u})
656
657 if [ $LOCAL = $REMOTE ]; then
658 echo "Launcher is up-to-date"
659
660 elif [ $LOCAL = $BASE ]; then
661 echo "Updating Launcher"
662 git pull || (echo 'failed to update' && exit 1)
663
664 for (( i=${#BASH_ARGV[@]}-1,j=0; i>=0,j<${#BASH_ARGV[@]}; i--,j++ ))
665 do
666 args[$j]=${BASH_ARGV[$i]}
667 done
668 exec /bin/bash $0 "${args[@]}" # $@ is empty, because of shift at the beginning. Use BASH_ARGV instead.
669
670 elif [ $REMOTE = $BASE ]; then
671 echo "Your version of Launcher is ahead of origin"
672
673 else
674 echo "Launcher has diverged source, this is only expected in Dev mode"
675 fi
676
677 fi
678
679 set_existing_container
680
681 if [ ! -z $existing ]
682 then
683 echo "Stopping old container"
684 (
685 set -x
686 $docker_path stop -t 10 $config
687 )
688 fi
689
690 run_bootstrap
691
692 if [ ! -z $existing ]
693 then
694 echo "Removing old container"
695 (
696 set -x
697 $docker_path rm $config
698 )
699 fi
700
701 run_start
702 exit 0
703 ;;
704
705
706 destroy)
707 (set -x; $docker_path stop -t 10 $config && $docker_path rm $config) || (echo "$config was not found" && exit 0)
708 exit 0
709 ;;
710 esac
711
712 usage