bump image
[discourse_docker.git] / launcher
1 #!/bin/bash
2
3 usage () {
4 echo "Usage: launcher COMMAND CONFIG [--skip-prereqs] [--docker-args STRING]"
5 echo "Commands:"
6 echo " start: Start/initialize a container"
7 echo " stop: Stop a running container"
8 echo " restart: Restart a container"
9 echo " destroy: Stop and remove a container"
10 echo " enter: Open a shell to run commands inside the container"
11 echo " logs: View the Docker logs for a container"
12 echo " bootstrap: Bootstrap a container for the config based on a template"
13 echo " rebuild: Rebuild a container (destroy old, bootstrap, start new)"
14 echo " cleanup: Remove all containers that have stopped for > 24 hours"
15 echo
16 echo "Options:"
17 echo " --skip-prereqs Don't check launcher prerequisites"
18 echo " --docker-args Extra arguments to pass when running docker"
19 echo " --skip-mac-address Don't assign a mac address"
20 exit 1
21 }
22
23 command=$1
24 config=$2
25 user_args=""
26
27 while [ ${#} -gt 0 ]; do
28 case "${1}" in
29 --debug)
30 DEBUG="1"
31 ;;
32 --skip-prereqs)
33 SKIP_PREREQS="1"
34 ;;
35 --skip-mac-address)
36 SKIP_MAC_ADDRESS="1"
37 ;;
38 --docker-args)
39 user_args="$2"
40 shift
41 ;;
42 esac
43
44 shift 1
45 done
46
47 # Docker doesn't like uppercase characters, spaces or special characters, catch it now before we build everything out and then find out
48 re='[A-Z/ !@#$%^&*()+~`=]'
49 if [[ $config =~ $re ]];
50 then
51 echo
52 echo "ERROR: Config name must not contain upper case characters, spaces or special characters. Correct config name and rerun $0."
53 echo
54 exit 1
55 fi
56
57 cd "$(dirname "$0")"
58
59 docker_min_version='1.8.0'
60 docker_rec_version='1.8.0'
61 git_min_version='1.8.0'
62 git_rec_version='1.8.0'
63
64 config_file=containers/"$config".yml
65 cidbootstrap=cids/"$config"_bootstrap.cid
66 local_discourse=local_discourse
67 image=discourse/base:2.0.20171008
68 docker_path=`which docker.io || which docker`
69 git_path=`which git`
70
71 if [ "${SUPERVISED}" = "true" ]; then
72 restart_policy="--restart=no"
73 attach_on_start="-a"
74 attach_on_run="-a stdout -a stderr"
75 else
76 attach_on_run="-d"
77 fi
78
79 if [ -n "$DOCKER_HOST" ]; then
80 docker_ip=`sed -e 's/^tcp:\/\/\(.*\):.*$/\1/' <<< "$DOCKER_HOST"`
81 elif [ -x "$(which ip 2>/dev/null)" ]; then
82 docker_ip=`ip addr show docker0 | \
83 grep 'inet ' | \
84 awk '{ split($2,a,"/"); print a[1] }';`
85 else
86 docker_ip=`ifconfig | \
87 grep -B1 "inet addr" | \
88 awk '{ if ( $1 == "inet" ) { print $2 } else if ( $2 == "Link" ) { printf "%s:" ,$1 } }' | \
89 grep docker0 | \
90 awk -F: '{ print $3 }';`
91 fi
92
93 compare_version() {
94 declare -a ver_a
95 declare -a ver_b
96 IFS=. read -a ver_a <<< "$1"
97 IFS=. read -a ver_b <<< "$2"
98
99 while [[ -n $ver_a ]]; do
100 if (( ver_a > ver_b )); then
101 return 0
102 elif (( ver_b > ver_a )); then
103 return 1
104 else
105 unset ver_a[0]
106 ver_a=("${ver_a[@]}")
107 unset ver_b[0]
108 ver_b=("${ver_b[@]}")
109 fi
110 done
111 return 1 # They are equal
112 }
113
114
115 install_docker() {
116 echo "Docker is not installed, you will need to install Docker in order to run Launcher"
117 echo "See https://docs.docker.com/installation/"
118 exit 1
119 }
120
121 check_prereqs() {
122
123 if [ -z $docker_path ]; then
124 install_docker
125 fi
126
127 # 1. docker daemon running?
128 # we send stderr to /dev/null cause we don't care about warnings,
129 # it usually complains about swap which does not matter
130 test=`$docker_path info 2> /dev/null`
131 if [[ $? -ne 0 ]] ; then
132 echo "Cannot connect to the docker daemon - verify it is running and you have access"
133 exit 1
134 fi
135
136 # 2. running an approved storage driver?
137 if ! $docker_path info 2> /dev/null | egrep -q '^Storage Driver: (aufs|btrfs|zfs|overlay|overlay2)$'; then
138 echo "Your Docker installation is not using a supported storage driver. If we were to proceed you may have a broken install."
139 echo "aufs is the recommended storage driver, although zfs/btrfs/overlay and overlay2 may work as well."
140 echo "Other storage drivers are known to be problematic."
141 echo "You can tell what filesystem you are using by running \"docker info\" and looking at the 'Storage Driver' line."
142 echo
143 echo "If you wish to continue anyway using your existing unsupported storage driver,"
144 echo "read the source code of launcher and figure out how to bypass this check."
145 exit 1
146 fi
147
148 # 3. running recommended docker version
149 test=($($docker_path --version)) # Get docker version string
150 test=${test[2]//,/} # Get version alone and strip comma if exists
151
152 # At least minimum docker version
153 if compare_version "${docker_min_version}" "${test}"; then
154 echo "ERROR: Docker version ${test} not supported, please upgrade to at least ${docker_min_version}, or recommended ${docker_rec_version}"
155 exit 1
156 fi
157
158 # Recommend newer docker version
159 if compare_version "${docker_rec_version}" "${test}"; then
160 echo "WARNING: Docker version ${test} deprecated, recommend upgrade to ${docker_rec_version} or newer."
161 fi
162
163 # 4. discourse docker image is downloaded
164 test=`$docker_path images | awk '{print $1 ":" $2 }' | grep "$image"`
165
166 if [ -z "$test" ]; then
167 echo
168 echo "WARNING: We are about to start downloading the Discourse base image"
169 echo "This process may take anywhere between a few minutes to an hour, depending on your network speed"
170 echo
171 echo "Please be patient"
172 echo
173 fi
174
175 # 5. running recommended git version
176 test=($($git_path --version)) # Get git version string
177 test=${test[2]//,/} # Get version alone and strip comma if exists
178
179 # At least minimum version
180 if compare_version "${git_min_version}" "${test}"; then
181 echo "ERROR: Git version ${test} not supported, please upgrade to at least ${git_min_version}, or recommended ${git_rec_version}"
182 exit 1
183 fi
184
185 # Recommend best version
186 if compare_version "${git_rec_version}" "${test}"; then
187 echo "WARNING: Git version ${test} deprecated, recommend upgrade to ${git_rec_version} or newer."
188 fi
189
190 # 6. able to attach stderr / out / tty
191 test=`$docker_path run $user_args -i --rm -a stdout -a stderr $image echo working`
192 if [[ "$test" =~ "working" ]] ; then : ; else
193 echo "Your Docker installation is not working correctly"
194 echo
195 echo "See: https://meta.discourse.org/t/docker-error-on-bootstrap/13657/18?u=sam"
196 exit 1
197 fi
198
199 # 7. enough space for the bootstrap on docker folder
200 folder=`$docker_path info --format '{{.DockerRootDir}}'`
201 safe_folder=${folder:-/var/lib/docker}
202 test=$(($(stat -f --format="%a*%S" $safe_folder)/1024**3 < 5))
203 if [[ $test -ne 0 ]] ; then
204 echo "You have less than 5GB of free space on the disk where $safe_folder is located. You will need more space to continue"
205 df -h $safe_folder
206 echo
207 read -p "Would you like to attempt to recover space by cleaning docker images and containers in the system?(y/N)" -n 1 -r
208 echo
209 if [[ $REPLY =~ ^[Yy]$ ]]
210 then
211 docker system prune
212 echo "If the cleanup was successful, you may try again now"
213 fi
214 exit 1
215 fi
216 }
217
218
219 if [ -z "$SKIP_PREREQS" ] && [ "$command" != "cleanup" ]; then
220 check_prereqs
221 fi
222
223 host_run() {
224 read -r -d '' env_ruby << 'RUBY'
225 require 'yaml'
226
227 input = STDIN.readlines.join
228 yaml = YAML.load(input)
229
230 if host_run = yaml['host_run']
231 params = yaml['params'] || {}
232 host_run.each do |run|
233 params.each do |k,v|
234 run = run.gsub("$#{k}", v)
235 end
236 STDOUT.write "#{run}--SEP--"
237 end
238 end
239 RUBY
240
241 host_run=`cat $config_file | $docker_path run $user_args --rm -i -a stdout -a stdin $image ruby -e "$env_ruby"`
242
243 while [ "$host_run" ] ; do
244 iter=${host_run%%--SEP--*}
245 echo
246 echo "Host run: $iter"
247 $iter || exit 1
248 echo
249 host_run="${host_run#*--SEP--}"
250 done
251 }
252
253
254 set_volumes() {
255 volumes=`cat $config_file | $docker_path run $user_args --rm -i -a stdout -a stdin $image ruby -e \
256 "require 'yaml'; puts YAML.load(STDIN.readlines.join)['volumes'].map{|v| '-v ' << v['volume']['host'] << ':' << v['volume']['guest'] << ' '}.join"`
257 }
258
259 set_links() {
260 links=`cat $config_file | $docker_path run $user_args --rm -i -a stdout -a stdin $image ruby -e \
261 "require 'yaml'; puts YAML.load(STDIN.readlines.join)['links'].map{|l| '--link ' << l['link']['name'] << ':' << l['link']['alias'] << ' '}.join"`
262 }
263
264 find_templates() {
265 local templates=`cat $1 | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \
266 "require 'yaml'; puts YAML.load(STDIN.readlines.join)['templates']"`
267
268 local arrTemplates=${templates// / }
269
270 if [ ! -z "$templates" ]; then
271 for template in "${arrTemplates[@]}"
272 do
273 local nested_templates=$(find_templates $template)
274
275 if [ ! -z "$nested_templates" ]; then
276 templates="$templates $nested_templates"
277 fi
278 done
279
280 echo $templates
281 else
282 echo ""
283 fi
284 }
285
286 set_template_info() {
287 templates=$(find_templates $config_file)
288
289 arrTemplates=(${templates// / })
290 config_data=$(cat $config_file)
291
292 input="hack: true"
293
294 for template in "${arrTemplates[@]}"
295 do
296 [ ! -z $template ] && {
297 input="$input _FILE_SEPERATOR_ $(cat $template)"
298 }
299 done
300
301 # we always want our config file last so it takes priority
302 input="$input _FILE_SEPERATOR_ $config_data"
303
304 read -r -d '' env_ruby << 'RUBY'
305 require 'yaml'
306
307 input=STDIN.readlines.join
308 # default to UTF-8 for the dbs sake
309 env = {'LANG' => 'en_US.UTF-8'}
310 input.split('_FILE_SEPERATOR_').each do |yml|
311 yml.strip!
312 begin
313 env.merge!(YAML.load(yml)['env'] || {})
314 rescue Psych::SyntaxError => e
315 puts e
316 puts "*ERROR."
317 rescue => e
318 puts yml
319 p e
320 end
321 end
322 puts env.map{|k,v| "-e\n#{k}=#{v}" }.join("\n")
323 RUBY
324
325 raw=`exec echo "$input" | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e "$env_ruby"`
326
327 env=()
328 ok=1
329 while read i; do
330 if [ "$i" == "*ERROR." ]; then
331 ok=0
332 elif [ -n "$i" ]; then
333 env[${#env[@]}]=$i
334 fi
335 done <<< "$raw"
336
337 if [ "$ok" -ne 1 ]; then
338 echo "${env[@]}"
339 echo "YAML syntax error. Please check your containers/*.yml config files."
340 exit 1
341 fi
342
343 read -r -d '' labels_ruby << 'RUBY'
344 require 'yaml'
345
346 input=STDIN.readlines.join
347 # default to UTF-8 for the dbs sake
348 labels = {}
349 input.split('_FILE_SEPERATOR_').each do |yml|
350 yml.strip!
351 begin
352 labels.merge!(YAML.load(yml)['labels'] || {})
353 rescue Psych::SyntaxError => e
354 puts e
355 puts "*ERROR."
356 rescue => e
357 puts yml
358 p e
359 end
360 end
361 puts labels.map{|k,v| "-l\n#{k}=#{v}" }.join("\n")
362 RUBY
363
364 raw=`exec echo "$input" | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e "$labels_ruby"`
365
366 labels=()
367 ok=1
368 while read i; do
369 if [ "$i" == "*ERROR." ]; then
370 ok=0
371 elif [ -n "$i" ]; then
372 labels[${#labels[@]}]=$i
373 fi
374 done <<< "$raw"
375
376 if [ "$ok" -ne 1 ]; then
377 echo "${labels[@]}"
378 echo "YAML syntax error. Please check your containers/*.yml config files."
379 exit 1
380 fi
381 }
382
383 if [ -z $docker_path ]; then
384 install_docker
385 fi
386
387 [ "$command" == "cleanup" ] && {
388 echo
389 echo "The following command will"
390 echo "- Delete all docker images for old containers"
391 echo "- Delete all stopped and orphan containers"
392 echo
393 read -p "Are you sure (Y/n): " -n 1 -r && echo
394 if [[ $REPLY =~ ^[Yy]$ || ! $REPLY ]]
395 then
396 space=$(df /var/lib/docker | awk '{ print $4 }' | grep -v Available)
397 echo "Starting Cleanup (bytes free $space)"
398
399 STATE_DIR=./.gc-state scripts/docker-gc
400
401 space=$(df /var/lib/docker | awk '{ print $4 }' | grep -v Available)
402 echo "Finished Cleanup (bytes free $space)"
403
404 else
405 exit 1
406 fi
407 exit 0
408 }
409
410 if [ -z "$command" -a -z "$config" ]; then
411 usage
412 fi
413
414 if [ ! "$command" == "setup" ]; then
415 if [[ ! -e $config_file ]]; then
416 echo "Config file was not found, ensure $config_file exists"
417 echo
418 echo "Available configs ( `cd containers && ls -dm *.yml | tr -s '\n' ' ' | awk '{ gsub(/\.yml/, ""); print }'`)"
419 exit 1
420 fi
421 fi
422
423 docker_version=($($docker_path --version))
424 docker_version=${test[2]//,/}
425 restart_policy=${restart_policy:---restart=always}
426
427 set_existing_container(){
428 existing=`$docker_path ps -a | awk '{ print $1, $(NF) }' | grep " $config$" | awk '{ print $1 }'`
429 }
430
431 run_stop() {
432
433 set_existing_container
434
435 if [ ! -z $existing ]
436 then
437 (
438 set -x
439 $docker_path stop -t 10 $config
440 )
441 else
442 echo "$config was not started !"
443 exit 1
444 fi
445 }
446
447 set_run_image() {
448 run_image=`cat $config_file | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \
449 "require 'yaml'; puts YAML.load(STDIN.readlines.join)['run_image']"`
450
451 if [ -z "$run_image" ]; then
452 run_image="$local_discourse/$config"
453 fi
454 }
455
456 set_boot_command() {
457 boot_command=`cat $config_file | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \
458 "require 'yaml'; puts YAML.load(STDIN.readlines.join)['boot_command']"`
459
460 if [ -z "$boot_command" ]; then
461
462 no_boot_command=`cat $config_file | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \
463 "require 'yaml'; puts YAML.load(STDIN.readlines.join)['no_boot_command']"`
464
465 if [ -z "$no_boot_command" ]; then
466 boot_command="/sbin/boot"
467 fi
468 fi
469 }
470
471 run_start() {
472
473 existing=`$docker_path ps | awk '{ print $1, $(NF) }' | grep " $config$" | awk '{ print $1 }'`
474 echo $existing
475 if [ ! -z $existing ]
476 then
477 echo "Nothing to do, your container has already started!"
478 exit 0
479 fi
480
481 existing=`$docker_path ps -a | awk '{ print $1, $(NF) }' | grep " $config$" | awk '{ print $1 }'`
482 if [ ! -z $existing ]
483 then
484 echo "starting up existing container"
485 (
486 set -x
487 $docker_path start $config
488 )
489 exit 0
490 fi
491
492 host_run
493
494 ports=`cat $config_file | $docker_path run $user_args --rm -i -a stdout -a stdin $image ruby -e \
495 "require 'yaml'; puts YAML.load(STDIN.readlines.join)['expose'].map{|p| \"-p #{p}\"}.join(' ')"`
496
497 docker_args=`cat $config_file | $docker_path run $user_args --rm -i -a stdout -a stdin $image ruby -e \
498 "require 'yaml'; puts YAML.load(STDIN.readlines.join)['docker_args']"`
499
500 set_template_info
501 set_volumes
502 set_links
503 set_run_image
504 set_boot_command
505
506 # get hostname and settings from container configuration
507 for envar in "${env[@]}"
508 do
509 if [[ $envar == DOCKER_USE_HOSTNAME* ]] || [[ $envar == DISCOURSE_HOSTNAME* ]]
510 then
511 # use as environment variable
512 eval $envar
513 fi
514 done
515
516 (
517 hostname=`hostname -s`
518 # overwrite hostname
519 if [ "$DOCKER_USE_HOSTNAME" = "true" ]
520 then
521 hostname=$DISCOURSE_HOSTNAME
522 else
523 hostname=$hostname-$config
524 fi
525
526 # we got to normalize so we only have allowed strings, this is more comprehensive but lets see how bash does first
527 # hostname=`$docker_path run $user_args --rm $image ruby -e 'print ARGV[0].gsub(/[^a-zA-Z-]/, "-")' $hostname`
528 # docker added more hostname rules
529 hostname=${hostname//_/-}
530
531
532 if [ -z "$SKIP_MAC_ADDRESS" ] ; then
533 mac_address="--mac-address $($docker_path run $user_args -i --rm -a stdout -a stderr $image /bin/sh -c "echo $hostname | md5sum | sed 's/^\(..\)\(..\)\(..\)\(..\)\(..\).*$/02:\1:\2:\3:\4:\5/'")"
534 fi
535
536 set -x
537 $docker_path run $links $attach_on_run $restart_policy "${env[@]}" "${labels[@]}" -h "$hostname" \
538 -e DOCKER_HOST_IP="$docker_ip" --name $config -t $ports $volumes $mac_address $docker_args $user_args \
539 $run_image $boot_command
540
541 )
542 exit 0
543
544 }
545
546
547 run_bootstrap() {
548
549 # I got no frigging clue what this does, ask Sam Saffron. It RUNS STUFF ON THE HOST I GUESS?
550 host_run
551
552 # Is the image available?
553 # If not, pull it here so the user is aware what's happening.
554 $docker_path history $image >/dev/null 2>&1 || $docker_path pull $image
555
556 set_template_info
557
558 base_image=`cat $config_file | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \
559 "require 'yaml'; puts YAML.load(STDIN.readlines.join)['base_image']"`
560
561 update_pups=`cat $config_file | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \
562 "require 'yaml'; puts YAML.load(STDIN.readlines.join)['update_pups']"`
563
564 if [[ ! X"" = X"$base_image" ]]; then
565 image=$base_image
566 fi
567
568 set_volumes
569 set_links
570
571 rm -f $cidbootstrap
572
573 run_command="cd /pups &&"
574 if [[ ! "false" = $update_pups ]]; then
575 run_command="$run_command git pull &&"
576 fi
577 run_command="$run_command /pups/bin/pups --stdin"
578
579 echo $run_command
580
581 unset ERR
582 (exec echo "$input" | $docker_path run $user_args $links "${env[@]}" -e DOCKER_HOST_IP="$docker_ip" --cidfile $cidbootstrap -i -a stdin -a stdout -a stderr $volumes $image \
583 /bin/bash -c "$run_command") || ERR=$?
584
585 unset FAILED
586 # magic exit code that indicates a retry
587 if [[ "$ERR" == 77 ]]; then
588 $docker_path rm `cat $cidbootstrap`
589 rm $cidbootstrap
590 exit 77
591 elif [[ "$ERR" > 0 ]]; then
592 FAILED=TRUE
593 fi
594
595 if [[ $FAILED = "TRUE" ]]; then
596 if [[ ! -z "$DEBUG" ]]; then
597 $docker_path commit `cat $cidbootstrap` $local_discourse/$config-debug || echo 'FAILED TO COMMIT'
598 echo "** DEBUG ** Maintaining image for diagnostics $local_discourse/$config-debug"
599 fi
600
601 $docker_path rm `cat $cidbootstrap`
602 rm $cidbootstrap
603 echo "** FAILED TO BOOTSTRAP ** please scroll up and look for earlier error messages, there may be more than one"
604 exit 1
605 fi
606
607 sleep 5
608
609 $docker_path commit `cat $cidbootstrap` $local_discourse/$config || echo 'FAILED TO COMMIT'
610 $docker_path rm `cat $cidbootstrap` && rm $cidbootstrap
611 }
612
613
614
615 case "$command" in
616 bootstrap)
617 run_bootstrap
618 echo "Successfully bootstrapped, to startup use ./launcher start $config"
619 exit 0
620 ;;
621
622 enter)
623 exec $docker_path exec -it $config /bin/bash --login
624 ;;
625
626 stop)
627 run_stop
628 exit 0
629 ;;
630
631 logs)
632
633 $docker_path logs $config
634 exit 0
635 ;;
636
637 restart)
638 run_stop
639 run_start
640 exit 0
641 ;;
642
643 start)
644 run_start
645 exit 0
646 ;;
647
648 rebuild)
649 if [ "$(git symbolic-ref --short HEAD)" == "master" ]; then
650 echo "Ensuring launcher is up to date"
651
652 git remote update
653
654 LOCAL=$(git rev-parse @)
655 REMOTE=$(git rev-parse @{u})
656 BASE=$(git merge-base @ @{u})
657
658 if [ $LOCAL = $REMOTE ]; then
659 echo "Launcher is up-to-date"
660
661 elif [ $LOCAL = $BASE ]; then
662 echo "Updating Launcher"
663 git pull || (echo 'failed to update' && exit 1)
664
665 for (( i=${#BASH_ARGV[@]}-1,j=0; i>=0,j<${#BASH_ARGV[@]}; i--,j++ ))
666 do
667 args[$j]=${BASH_ARGV[$i]}
668 done
669 exec /bin/bash $0 "${args[@]}" # $@ is empty, because of shift at the beginning. Use BASH_ARGV instead.
670
671 elif [ $REMOTE = $BASE ]; then
672 echo "Your version of Launcher is ahead of origin"
673
674 else
675 echo "Launcher has diverged source, this is only expected in Dev mode"
676 fi
677
678 fi
679
680 set_existing_container
681
682 if [ ! -z $existing ]
683 then
684 echo "Stopping old container"
685 (
686 set -x
687 $docker_path stop -t 10 $config
688 )
689 fi
690
691 run_bootstrap
692
693 if [ ! -z $existing ]
694 then
695 echo "Removing old container"
696 (
697 set -x
698 $docker_path rm $config
699 )
700 fi
701
702 run_start
703 exit 0
704 ;;
705
706
707 destroy)
708 (set -x; $docker_path stop -t 10 $config && $docker_path rm $config) || (echo "$config was not found" && exit 0)
709 exit 0
710 ;;
711 esac
712
713 usage