move normalization
[discourse_docker.git] / launcher
1 #!/bin/bash
2
3 command=$1
4 config=$2
5 opt=$3
6
7 # Docker doesn't like uppercase characters, spaces or special characters, catch it now before we build everything out and then find out
8 re='[A-Z/ !@#$%^&*()+~`=]'
9 if [[ $config =~ $re ]];
10 then
11 echo
12 echo "ERROR: Config name must not contain upper case characters, spaces or special characters. Correct config name and rerun $0."
13 echo
14 exit 1
15 fi
16
17 cd "$(dirname "$0")"
18
19 docker_min_version='1.6.0'
20 docker_rec_version='1.6.0'
21
22 config_file=containers/"$config".yml
23 cidbootstrap=cids/"$config"_bootstrap.cid
24 local_discourse=local_discourse
25 image=discourse/discourse:1.0.17
26 docker_path=`which docker.io || which docker`
27
28
29 if [ "${SUPERVISED}" = "true" ]; then
30 restart_policy="--restart=no"
31 attach_on_start="-a"
32 attach_on_run="-a stdout -a stderr"
33 else
34 attach_on_run="-d"
35 fi
36
37 if [ -n "$DOCKER_HOST" ]; then
38 docker_ip=`sed -e 's/^tcp:\/\/\(.*\):.*$/\1/' <<< "$DOCKER_HOST"`
39 elif [ -x "$(which ip 2>/dev/null)" ]; then
40 docker_ip=`ip addr show docker0 | \
41 grep 'inet ' | \
42 awk '{ split($2,a,"/"); print a[1] }';`
43 else
44 docker_ip=`ifconfig | \
45 grep -B1 "inet addr" | \
46 awk '{ if ( $1 == "inet" ) { print $2 } else if ( $2 == "Link" ) { printf "%s:" ,$1 } }' | \
47 grep docker0 | \
48 awk -F: '{ print $3 }';`
49 fi
50
51
52 usage () {
53 echo "Usage: launcher COMMAND CONFIG [--skip-prereqs]"
54 echo "Commands:"
55 echo " start: Start/initialize a container"
56 echo " stop: Stop a running container"
57 echo " restart: Restart a container"
58 echo " destroy: Stop and remove a container"
59 echo " enter: Use nsenter to enter a container"
60 echo " ssh: Start a bash shell in a running container"
61 echo " logs: Docker logs for container"
62 echo " bootstrap: Bootstrap a container for the config based on a template"
63 echo " rebuild: Rebuild a container (destroy old, bootstrap, start new)"
64 echo " cleanup: Remove all containers that have stopped for > 24 hours"
65 echo
66 echo "Options:"
67 echo " --skip-prereqs Don't check prerequisites or resource requirements"
68 echo " --docker-args Extra arguments to pass when running docker"
69 exit 1
70 }
71
72 compare_version() {
73 declare -a ver_a
74 declare -a ver_b
75 IFS=. read -a ver_a <<< "$1"
76 IFS=. read -a ver_b <<< "$2"
77
78 while [[ -n $ver_a ]]; do
79 if (( ver_a > ver_b )); then
80 return 0
81 elif (( ver_b > ver_a )); then
82 return 1
83 else
84 unset ver_a[0]
85 ver_a=("${ver_a[@]}")
86 unset ver_b[0]
87 ver_b=("${ver_b[@]}")
88 fi
89 done
90 return 1 # They are equal
91 }
92
93 prereqs() {
94
95 # 1. docker daemon running?
96 # we send stderr to /dev/null cause we don't care about warnings,
97 # it usually complains about swap which does not matter
98 test=`$docker_path info 2> /dev/null`
99
100 if [[ $? -ne 0 ]] ; then
101 echo "Cannot connect to the docker daemon - verify it is running and you have access"
102 exit 1
103 fi
104
105 # 2. running aufs or btrfs
106 test=`$docker_path info 2> /dev/null | grep 'Driver: '`
107 if [[ "$test" =~ [aufs|btrfs|zfs|overlay] ]] ; then : ; else
108 echo "Your Docker installation is not using a supported filesystem if we were to proceed you may have a broken install."
109 echo "aufs is the recommended filesystem you should be using (zfs/btrfs and overlay may work as well)"
110 echo "You can tell what filesystem you are using by running \"docker info\" and looking at the driver"
111 echo ""
112 echo "If you wish to continue anyway using your existing unsupported filesystem"
113 echo "read the source code of launcher and figure out how to bypass this."
114 exit 1
115 fi
116
117 # 3. running recommended docker version
118 test=($($docker_path --version)) # Get docker version string
119 test=${test[2]//,/} # Get version alone and strip comma if exists
120
121 [[ "$test" =~ "0.12.0" ]] && echo "You are running a broken version of Docker, please upgrade ASAP. See: https://meta.discourse.org/t/the-installation-stopped-in-the-middle/16311/ for more details." && exit 1
122
123 # At least minimum version
124 if compare_version "${docker_min_version}" "${test}"; then
125 echo "ERROR: Docker version ${test} not supported, please upgrade to at least ${docker_min_version}, or recommended ${docker_rec_version}"
126 exit 1
127 fi
128
129 # Recommend best version
130 if compare_version "${docker_rec_version}" "${test}"; then
131 echo "WARNING: Docker version ${test} deprecated, recommend upgrade to ${docker_rec_version} or newer."
132 fi
133
134 # 4. discourse docker image is downloaded
135 test=`$docker_path images | awk '{print $1 ":" $2 }' | grep "$image"`
136
137 if [ -z "$test" ]; then
138 echo
139 echo "WARNING: We are about to start downloading the Discourse base image"
140 echo "This process may take anywhere between a few minutes to an hour, depending on your network speed"
141 echo
142 echo "Please be patient"
143 echo
144
145 fi
146
147 # 5. able to attach stderr / out / tty
148 test=`$docker_path run $user_args -i --rm -a stdout -a stderr $image echo working`
149 if [[ "$test" =~ "working" ]] ; then : ; else
150 echo "Your Docker installation is not working correctly"
151 echo
152 echo "See: https://meta.discourse.org/t/docker-error-on-bootstrap/13657/18?u=sam"
153 exit 1
154 fi
155
156 }
157
158 check_resources() {
159 # Memory
160 resources="ok"
161 avail_mem="$(LANG=C free -m | grep '^Mem:' | awk '{print $2}')"
162 if [ "$avail_mem" -lt 900 ]; then
163 resources="insufficient"
164 echo "WARNING: You do not appear to have sufficient memory to run Discourse."
165 echo
166 echo "Your system may not work properly, or future upgrades of Discourse may"
167 echo "not complete successfully."
168 echo
169 echo "See https://github.com/discourse/discourse/blob/master/docs/INSTALL-cloud.md#create-new-cloud-server"
170 elif [ "$avail_mem" -lt 1800 ]; then
171 total_swap="$(LANG=C free -m | grep ^Swap: | awk '{print $2}')"
172 if [ "$total_swap" -lt 1000 ]; then
173 resources="insufficient"
174 echo "WARNING: You must have at least 1GB of swap when running with less"
175 echo "than 2GB of RAM."
176 echo
177 echo "Your system may not work properly, or future upgrades of Discourse may"
178 echo "not complete successfully."
179 echo
180 echo "See https://github.com/discourse/discourse/blob/master/docs/INSTALL-cloud.md#set-up-swap-if-needed"
181 fi
182 fi
183
184 # Disk space
185 free_disk="$(df /var | tail -n 1 | awk '{print $4}')"
186 if [ "$free_disk" -lt 5000 ]; then
187 resources="insufficient"
188 echo "WARNING: You must have at least 5GB of *free* disk space to run Discourse."
189 echo
190 echo "Insufficient disk space may result in problems running your site, and may"
191 echo "not even allow Discourse installation to complete successfully."
192 echo
193 echo "Please free up some space, or expand your disk, before continuing."
194 exit 1
195 fi
196
197 if [ -t 0 ] && [ "$resources" != "ok" ]; then
198 echo
199 read -p "Press ENTER to continue, or Ctrl-C to exit and give your system more resources"
200 fi
201 }
202
203 check_ports() {
204 local valid=$(netstat -tln | awk '{print $4}' | grep ":${1}\$")
205
206 if [ -n "$valid" ]; then
207 echo "Launcher has detected that port ${1} is in use."
208 echo ""
209 echo "If you are trying to run Discourse simultaneously with another web server like Apache or nginx, you will need to bind to a different port."
210 echo "See https://meta.discourse.org/t/17247 for help."
211 echo "To continue anyway, re-run Launcher with --skip-prereqs"
212 exit 1
213 fi
214 }
215
216 if [ "$opt" != "--skip-prereqs" ] ; then
217 prereqs
218 fi
219
220 if [ "$opt" == "--docker-args" ] ; then
221 user_args=$4
222 else
223 user_args=""
224 fi
225
226 get_ssh_pub_key() {
227 local ${ssh_key_locations}
228 ssh_key_locations=(
229 ~/.ssh/id_ed25519.pub
230 ~/.ssh/id_ecdsa.pub
231 ~/.ssh/id_rsa.pub
232 ~/.ssh/id_dsa.pub
233 ~core/.ssh/authorized_keys
234 )
235
236 local $keyfile
237 for keyfile in "${ssh_key_locations[@]}"; do
238 if [[ -e ${keyfile} ]] ; then
239 ssh_pub_key="$(cat ${keyfile})"
240 return 0
241 fi
242 done
243
244 return 0
245 }
246
247
248 install_docker() {
249
250 echo "Docker is not installed, you will need to install Docker in order to run Discourse"
251 echo "Please visit https://docs.docker.com/installation/ for instructions on how to do this for your system"
252 echo
253 echo "If you are running Ubuntu Trusty or later, you can try the following:"
254 echo
255
256 echo "sudo apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D"
257 echo "sudo sh -c \"echo deb https://apt.dockerproject.org/repo ubuntu-$(lsb_release -sc) main > /etc/apt/sources.list.d/docker.list\""
258 echo "sudo apt-get update"
259 echo "sudo apt-get install docker-engine"
260
261 exit 1
262 }
263
264 host_run() {
265 read -r -d '' env_ruby << 'RUBY'
266 require 'yaml'
267
268 input = STDIN.readlines.join
269 yaml = YAML.load(input)
270
271 if host_run = yaml['host_run']
272 params = yaml['params'] || {}
273 host_run.each do |run|
274 params.each do |k,v|
275 run = run.gsub("$#{k}", v)
276 end
277 STDOUT.write "#{run}--SEP--"
278 end
279 end
280 RUBY
281
282 host_run=`cat $config_file | $docker_path run $user_args --rm -i -a stdout -a stdin $image ruby -e "$env_ruby"`
283
284 while [ "$host_run" ] ; do
285 iter=${host_run%%--SEP--*}
286 echo
287 echo "Host run: $iter"
288 $iter || exit 1
289 echo
290 host_run="${host_run#*--SEP--}"
291 done
292 }
293
294
295 set_volumes() {
296 volumes=`cat $config_file | $docker_path run $user_args --rm -i -a stdout -a stdin $image ruby -e \
297 "require 'yaml'; puts YAML.load(STDIN.readlines.join)['volumes'].map{|v| '-v ' << v['volume']['host'] << ':' << v['volume']['guest'] << ' '}.join"`
298 }
299
300 set_links() {
301 links=`cat $config_file | $docker_path run $user_args --rm -i -a stdout -a stdin $image ruby -e \
302 "require 'yaml'; puts YAML.load(STDIN.readlines.join)['links'].map{|l| '--link ' << l['link']['name'] << ':' << l['link']['alias'] << ' '}.join"`
303 }
304
305 set_template_info() {
306
307 templates=`cat $config_file | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \
308 "require 'yaml'; puts YAML.load(STDIN.readlines.join)['templates']"`
309
310
311 arrTemplates=(${templates// / })
312 config_data=$(cat $config_file)
313
314 input="hack: true"
315
316
317 for template in "${arrTemplates[@]}"
318 do
319 [ ! -z $template ] && {
320 input="$input _FILE_SEPERATOR_ $(cat $template)"
321 }
322 done
323
324 # we always want our config file last so it takes priority
325 input="$input _FILE_SEPERATOR_ $config_data"
326
327 read -r -d '' env_ruby << 'RUBY'
328 require 'yaml'
329
330 input=STDIN.readlines.join
331 # default to UTF-8 for the dbs sake
332 env = {'LANG' => 'en_US.UTF-8'}
333 input.split('_FILE_SEPERATOR_').each do |yml|
334 yml.strip!
335 begin
336 env.merge!(YAML.load(yml)['env'] || {})
337 rescue Psych::SyntaxError => e
338 puts e
339 puts "*ERROR."
340 rescue => e
341 puts yml
342 p e
343 end
344 end
345 puts env.map{|k,v| "-e\n#{k}=#{v}" }.join("\n")
346 RUBY
347
348 raw=`exec echo "$input" | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e "$env_ruby"`
349
350 env=()
351 ok=1
352 while read i; do
353 if [ "$i" == "*ERROR." ]; then
354 ok=0
355 elif [ -n "$i" ]; then
356 env[${#env[@]}]=$i
357 fi
358 done <<< "$raw"
359
360 if [ "$ok" -ne 1 ]; then
361 echo "${env[@]}"
362 echo "YAML syntax error. Please check your /var/discourse/containers/*.yml config files."
363 exit 1
364 fi
365 }
366
367 [ -z $docker_path ] && {
368 install_docker
369 }
370
371 [ "$command" == "cleanup" ] && {
372 echo
373 echo "The following command will"
374 echo "- Delete all docker images for old containers"
375 echo "- Delete all stopped and orphan containers"
376 echo
377 read -p "Are you sure (Y/n): " -n 1 -r && echo
378 if [[ $REPLY =~ ^[Yy]$ || ! $REPLY ]]
379 then
380 space=$(df /var/lib/docker | awk '{ print $4 }' | grep -v Available)
381 echo "Starting Cleanup (bytes free $space)"
382
383 STATE_DIR=./.gc-state scripts/docker-gc
384
385 space=$(df /var/lib/docker | awk '{ print $4 }' | grep -v Available)
386 echo "Finished Cleanup (bytes free $space)"
387
388 else
389 exit 1
390 fi
391 exit 0
392 }
393
394 [ $# -lt 2 ] && {
395 usage
396 }
397
398 if [ ! -e $config_file ]
399 then
400 echo "Config file was not found, ensure $config_file exists"
401 echo ""
402 echo "Available configs ( `cd containers && ls -dm *.yml | tr -s '\n' ' ' | awk '{ gsub(/\.yml/, ""); print }'`)"
403 exit 1
404 fi
405
406
407 docker_version=($($docker_path --version))
408 docker_version=${test[2]//,/}
409
410 if compare_version "1.2.0" "$docker_version"; then
411 echo "We recommend you upgrade docker, the version you are running has no restart policies, on reboot your container may not start up"
412 restart_policy=""
413 else
414 restart_policy=${restart_policy:---restart=always}
415 fi
416
417 set_existing_container(){
418 existing=`$docker_path ps -a | awk '{ print $1, $(NF) }' | grep " $config$" | awk '{ print $1 }'`
419 }
420
421 run_stop(){
422
423 set_existing_container
424
425 if [ ! -z $existing ]
426 then
427 (
428 set -x
429 $docker_path stop -t 10 $config
430 )
431 else
432 echo "$config was not started !"
433 exit 1
434 fi
435 }
436
437 set_run_image() {
438 run_image=`cat $config_file | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \
439 "require 'yaml'; puts YAML.load(STDIN.readlines.join)['run_image']"`
440
441 if [ -z "$run_image" ]; then
442 run_image="$local_discourse/$config"
443 fi
444 }
445
446 set_boot_command() {
447 boot_command=`cat $config_file | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \
448 "require 'yaml'; puts YAML.load(STDIN.readlines.join)['boot_command']"`
449
450 if [ -z "$boot_command" ]; then
451
452 no_boot_command=`cat $config_file | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \
453 "require 'yaml'; puts YAML.load(STDIN.readlines.join)['no_boot_command']"`
454
455 if [ -z "$no_boot_command" ]; then
456 boot_command="/sbin/boot"
457 fi
458 fi
459 }
460
461 run_start(){
462
463 existing=`$docker_path ps | awk '{ print $1, $(NF) }' | grep " $config$" | awk '{ print $1 }'`
464 echo $existing
465 if [ ! -z $existing ]
466 then
467 echo "Nothing to do, your container has already started!"
468 exit 1
469 fi
470
471 existing=`$docker_path ps -a | awk '{ print $1, $(NF) }' | grep " $config$" | awk '{ print $1 }'`
472 if [ ! -z $existing ]
473 then
474 echo "starting up existing container"
475 (
476 set -x
477 $docker_path start $config
478 )
479 exit 0
480 fi
481
482 host_run
483 ports=`cat $config_file | $docker_path run $user_args --rm -i -a stdout -a stdin $image ruby -e \
484 "require 'yaml'; puts YAML.load(STDIN.readlines.join)['expose'].map{|p| \"-p #{p}\"}.join(' ')"`
485
486 IFS='-p ' read -a array <<< "$ports"
487 for element in "${array[@]}"
488 do
489 IFS=':' read -a args <<< "$element"
490 if [ "${#args[@]}" == "2" ]; then
491 check_ports "${args[0]}"
492 elif [ "${#args[@]}" == "3" ]; then
493 check_ports "${args[1]}"
494 fi
495 done
496
497 docker_args=`cat $config_file | $docker_path run $user_args --rm -i -a stdout -a stdin $image ruby -e \
498 "require 'yaml'; puts YAML.load(STDIN.readlines.join)['docker_args']"`
499
500 set_template_info
501 set_volumes
502 set_links
503 set_run_image
504 set_boot_command
505
506 # get hostname and settings from container configuration
507 for envar in "${env[@]}"
508 do
509 if [[ $envar == DOCKER_USE_HOSTNAME* ]] || [[ $envar == DISCOURSE_HOSTNAME* ]]
510 then
511 # use as environment variable
512 eval $envar
513 fi
514 done
515
516 (
517 hostname=`hostname -s`
518 # overwrite hostname
519 if [ "$DOCKER_USE_HOSTNAME" = "true" ]
520 then
521 hostname=$DISCOURSE_HOSTNAME
522 else
523 hostname=$hostname-$config
524 fi
525
526 # we got to normalize so we only have allowed strings, this is more comprehensive but lets see how bash does first
527 # hostname=`$docker_path run $user_args --rm $image ruby -e 'print ARGV[0].gsub(/[^a-zA-Z-]/, "-")' $hostname`
528 # docker added more hostname rules
529 hostname=${hostname/_/-}
530
531 set -x
532 $docker_path run $user_args $links $attach_on_run $restart_policy "${env[@]}" -h "$hostname" \
533 -e DOCKER_HOST_IP=$docker_ip --name $config -t $ports $volumes $docker_args $run_image $boot_command
534
535 )
536 exit 0
537
538 }
539
540 run_bootstrap(){
541
542 if [ "$opt" != "--skip-prereqs" ] ; then
543 check_resources
544 fi
545
546 host_run
547
548 get_ssh_pub_key
549
550 # Is the image available?
551 # If not, pull it here so the user is aware what's happening.
552 $docker_path history $image >/dev/null 2>&1 || $docker_path pull $image
553
554 set_template_info
555
556 base_image=`cat $config_file | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \
557 "require 'yaml'; puts YAML.load(STDIN.readlines.join)['base_image']"`
558
559 update_pups=`cat $config_file | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \
560 "require 'yaml'; puts YAML.load(STDIN.readlines.join)['update_pups']"`
561
562 if [[ ! X"" = X"$base_image" ]]; then
563 image=$base_image
564 fi
565
566 set_volumes
567 set_links
568
569 rm -f $cidbootstrap
570
571 run_command="cd /pups &&"
572 if [[ ! "false" = $update_pups ]]; then
573 run_command="$run_command git pull &&"
574 fi
575 run_command="$run_command /pups/bin/pups --stdin"
576
577 echo $run_command
578
579 env=("${env[@]}" "-e" "SSH_PUB_KEY=$ssh_pub_key")
580
581 (exec echo "$input" | $docker_path run $user_args $links "${env[@]}" -e DOCKER_HOST_IP=$docker_ip --cidfile $cidbootstrap -i -a stdin -a stdout -a stderr $volumes $image \
582 /bin/bash -c "$run_command") \
583 || ($docker_path rm `cat $cidbootstrap` && rm $cidbootstrap)
584
585 [ ! -e $cidbootstrap ] && echo "** FAILED TO BOOTSTRAP ** please scroll up and look for earlier error messages, there may be more than one" && exit 1
586
587 sleep 5
588
589 $docker_path commit `cat $cidbootstrap` $local_discourse/$config || echo 'FAILED TO COMMIT'
590 $docker_path rm `cat $cidbootstrap` && rm $cidbootstrap
591 }
592
593
594
595 case "$command" in
596 bootstrap)
597 run_bootstrap
598 echo "Successfully bootstrapped, to startup use ./launcher start $config"
599 exit 0
600 ;;
601
602 enter)
603 exec $docker_path exec -it $config /bin/bash --login
604 ;;
605
606 ssh)
607 existing=`$docker_path ps | awk '{ print $1, $(NF) }' | grep " $config$" | awk '{ print $1 }'`
608
609 if [[ ! -z $existing ]]; then
610 address="`$docker_path port $config 22`"
611 split=(${address//:/ })
612 exec ssh -o StrictHostKeyChecking=no root@${split[0]} -p ${split[1]}
613 else
614 echo "$config is not running!"
615 exit 1
616 fi
617 ;;
618
619 stop)
620 run_stop
621 exit 0
622 ;;
623
624 logs)
625
626 $docker_path logs $config
627 exit 0
628 ;;
629
630 restart)
631 run_stop
632 run_start
633 exit 0
634 ;;
635
636 start)
637 run_start
638 exit 0
639 ;;
640
641 rebuild)
642 if [ "$(git symbolic-ref --short HEAD)" == "master" ]; then
643 echo "Ensuring discourse docker is up to date"
644
645 git remote update
646
647 LOCAL=$(git rev-parse @)
648 REMOTE=$(git rev-parse @{u})
649 BASE=$(git merge-base @ @{u})
650
651 if [ $LOCAL = $REMOTE ]; then
652 echo "Discourse Docker is up-to-date"
653
654 elif [ $LOCAL = $BASE ]; then
655 echo "Updating Discourse Docker"
656 git pull || (echo 'failed to update' && exit 1)
657 exec /bin/bash $0 $@
658
659 elif [ $REMOTE = $BASE ]; then
660 echo "Your version of Discourse Docker is ahead of origin"
661
662 else
663 echo "Discourse Docker has diverged source, this is only expected in Dev mode"
664 fi
665
666 fi
667
668 set_existing_container
669
670 if [ ! -z $existing ]
671 then
672 echo "Stopping old container"
673 (
674 set -x
675 $docker_path stop -t 10 $config
676 )
677 fi
678
679 run_bootstrap
680
681 if [ ! -z $existing ]
682 then
683 echo "Removing old container"
684 (
685 set -x
686 $docker_path rm $config
687 )
688 fi
689
690 run_start
691 exit 0
692 ;;
693
694
695 destroy)
696 (set -x; $docker_path stop -t 10 $config && $docker_path rm $config) || (echo "$config was not found" && exit 0)
697 exit 0
698 ;;
699 esac
700
701 usage