make bootstrap auto-set unicorn_workers and db memory
[discourse_docker.git] / launcher
1 #!/bin/bash
2
3 usage () {
4 echo "Usage: launcher COMMAND CONFIG [--skip-prereqs]"
5 echo "Commands:"
6 echo " start: Start/initialize a container"
7 echo " stop: Stop a running container"
8 echo " restart: Restart a container"
9 echo " destroy: Stop and remove a container"
10 echo " enter: Use nsenter to enter a container"
11 echo " logs: Docker logs for container"
12 echo " bootstrap: Bootstrap a container for the config based on a template"
13 echo " rebuild: Rebuild a container (destroy old, bootstrap, start new)"
14 echo " cleanup: Remove all containers that have stopped for > 24 hours"
15 echo
16 echo "Options:"
17 echo " --skip-prereqs Don't check prerequisites or resource requirements"
18 echo " --docker-args Extra arguments to pass when running docker"
19 exit 1
20 }
21
22 command=$1
23 config=$2
24 opt=$3
25
26 # Docker doesn't like uppercase characters, spaces or special characters, catch it now before we build everything out and then find out
27 re='[A-Z/ !@#$%^&*()+~`=]'
28 if [[ $config =~ $re ]];
29 then
30 echo
31 echo "ERROR: Config name must not contain upper case characters, spaces or special characters. Correct config name and rerun $0."
32 echo
33 exit 1
34 fi
35
36 cd "$(dirname "$0")"
37
38 docker_min_version='1.6.0'
39 docker_rec_version='1.6.0'
40
41 config_file=containers/"$config".yml
42 cidbootstrap=cids/"$config"_bootstrap.cid
43 local_discourse=local_discourse
44 image=discourse/discourse:1.0.17
45 docker_path=`which docker.io || which docker`
46 template_path=samples/standalone.yml
47 changelog=/tmp/changelog # used to test whether sed did anything
48
49 if [ "${SUPERVISED}" = "true" ]; then
50 restart_policy="--restart=no"
51 attach_on_start="-a"
52 attach_on_run="-a stdout -a stderr"
53 else
54 attach_on_run="-d"
55 fi
56
57 if [ -n "$DOCKER_HOST" ]; then
58 docker_ip=`sed -e 's/^tcp:\/\/\(.*\):.*$/\1/' <<< "$DOCKER_HOST"`
59 elif [ -x "$(which ip 2>/dev/null)" ]; then
60 docker_ip=`ip addr show docker0 | \
61 grep 'inet ' | \
62 awk '{ split($2,a,"/"); print a[1] }';`
63 else
64 docker_ip=`ifconfig | \
65 grep -B1 "inet addr" | \
66 awk '{ if ( $1 == "inet" ) { print $2 } else if ( $2 == "Link" ) { printf "%s:" ,$1 } }' | \
67 grep docker0 | \
68 awk -F: '{ print $3 }';`
69 fi
70
71 compare_version() {
72 declare -a ver_a
73 declare -a ver_b
74 IFS=. read -a ver_a <<< "$1"
75 IFS=. read -a ver_b <<< "$2"
76
77 while [[ -n $ver_a ]]; do
78 if (( ver_a > ver_b )); then
79 return 0
80 elif (( ver_b > ver_a )); then
81 return 1
82 else
83 unset ver_a[0]
84 ver_a=("${ver_a[@]}")
85 unset ver_b[0]
86 ver_b=("${ver_b[@]}")
87 fi
88 done
89 return 1 # They are equal
90 }
91
92
93 install_docker() {
94
95 echo "Docker is not installed, you will need to install Docker in order to run Discourse"
96 echo "Please visit https://docs.docker.com/installation/ for instructions on how to do this for your system"
97 echo
98 echo "If you are running a recent Ubuntu Server, try the following:"
99 echo "sudo apt-get install docker-engine"
100
101 exit 1
102 }
103
104 prereqs() {
105
106 if [ -z $docker_path ]; then
107 install_docker
108 fi
109
110 # 1. docker daemon running?
111 # we send stderr to /dev/null cause we don't care about warnings,
112 # it usually complains about swap which does not matter
113 test=`$docker_path info 2> /dev/null`
114 if [[ $? -ne 0 ]] ; then
115 echo "Cannot connect to the docker daemon - verify it is running and you have access"
116 exit 1
117 fi
118
119 # 2. running aufs or btrfs
120 test=`$docker_path info 2> /dev/null | grep 'Driver: '`
121 if [[ "$test" =~ [aufs|btrfs|zfs|overlay] ]] ; then : ; else
122 echo "Your Docker installation is not using a supported filesystem if we were to proceed you may have a broken install."
123 echo "aufs is the recommended filesystem you should be using (zfs/btrfs and overlay may work as well)"
124 echo "You can tell what filesystem you are using by running \"docker info\" and looking at the driver"
125 echo ""
126 echo "If you wish to continue anyway using your existing unsupported filesystem"
127 echo "read the source code of launcher and figure out how to bypass this."
128 exit 1
129 fi
130
131 # 3. running recommended docker version
132 test=($($docker_path --version)) # Get docker version string
133 test=${test[2]//,/} # Get version alone and strip comma if exists
134
135 # At least minimum version
136 if compare_version "${docker_min_version}" "${test}"; then
137 echo "ERROR: Docker version ${test} not supported, please upgrade to at least ${docker_min_version}, or recommended ${docker_rec_version}"
138 exit 1
139 fi
140
141 # Recommend best version
142 if compare_version "${docker_rec_version}" "${test}"; then
143 echo "WARNING: Docker version ${test} deprecated, recommend upgrade to ${docker_rec_version} or newer."
144 fi
145
146 # 4. discourse docker image is downloaded
147 test=`$docker_path images | awk '{print $1 ":" $2 }' | grep "$image"`
148
149 if [ -z "$test" ]; then
150 echo
151 echo "WARNING: We are about to start downloading the Discourse base image"
152 echo "This process may take anywhere between a few minutes to an hour, depending on your network speed"
153 echo
154 echo "Please be patient"
155 echo
156
157 fi
158
159 # 5. able to attach stderr / out / tty
160 test=`$docker_path run $user_args -i --rm -a stdout -a stderr $image echo working`
161 if [[ "$test" =~ "working" ]] ; then : ; else
162 echo "Your Docker installation is not working correctly"
163 echo
164 echo "See: https://meta.discourse.org/t/docker-error-on-bootstrap/13657/18?u=sam"
165 exit 1
166 fi
167
168 }
169
170 check_resources() {
171 # Memory
172 resources="ok"
173 avail_mem="$(LANG=C free -m | grep '^Mem:' | awk '{print $2}')"
174 if [ "$avail_mem" -lt 900 ]; then
175 resources="insufficient"
176 echo "WARNING: You do not appear to have sufficient memory to run Discourse."
177 echo
178 echo "Your system may not work properly, or future upgrades of Discourse may"
179 echo "not complete successfully."
180 echo
181 echo "See https://github.com/discourse/discourse/blob/master/docs/INSTALL-cloud.md#create-new-cloud-server"
182 elif [ "$avail_mem" -lt 1800 ]; then
183 total_swap="$(LANG=C free -m | grep ^Swap: | awk '{print $2}')"
184 if [ "$total_swap" -lt 1000 ]; then
185 resources="insufficient"
186 echo "WARNING: You must have at least 1GB of swap when running with less"
187 echo "than 2GB of RAM."
188 echo
189 echo "Your system may not work properly, or future upgrades of Discourse may"
190 echo "not complete successfully."
191 echo
192 echo "See https://github.com/discourse/discourse/blob/master/docs/INSTALL-cloud.md#set-up-swap-if-needed"
193 fi
194 fi
195
196 # Disk space
197 free_disk="$(df /var | tail -n 1 | awk '{print $4}')"
198 if [ "$free_disk" -lt 5000 ]; then
199 resources="insufficient"
200 echo "WARNING: You must have at least 5GB of *free* disk space to run Discourse."
201 echo
202 echo "Insufficient disk space may result in problems running your site, and may"
203 echo "not even allow Discourse installation to complete successfully."
204 echo
205 echo "Please free up some space, or expand your disk, before continuing."
206 exit 1
207 fi
208
209 if [ -t 0 ] && [ "$resources" != "ok" ]; then
210 echo
211 read -p "Press ENTER to continue, or Ctrl-C to exit and give your system more resources"
212 fi
213 }
214
215 check_ports() {
216 local valid=$(netstat -tln | awk '{print $4}' | grep ":${1}\$")
217
218 if [ -n "$valid" ]; then
219 echo "Launcher has detected that port ${1} is in use."
220 echo ""
221 echo "If you are trying to run Discourse simultaneously with another web server like Apache or nginx, you will need to bind to a different port."
222 echo "See https://meta.discourse.org/t/17247 for help."
223 echo "To continue anyway, re-run Launcher with --skip-prereqs"
224 exit 1
225 fi
226 }
227
228 if [ "$opt" != "--skip-prereqs" ] ; then
229 prereqs
230 fi
231
232 if [ "$opt" == "--docker-args" ] ; then
233 user_args=$4
234 else
235 user_args=""
236 fi
237
238 host_run() {
239 read -r -d '' env_ruby << 'RUBY'
240 require 'yaml'
241
242 input = STDIN.readlines.join
243 yaml = YAML.load(input)
244
245 if host_run = yaml['host_run']
246 params = yaml['params'] || {}
247 host_run.each do |run|
248 params.each do |k,v|
249 run = run.gsub("$#{k}", v)
250 end
251 STDOUT.write "#{run}--SEP--"
252 end
253 end
254 RUBY
255
256 host_run=`cat $config_file | $docker_path run $user_args --rm -i -a stdout -a stdin $image ruby -e "$env_ruby"`
257
258 while [ "$host_run" ] ; do
259 iter=${host_run%%--SEP--*}
260 echo
261 echo "Host run: $iter"
262 $iter || exit 1
263 echo
264 host_run="${host_run#*--SEP--}"
265 done
266 }
267
268
269 set_volumes() {
270 volumes=`cat $config_file | $docker_path run $user_args --rm -i -a stdout -a stdin $image ruby -e \
271 "require 'yaml'; puts YAML.load(STDIN.readlines.join)['volumes'].map{|v| '-v ' << v['volume']['host'] << ':' << v['volume']['guest'] << ' '}.join"`
272 }
273
274 set_links() {
275 links=`cat $config_file | $docker_path run $user_args --rm -i -a stdout -a stdin $image ruby -e \
276 "require 'yaml'; puts YAML.load(STDIN.readlines.join)['links'].map{|l| '--link ' << l['link']['name'] << ':' << l['link']['alias'] << ' '}.join"`
277 }
278
279 set_template_info() {
280
281 templates=`cat $config_file | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \
282 "require 'yaml'; puts YAML.load(STDIN.readlines.join)['templates']"`
283
284 arrTemplates=(${templates// / })
285 config_data=$(cat $config_file)
286
287 input="hack: true"
288
289 for template in "${arrTemplates[@]}"
290 do
291 [ ! -z $template ] && {
292 input="$input _FILE_SEPERATOR_ $(cat $template)"
293 }
294 done
295
296 # we always want our config file last so it takes priority
297 input="$input _FILE_SEPERATOR_ $config_data"
298
299 read -r -d '' env_ruby << 'RUBY'
300 require 'yaml'
301
302 input=STDIN.readlines.join
303 # default to UTF-8 for the dbs sake
304 env = {'LANG' => 'en_US.UTF-8'}
305 input.split('_FILE_SEPERATOR_').each do |yml|
306 yml.strip!
307 begin
308 env.merge!(YAML.load(yml)['env'] || {})
309 rescue Psych::SyntaxError => e
310 puts e
311 puts "*ERROR."
312 rescue => e
313 puts yml
314 p e
315 end
316 end
317 puts env.map{|k,v| "-e\n#{k}=#{v}" }.join("\n")
318 RUBY
319
320 raw=`exec echo "$input" | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e "$env_ruby"`
321
322 env=()
323 ok=1
324 while read i; do
325 if [ "$i" == "*ERROR." ]; then
326 ok=0
327 elif [ -n "$i" ]; then
328 env[${#env[@]}]=$i
329 fi
330 done <<< "$raw"
331
332 if [ "$ok" -ne 1 ]; then
333 echo "${env[@]}"
334 echo "YAML syntax error. Please check your /var/discourse/containers/*.yml config files."
335 exit 1
336 fi
337 }
338
339 if [ -z $docker_path ]; then
340 install_docker
341 fi
342
343 [ "$command" == "cleanup" ] && {
344 echo
345 echo "The following command will"
346 echo "- Delete all docker images for old containers"
347 echo "- Delete all stopped and orphan containers"
348 echo
349 read -p "Are you sure (Y/n): " -n 1 -r && echo
350 if [[ $REPLY =~ ^[Yy]$ || ! $REPLY ]]
351 then
352 space=$(df /var/lib/docker | awk '{ print $4 }' | grep -v Available)
353 echo "Starting Cleanup (bytes free $space)"
354
355 STATE_DIR=./.gc-state scripts/docker-gc
356
357 space=$(df /var/lib/docker | awk '{ print $4 }' | grep -v Available)
358 echo "Finished Cleanup (bytes free $space)"
359
360 else
361 exit 1
362 fi
363 exit 0
364 }
365
366 [ $# -lt 2 ] && {
367 usage
368 }
369
370 if [[ ! -e $config_file && $command -ne "memconfig" ]]
371 then
372 echo "Config file was not found, ensure $config_file exists"
373 echo ""
374 echo "Available configs ( `cd containers && ls -dm *.yml | tr -s '\n' ' ' | awk '{ gsub(/\.yml/, ""); print }'`)"
375 exit 1
376 fi
377
378 docker_version=($($docker_path --version))
379 docker_version=${test[2]//,/}
380 restart_policy=${restart_policy:---restart=always}
381
382 set_existing_container(){
383 existing=`$docker_path ps -a | awk '{ print $1, $(NF) }' | grep " $config$" | awk '{ print $1 }'`
384 }
385
386 run_stop() {
387
388 set_existing_container
389
390 if [ ! -z $existing ]
391 then
392 (
393 set -x
394 $docker_path stop -t 10 $config
395 )
396 else
397 echo "$config was not started !"
398 exit 1
399 fi
400 }
401
402 set_run_image() {
403 run_image=`cat $config_file | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \
404 "require 'yaml'; puts YAML.load(STDIN.readlines.join)['run_image']"`
405
406 if [ -z "$run_image" ]; then
407 run_image="$local_discourse/$config"
408 fi
409 }
410
411 set_boot_command() {
412 boot_command=`cat $config_file | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \
413 "require 'yaml'; puts YAML.load(STDIN.readlines.join)['boot_command']"`
414
415 if [ -z "$boot_command" ]; then
416
417 no_boot_command=`cat $config_file | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \
418 "require 'yaml'; puts YAML.load(STDIN.readlines.join)['no_boot_command']"`
419
420 if [ -z "$no_boot_command" ]; then
421 boot_command="/sbin/boot"
422 fi
423 fi
424 }
425
426 scale_ram_and_cpu() {
427
428 # get free mem
429 avail_mem="$(LANG=C free -m | grep '^Mem:' | awk '{print $2}')"
430 avail_gb=`expr $(($avail_mem / 950))`
431 avail_cores=`grep -c processor /proc/cpuinfo`
432 echo "Found ${avail_gb}GB of memory and $avail_cores CPU cores"
433
434 # set db_shared_buffers: "128MB" (1GB) or 256MB * GB
435 if [ "$avail_gb" -eq "1" ]
436 then
437 db_shared_buffers="128"
438 else
439 db_shared_buffers=`expr $avail_gb \* 256`
440 fi
441 sed -i -e "s/^ #db_shared_buffers:.*/ db_shared_buffers: \"${db_shared_buffers}MB\"/w $changelog" $config_file
442 if [ -s $changelog ]
443 then
444 echo "setting db_shared_buffers = ${db_shared_buffers}MB based on detected CPU/RAM"
445 rm $changelog
446 fi
447
448
449 # set UNICORN_WORKERS: 2*GB or 2*cores (the same on DO)
450 if [ "$avail_gb" -le "2" ]
451 then
452 unicorn_workers=`expr $avail_gb \* 2`
453 else
454 unicorn_workers=`expr \$avail_cores \* 2`
455 fi
456 sed -i -e "s/^ #UNICORN_WORKERS:.*/ UNICORN_WORKERS: ${unicorn_workers}/w $changelog" $config_file
457 if [ -s $changelog ]
458 then
459 echo "setting UNICORN_WORKERS = ${unicorn_workers} based on detected CPU/RAM"
460 rm $changelog
461 fi
462
463 }
464
465 run_start() {
466
467 existing=`$docker_path ps | awk '{ print $1, $(NF) }' | grep " $config$" | awk '{ print $1 }'`
468 echo $existing
469 if [ ! -z $existing ]
470 then
471 echo "Nothing to do, your container has already started!"
472 exit 1
473 fi
474
475 existing=`$docker_path ps -a | awk '{ print $1, $(NF) }' | grep " $config$" | awk '{ print $1 }'`
476 if [ ! -z $existing ]
477 then
478 echo "starting up existing container"
479 (
480 set -x
481 $docker_path start $config
482 )
483 exit 0
484 fi
485
486 host_run
487 ports=`cat $config_file | $docker_path run $user_args --rm -i -a stdout -a stdin $image ruby -e \
488 "require 'yaml'; puts YAML.load(STDIN.readlines.join)['expose'].map{|p| \"-p #{p}\"}.join(' ')"`
489
490 IFS='-p ' read -a array <<< "$ports"
491 for element in "${array[@]}"
492 do
493 IFS=':' read -a args <<< "$element"
494 if [ "${#args[@]}" == "2" ]; then
495 check_ports "${args[0]}"
496 elif [ "${#args[@]}" == "3" ]; then
497 check_ports "${args[1]}"
498 fi
499 done
500
501 docker_args=`cat $config_file | $docker_path run $user_args --rm -i -a stdout -a stdin $image ruby -e \
502 "require 'yaml'; puts YAML.load(STDIN.readlines.join)['docker_args']"`
503
504 set_template_info
505 set_volumes
506 set_links
507 set_run_image
508 set_boot_command
509
510 # get hostname and settings from container configuration
511 for envar in "${env[@]}"
512 do
513 if [[ $envar == DOCKER_USE_HOSTNAME* ]] || [[ $envar == DISCOURSE_HOSTNAME* ]]
514 then
515 # use as environment variable
516 eval $envar
517 fi
518 done
519
520 (
521 hostname=`hostname -s`
522 # overwrite hostname
523 if [ "$DOCKER_USE_HOSTNAME" = "true" ]
524 then
525 hostname=$DISCOURSE_HOSTNAME
526 else
527 hostname=$hostname-$config
528 fi
529
530 # we got to normalize so we only have allowed strings, this is more comprehensive but lets see how bash does first
531 # hostname=`$docker_path run $user_args --rm $image ruby -e 'print ARGV[0].gsub(/[^a-zA-Z-]/, "-")' $hostname`
532 # docker added more hostname rules
533 hostname=${hostname/_/-}
534
535 set -x
536 $docker_path run $user_args $links $attach_on_run $restart_policy "${env[@]}" -h "$hostname" \
537 -e DOCKER_HOST_IP=$docker_ip --name $config -t $ports $volumes $docker_args $run_image $boot_command
538
539 )
540 exit 0
541
542 }
543
544 valid_config_check() {
545
546 valid_config="y"
547 for x in DISCOURSE_SMTP_ADDRESS DISCOURSE_SMTP_USER_NAME DISCOURSE_SMTP_PASSWORD \
548 DISCOURSE_DEVELOPER_EMAILS DISCOURSE_HOSTNAME
549 do
550 mail_var=`grep "^ $x:" $config_file`
551 result=$?
552 if (( result == 0 ))
553 then
554 if [[ $mail_var = *"example.com"* ]]
555 then
556 echo "Warning: $x left at incorrect default of example.com"
557 valid_config="n"
558 fi
559 else
560 echo "Warning: $x not configured."
561 valid_config="n"
562 fi
563 done
564 if [ -t 0 ] && [ "$valid_config" != "y" ]; then
565 echo ""
566 echo "Please edit $config_file to add missing settings."
567 exit 1
568 fi
569 }
570
571 run_bootstrap() {
572
573 # Does your system meet the minimum requirements?
574 if [ "$opt" != "--skip-prereqs" ] ; then
575 check_resources
576 fi
577
578 # is our configuration file valid?
579 valid_config_check
580
581 # make minor scaling adjustments for RAM and CPU
582 scale_ram_and_cpu
583
584 host_run
585
586 # Is the image available?
587 # If not, pull it here so the user is aware what's happening.
588 $docker_path history $image >/dev/null 2>&1 || $docker_path pull $image
589
590 set_template_info
591
592 base_image=`cat $config_file | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \
593 "require 'yaml'; puts YAML.load(STDIN.readlines.join)['base_image']"`
594
595 update_pups=`cat $config_file | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \
596 "require 'yaml'; puts YAML.load(STDIN.readlines.join)['update_pups']"`
597
598 if [[ ! X"" = X"$base_image" ]]; then
599 image=$base_image
600 fi
601
602 set_volumes
603 set_links
604
605 rm -f $cidbootstrap
606
607 run_command="cd /pups &&"
608 if [[ ! "false" = $update_pups ]]; then
609 run_command="$run_command git pull &&"
610 fi
611 run_command="$run_command /pups/bin/pups --stdin"
612
613 echo $run_command
614
615 (exec echo "$input" | $docker_path run $user_args $links "${env[@]}" -e DOCKER_HOST_IP=$docker_ip --cidfile $cidbootstrap -i -a stdin -a stdout -a stderr $volumes $image \
616 /bin/bash -c "$run_command") \
617 || ($docker_path rm `cat $cidbootstrap` && rm $cidbootstrap)
618
619 [ ! -e $cidbootstrap ] && echo "** FAILED TO BOOTSTRAP ** please scroll up and look for earlier error messages, there may be more than one" && exit 1
620
621 sleep 5
622
623 $docker_path commit `cat $cidbootstrap` $local_discourse/$config || echo 'FAILED TO COMMIT'
624 $docker_path rm `cat $cidbootstrap` && rm $cidbootstrap
625 }
626
627
628
629 case "$command" in
630 bootstrap)
631 run_bootstrap
632 echo "Successfully bootstrapped, to startup use ./launcher start $config"
633 exit 0
634 ;;
635
636 enter)
637 exec $docker_path exec -it $config /bin/bash --login
638 ;;
639
640 stop)
641 run_stop
642 exit 0
643 ;;
644
645 logs)
646
647 $docker_path logs $config
648 exit 0
649 ;;
650
651 restart)
652 run_stop
653 run_start
654 exit 0
655 ;;
656
657 start)
658 run_start
659 exit 0
660 ;;
661
662 rebuild)
663 if [ "$(git symbolic-ref --short HEAD)" == "master" ]; then
664 echo "Ensuring discourse docker is up to date"
665
666 git remote update
667
668 LOCAL=$(git rev-parse @)
669 REMOTE=$(git rev-parse @{u})
670 BASE=$(git merge-base @ @{u})
671
672 if [ $LOCAL = $REMOTE ]; then
673 echo "Discourse Docker is up-to-date"
674
675 elif [ $LOCAL = $BASE ]; then
676 echo "Updating Discourse Docker"
677 git pull || (echo 'failed to update' && exit 1)
678 exec /bin/bash $0 $@
679
680 elif [ $REMOTE = $BASE ]; then
681 echo "Your version of Discourse Docker is ahead of origin"
682
683 else
684 echo "Discourse Docker has diverged source, this is only expected in Dev mode"
685 fi
686
687 fi
688
689 set_existing_container
690
691 if [ ! -z $existing ]
692 then
693 echo "Stopping old container"
694 (
695 set -x
696 $docker_path stop -t 10 $config
697 )
698 fi
699
700 run_bootstrap
701
702 if [ ! -z $existing ]
703 then
704 echo "Removing old container"
705 (
706 set -x
707 $docker_path rm $config
708 )
709 fi
710
711 run_start
712 exit 0
713 ;;
714
715
716 destroy)
717 (set -x; $docker_path stop -t 10 $config && $docker_path rm $config) || (echo "$config was not found" && exit 0)
718 exit 0
719 ;;
720 esac
721
722 usage