launcher

   1 #!/bin/bash
   2
   3 command=$1
   4 config=$2
   5 opt=$3
   6
   7 # Docker doesn't like uppercase characters, spaces or special characters, catch it now before we build everything out and then find out
   8 re='[A-Z/ !@#$%^&*()+~`=]'
   9 if [[ $config =~ $re ]];
  10   then
  11     echo
  12     echo "ERROR: Config name must not contain upper case characters, spaces or special characters. Correct config name and rerun $0."
  13     echo
  14     exit 1
  15 fi
  16
  17 cd "$(dirname "$0")"
  18
  19 docker_min_version='1.6.0'
  20 docker_rec_version='1.6.0'
  21
  22 config_file=containers/"$config".yml
  23 cidbootstrap=cids/"$config"_bootstrap.cid
  24 local_discourse=local_discourse
  25 image=discourse/discourse:1.0.16
  26 docker_path=`which docker.io || which docker`
  27
  28
  29 if [ "${SUPERVISED}" = "true" ]; then
  30   restart_policy="--restart=no"
  31   attach_on_start="-a"
  32   attach_on_run="-a stdout -a stderr"
  33 else
  34   attach_on_run="-d"
  35 fi
  36
  37 if [ -n "$DOCKER_HOST" ]; then
  38   docker_ip=`sed -e 's/^tcp:\/\/\(.*\):.*$/\1/' <<< "$DOCKER_HOST"`
  39 elif [ -x "$(which ip 2>/dev/null)" ]; then
  40   docker_ip=`ip addr show docker0 | \
  41                   grep 'inet ' | \
  42                   awk '{ split($2,a,"/"); print a[1] }';`
  43 else
  44   docker_ip=`ifconfig | \
  45                   grep -B1 "inet addr" | \
  46                   awk '{ if ( $1 == "inet" ) { print $2 } else if ( $2 == "Link" ) { printf "%s:" ,$1 } }' | \
  47                   grep docker0 | \
  48                   awk -F: '{ print $3 }';`
  49 fi
  50
  51
  52 usage () {
  53   echo "Usage: launcher COMMAND CONFIG [--skip-prereqs]"
  54   echo "Commands:"
  55   echo "    start:      Start/initialize a container"
  56   echo "    stop:       Stop a running container"
  57   echo "    restart:    Restart a container"
  58   echo "    destroy:    Stop and remove a container"
  59   echo "    enter:      Use nsenter to enter a container"
  60   echo "    ssh:        Start a bash shell in a running container"
  61   echo "    logs:       Docker logs for container"
  62   echo "    bootstrap:  Bootstrap a container for the config based on a template"
  63   echo "    rebuild:    Rebuild a container (destroy old, bootstrap, start new)"
  64   echo "    cleanup:    Remove all containers that have stopped for > 24 hours"
  65   echo
  66   echo "Options:"
  67   echo "    --skip-prereqs   Don't check prerequisites or resource requirements"
  68   echo "    --docker-args    Extra arguments to pass when running docker"
  69   exit 1
  70 }
  71
  72 compare_version() {
  73     declare -a ver_a
  74     declare -a ver_b
  75     IFS=. read -a ver_a <<< "$1"
  76     IFS=. read -a ver_b <<< "$2"
  77
  78     while [[ -n $ver_a ]]; do
  79         if (( ver_a > ver_b )); then
  80             return 0
  81         elif (( ver_b > ver_a )); then
  82             return 1
  83         else
  84             unset ver_a[0]
  85             ver_a=("${ver_a[@]}")
  86             unset ver_b[0]
  87             ver_b=("${ver_b[@]}")
  88         fi
  89     done
  90     return 1  # They are equal
  91 }
  92
  93 prereqs() {
  94
  95   # 1. docker daemon running?
  96   # we send stderr to /dev/null cause we don't care about warnings,
  97   # it usually complains about swap which does not matter
  98   test=`$docker_path info 2> /dev/null`
  99
 100   if [[ $? -ne 0 ]] ; then
 101     echo "Cannot connect to the docker daemon - verify it is running and you have access"
 102     exit 1
 103   fi
 104
 105   # 2. running aufs or btrfs
 106   test=`$docker_path info 2> /dev/null | grep 'Driver: '`
 107   if [[ "$test" =~ [aufs|btrfs|zfs|overlay] ]] ; then : ; else
 108     echo "Your Docker installation is not using a supported filesystem if we were to proceed you may have a broken install."
 109     echo "aufs is the recommended filesystem you should be using (zfs/btrfs and overlay may work as well)"
 110     echo "You can tell what filesystem you are using by running \"docker info\" and looking at the driver"
 111     echo ""
 112     echo "If you wish to continue anyway using your existing unsupported filesystem"
 113     echo "read the source code of launcher and figure out how to bypass this."
 114     exit 1
 115   fi
 116
 117   # 3. running recommended docker version
 118   test=($($docker_path --version))  # Get docker version string
 119   test=${test[2]//,/}  # Get version alone and strip comma if exists
 120
 121   [[ "$test" =~ "0.12.0" ]] && echo "You are running a broken version of Docker, please upgrade ASAP. See: https://meta.discourse.org/t/the-installation-stopped-in-the-middle/16311/ for more details." && exit 1
 122
 123   # At least minimum version
 124   if compare_version "${docker_min_version}" "${test}"; then
 125     echo "ERROR: Docker version ${test} not supported, please upgrade to at least ${docker_min_version}, or recommended ${docker_rec_version}"
 126     exit 1
 127   fi
 128
 129   # Recommend best version
 130   if compare_version "${docker_rec_version}" "${test}"; then
 131     echo "WARNING: Docker version ${test} deprecated, recommend upgrade to ${docker_rec_version} or newer."
 132   fi
 133
 134   # 4. discourse docker image is downloaded
 135   test=`$docker_path images | awk '{print $1 ":" $2 }' | grep "$image"`
 136
 137   if [ -z "$test" ]; then
 138     echo
 139     echo "WARNING: We are about to start downloading the Discourse base image"
 140     echo "This process may take anywhere between a few minutes to an hour, depending on your network speed"
 141     echo
 142     echo "Please be patient"
 143     echo
 144
 145   fi
 146
 147   # 5. able to attach stderr / out / tty
 148   test=`$docker_path run $user_args -i --rm -a stdout -a stderr $image echo working`
 149   if [[ "$test" =~ "working" ]] ; then : ; else
 150     echo "Your Docker installation is not working correctly"
 151     echo
 152     echo "See: https://meta.discourse.org/t/docker-error-on-bootstrap/13657/18?u=sam"
 153     exit 1
 154   fi
 155
 156 }
 157
 158 check_resources() {
 159   # Memory
 160   resources="ok"
 161   avail_mem="$(LANG=C free -m | grep '^Mem:' | awk '{print $2}')"
 162   if [ "$avail_mem" -lt 900 ]; then
 163     resources="insufficient"
 164     echo "WARNING: You do not appear to have sufficient memory to run Discourse."
 165     echo
 166     echo "Your system may not work properly, or future upgrades of Discourse may"
 167     echo "not complete successfully."
 168     echo
 169     echo "See https://github.com/discourse/discourse/blob/master/docs/INSTALL-cloud.md#create-new-cloud-server"
 170   elif [ "$avail_mem" -lt 1800 ]; then
 171     total_swap="$(LANG=C free -m | grep ^Swap: | awk '{print $2}')"
 172     if [ "$total_swap" -lt 1000 ]; then
 173       resources="insufficient"
 174       echo "WARNING: You must have at least 1GB of swap when running with less"
 175       echo "than 2GB of RAM."
 176       echo
 177       echo "Your system may not work properly, or future upgrades of Discourse may"
 178       echo "not complete successfully."
 179       echo
 180       echo "See https://github.com/discourse/discourse/blob/master/docs/INSTALL-cloud.md#set-up-swap-if-needed"
 181     fi
 182   fi
 183
 184   # Disk space
 185   free_disk="$(df /var | tail -n 1 | awk '{print $4}')"
 186   if [ "$free_disk" -lt 5000 ]; then
 187     resources="insufficient"
 188     echo "WARNING: You must have at least 5GB of *free* disk space to run Discourse."
 189     echo
 190     echo "Insufficient disk space may result in problems running your site, and may"
 191     echo "not even allow Discourse installation to complete successfully."
 192     echo
 193     echo "Please free up some space, or expand your disk, before continuing."
 194     exit 1
 195   fi
 196
 197   if [ -t 0 ] && [ "$resources" != "ok" ]; then
 198     echo
 199     read -p "Press ENTER to continue, or Ctrl-C to exit and give your system more resources"
 200   fi
 201 }
 202
 203 if [ "$opt" != "--skip-prereqs" ] ; then
 204   prereqs
 205 fi
 206
 207 if [ "$opt" == "--docker-args" ] ; then
 208   user_args=$4
 209 else
 210   user_args=""
 211 fi
 212
 213 get_ssh_pub_key() {
 214   local ${ssh_key_locations}
 215   ssh_key_locations=(
 216     ~/.ssh/id_ed25519.pub
 217     ~/.ssh/id_ecdsa.pub
 218     ~/.ssh/id_rsa.pub
 219     ~/.ssh/id_dsa.pub
 220     ~core/.ssh/authorized_keys
 221   )
 222
 223   local $keyfile
 224   for keyfile in "${ssh_key_locations[@]}"; do
 225     if [[ -e ${keyfile} ]] ; then
 226       ssh_pub_key="$(cat ${keyfile})"
 227       return 0
 228     fi
 229   done
 230
 231   return 0
 232 }
 233
 234
 235 install_docker() {
 236
 237   echo "Docker is not installed, you will need to install Docker in order to run Discourse"
 238   echo "Please visit https://docs.docker.com/installation/ for instructions on how to do this for your system"
 239   echo
 240   echo "If you are running Ubuntu Trusty or later, you can try the following:"
 241   echo
 242
 243   echo "sudo apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D"
 244   echo "sudo sh -c \"echo deb https://apt.dockerproject.org/repo ubuntu-$(lsb_release -sc) main > /etc/apt/sources.list.d/docker.list\""
 245   echo "sudo apt-get update"
 246   echo "sudo apt-get install docker-engine"
 247
 248   exit 1
 249 }
 250
 251 host_run() {
 252   read -r -d '' env_ruby << 'RUBY'
 253   require 'yaml'
 254
 255   input = STDIN.readlines.join
 256   yaml = YAML.load(input)
 257
 258   if host_run = yaml['host_run']
 259     params = yaml['params'] || {}
 260     host_run.each do |run|
 261       params.each do |k,v|
 262         run = run.gsub("$#{k}", v)
 263       end
 264       STDOUT.write "#{run}--SEP--"
 265     end
 266   end
 267 RUBY
 268
 269   host_run=`cat $config_file | $docker_path run $user_args --rm -i -a stdout -a stdin $image ruby -e "$env_ruby"`
 270
 271   while [ "$host_run" ] ; do
 272     iter=${host_run%%--SEP--*}
 273     echo
 274     echo "Host run: $iter"
 275     $iter || exit 1
 276     echo
 277     host_run="${host_run#*--SEP--}"
 278   done
 279 }
 280
 281
 282 set_volumes() {
 283   volumes=`cat $config_file | $docker_path run $user_args --rm -i -a stdout -a stdin $image ruby -e \
 284         "require 'yaml'; puts YAML.load(STDIN.readlines.join)['volumes'].map{|v| '-v ' << v['volume']['host'] << ':' << v['volume']['guest'] << ' '}.join"`
 285 }
 286
 287 set_links() {
 288     links=`cat $config_file | $docker_path run $user_args --rm -i -a stdout -a stdin $image ruby -e \
 289         "require 'yaml'; puts YAML.load(STDIN.readlines.join)['links'].map{|l| '--link ' << l['link']['name'] << ':' << l['link']['alias'] << ' '}.join"`
 290 }
 291
 292 set_template_info() {
 293
 294     templates=`cat $config_file | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \
 295       "require 'yaml'; puts YAML.load(STDIN.readlines.join)['templates']"`
 296
 297
 298     arrTemplates=(${templates// / })
 299     config_data=$(cat $config_file)
 300
 301     input="hack: true"
 302
 303
 304     for template in "${arrTemplates[@]}"
 305     do
 306       [ ! -z $template ] && {
 307         input="$input _FILE_SEPERATOR_ $(cat $template)"
 308       }
 309     done
 310
 311     # we always want our config file last so it takes priority
 312     input="$input _FILE_SEPERATOR_ $config_data"
 313
 314     read -r -d '' env_ruby << 'RUBY'
 315     require 'yaml'
 316
 317     input=STDIN.readlines.join
 318     # default to UTF-8 for the dbs sake
 319     env = {'LANG' => 'en_US.UTF-8'}
 320     input.split('_FILE_SEPERATOR_').each do |yml|
 321        yml.strip!
 322        begin
 323          env.merge!(YAML.load(yml)['env'] || {})
 324        rescue Psych::SyntaxError => e
 325         puts e
 326         puts "*ERROR."
 327        rescue => e
 328         puts yml
 329         p e
 330        end
 331     end
 332     puts env.map{|k,v| "-e\n#{k}=#{v}" }.join("\n")
 333 RUBY
 334
 335     raw=`exec echo "$input" | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e "$env_ruby"`
 336
 337     env=()
 338     ok=1
 339     while read i; do
 340       if [ "$i" == "*ERROR." ]; then
 341         ok=0
 342       elif [ -n "$i" ]; then
 343         env[${#env[@]}]=$i
 344       fi
 345     done <<< "$raw"
 346
 347     if [ "$ok" -ne 1 ]; then
 348       echo "${env[@]}"
 349       echo "YAML syntax error. Please check your /var/discourse/containers/*.yml config files."
 350       exit 1
 351     fi
 352 }
 353
 354 [ -z $docker_path ] && {
 355   install_docker
 356 }
 357
 358 [ "$command" == "cleanup" ] && {
 359   echo
 360   echo "The following command will"
 361   echo "- Delete all docker images for old containers"
 362   echo "- Delete all stopped and orphan containers"
 363   echo
 364   read -p "Are you sure (Y/n): " -n 1 -r && echo
 365   if [[ $REPLY =~ ^[Yy]$ || ! $REPLY ]]
 366     then
 367       space=$(df /var/lib/docker | awk '{ print $4 }' | grep -v Available)
 368       echo "Starting Cleanup (bytes free $space)"
 369
 370       STATE_DIR=./.gc-state scripts/docker-gc
 371
 372       space=$(df /var/lib/docker | awk '{ print $4 }' | grep -v Available)
 373       echo "Finished Cleanup (bytes free $space)"
 374
 375     else
 376       exit 1
 377   fi
 378   exit 0
 379 }
 380
 381 [ $# -lt 2 ] && {
 382   usage
 383 }
 384
 385 if [ ! -e $config_file ]
 386   then
 387     echo "Config file was not found, ensure $config_file exists"
 388     echo ""
 389     echo "Available configs ( `cd containers && ls -dm *.yml | tr -s '\n' ' ' | awk '{ gsub(/\.yml/, ""); print }'`)"
 390     exit 1
 391 fi
 392
 393
 394 docker_version=($($docker_path --version))
 395 docker_version=${test[2]//,/}
 396
 397 if compare_version "1.2.0" "$docker_version"; then
 398   echo "We recommend you upgrade docker, the version you are running has no restart policies, on reboot your container may not start up"
 399   restart_policy=""
 400 else
 401   restart_policy=${restart_policy:---restart=always}
 402 fi
 403
 404 set_existing_container(){
 405   existing=`$docker_path ps -a | awk '{ print $1, $(NF) }' | grep " $config$" | awk '{ print $1 }'`
 406 }
 407
 408 run_stop(){
 409
 410   set_existing_container
 411
 412   if [ ! -z $existing ]
 413      then
 414        (
 415         set -x
 416         $docker_path stop -t 10 $config
 417        )
 418      else
 419        echo "$config was not started !"
 420        exit 1
 421   fi
 422 }
 423
 424 set_run_image() {
 425   run_image=`cat $config_file | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \
 426     "require 'yaml'; puts YAML.load(STDIN.readlines.join)['run_image']"`
 427
 428   if [ -z "$run_image" ]; then
 429     run_image="$local_discourse/$config"
 430   fi
 431 }
 432
 433 set_boot_command() {
 434   boot_command=`cat $config_file | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \
 435     "require 'yaml'; puts YAML.load(STDIN.readlines.join)['boot_command']"`
 436
 437   if [ -z "$boot_command" ]; then
 438
 439     no_boot_command=`cat $config_file | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \
 440       "require 'yaml'; puts YAML.load(STDIN.readlines.join)['no_boot_command']"`
 441
 442     if [ -z "$no_boot_command" ]; then
 443       boot_command="/sbin/boot"
 444     fi
 445   fi
 446 }
 447
 448 run_start(){
 449
 450    existing=`$docker_path ps | awk '{ print $1, $(NF) }' | grep " $config$" | awk '{ print $1 }'`
 451    echo $existing
 452    if [ ! -z $existing ]
 453    then
 454      echo "Nothing to do, your container has already started!"
 455      exit 1
 456    fi
 457
 458    existing=`$docker_path ps -a | awk '{ print $1, $(NF) }' | grep " $config$" | awk '{ print $1 }'`
 459    if [ ! -z $existing ]
 460    then
 461      echo "starting up existing container"
 462      (
 463        set -x
 464        $docker_path start $config
 465      )
 466      exit 0
 467    fi
 468
 469    host_run
 470    ports=`cat $config_file | $docker_path run $user_args --rm -i -a stdout -a stdin $image ruby -e \
 471           "require 'yaml'; puts YAML.load(STDIN.readlines.join)['expose'].map{|p| '-p ' << p.to_s << ' '}.join"`
 472
 473    docker_args=`cat $config_file | $docker_path run $user_args --rm -i -a stdout -a stdin $image ruby -e \
 474           "require 'yaml'; puts YAML.load(STDIN.readlines.join)['docker_args']"`
 475
 476    set_template_info
 477    set_volumes
 478    set_links
 479    set_run_image
 480    set_boot_command
 481
 482    (
 483      hostname=`hostname`
 484      set -x
 485      $docker_path run $user_args $links $attach_on_run $restart_policy "${env[@]}" -h "$hostname-$config" \
 486         -e DOCKER_HOST_IP=$docker_ip --name $config -t $ports $volumes $docker_args $run_image $boot_command
 487
 488    )
 489    exit 0
 490
 491 }
 492
 493 run_bootstrap(){
 494
 495   if [ "$opt" != "--skip-prereqs" ] ; then
 496     check_resources
 497   fi
 498
 499   host_run
 500
 501   get_ssh_pub_key
 502
 503   # Is the image available?
 504   # If not, pull it here so the user is aware what's happening.
 505   $docker_path history $image >/dev/null 2>&1 || $docker_path pull $image
 506
 507   set_template_info
 508
 509   base_image=`cat $config_file | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \
 510     "require 'yaml'; puts YAML.load(STDIN.readlines.join)['base_image']"`
 511
 512   update_pups=`cat $config_file | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \
 513     "require 'yaml'; puts YAML.load(STDIN.readlines.join)['update_pups']"`
 514
 515   if [[ ! X"" = X"$base_image" ]]; then
 516     image=$base_image
 517   fi
 518
 519   set_volumes
 520   set_links
 521
 522   rm -f $cidbootstrap
 523
 524   run_command="cd /pups &&"
 525   if [[ ! "false" =  $update_pups ]]; then
 526     run_command="$run_command git pull &&"
 527   fi
 528   run_command="$run_command /pups/bin/pups --stdin"
 529
 530   echo $run_command
 531
 532   env=("${env[@]}" "-e" "SSH_PUB_KEY=$ssh_pub_key")
 533
 534   (exec echo "$input" | $docker_path run $user_args $links "${env[@]}" -e DOCKER_HOST_IP=$docker_ip --cidfile $cidbootstrap -i -a stdin -a stdout -a stderr $volumes $image \
 535      /bin/bash -c "$run_command") \
 536      || ($docker_path rm `cat $cidbootstrap` && rm $cidbootstrap)
 537
 538   [ ! -e $cidbootstrap ] && echo "** FAILED TO BOOTSTRAP ** please scroll up and look for earlier error messages, there may be more than one" && exit 1
 539
 540   sleep 5
 541
 542   $docker_path commit `cat $cidbootstrap` $local_discourse/$config || echo 'FAILED TO COMMIT'
 543   $docker_path rm `cat $cidbootstrap` && rm $cidbootstrap
 544 }
 545
 546
 547
 548 case "$command" in
 549   bootstrap)
 550       run_bootstrap
 551       echo "Successfully bootstrapped, to startup use ./launcher start $config"
 552       exit 0
 553       ;;
 554
 555   enter)
 556       exec $docker_path exec -it $config /bin/bash
 557       ;;
 558
 559   ssh)
 560        existing=`$docker_path ps | awk '{ print $1, $(NF) }' | grep " $config$" | awk '{ print $1 }'`
 561
 562        if [[ ! -z $existing ]]; then
 563          address="`$docker_path port $config 22`"
 564          split=(${address//:/ })
 565          exec ssh -o StrictHostKeyChecking=no root@${split[0]} -p ${split[1]}
 566        else
 567          echo "$config is not running!"
 568          exit 1
 569        fi
 570       ;;
 571
 572   stop)
 573       run_stop
 574       exit 0
 575       ;;
 576
 577   logs)
 578
 579       $docker_path logs $config
 580       exit 0
 581       ;;
 582
 583   restart)
 584       run_stop
 585       run_start
 586       exit 0
 587       ;;
 588
 589   start)
 590       run_start
 591       exit 0
 592       ;;
 593
 594   rebuild)
 595       if [ "$(git symbolic-ref --short HEAD)" == "master" ]; then
 596         echo "Ensuring discourse docker is up to date"
 597
 598         git remote update
 599
 600         LOCAL=$(git rev-parse @)
 601         REMOTE=$(git rev-parse @{u})
 602         BASE=$(git merge-base @ @{u})
 603
 604         if [ $LOCAL = $REMOTE ]; then
 605           echo "Discourse Docker is up-to-date"
 606
 607         elif [ $LOCAL = $BASE ]; then
 608           echo "Updating Discourse Docker"
 609           git pull || (echo 'failed to update' && exit 1)
 610           exec /bin/bash $0 $@
 611
 612         elif [ $REMOTE = $BASE ]; then
 613           echo "Your version of Discourse Docker is ahead of origin"
 614
 615         else
 616           echo "Discourse Docker has diverged source, this is only expected in Dev mode"
 617         fi
 618
 619       fi
 620
 621       set_existing_container
 622
 623       if [ ! -z $existing ]
 624         then
 625           echo "Stopping old container"
 626           (
 627             set -x
 628             $docker_path stop -t 10 $config
 629           )
 630       fi
 631
 632       run_bootstrap
 633
 634       if [ ! -z $existing ]
 635         then
 636           echo "Removing old container"
 637           (
 638             set -x
 639             $docker_path rm $config
 640           )
 641       fi
 642
 643       run_start
 644       exit 0
 645       ;;
 646
 647
 648   destroy)
 649       (set -x; $docker_path stop -t 10 $config && $docker_path rm $config) || (echo "$config was not found" && exit 0)
 650       exit 0
 651       ;;
 652 esac
 653
 654 usage