launcher

   1 #!/bin/bash
   2
   3 command=$1
   4 config=$2
   5 opt=$3
   6
   7 # Docker doesn't like uppercase characters, spaces or special characters, catch it now before we build everything out and then find out
   8 re='[A-Z/ !@#$%^&*()+~`=]'
   9 if [[ $config =~ $re ]];
  10   then
  11     echo
  12     echo "ERROR: Config name must not contain upper case characters, spaces or special characters. Correct config name and rerun $0."
  13     echo
  14     exit 1
  15 fi
  16
  17 cd "$(dirname "$0")"
  18
  19 docker_min_version='1.2.0'
  20 docker_rec_version='1.2.0'
  21
  22 config_file=containers/"$config".yml
  23 cidbootstrap=cids/"$config"_bootstrap.cid
  24 local_discourse=local_discourse
  25 image=discourse/discourse:1.0.15
  26 docker_path=`which docker.io || which docker`
  27
  28 if [ "${SUPERVISED}" = "true" ]; then
  29   restart_policy="--restart=no"
  30   attach_on_start="-a"
  31   attach_on_run="-a stdout -a stderr"
  32 else
  33   attach_on_run="-d"
  34 fi
  35
  36 if [ -n "$DOCKER_HOST" ]; then
  37   docker_ip=`sed -e 's/^tcp:\/\/\(.*\):.*$/\1/' <<< "$DOCKER_HOST"`
  38 elif [ -x "$(which ip 2>/dev/null)" ]; then
  39   docker_ip=`ip addr show docker0 | \
  40                   grep 'inet ' | \
  41                   awk '{ split($2,a,"/"); print a[1] }';`
  42 else
  43   docker_ip=`ifconfig | \
  44                   grep -B1 "inet addr" | \
  45                   awk '{ if ( $1 == "inet" ) { print $2 } else if ( $2 == "Link" ) { printf "%s:" ,$1 } }' | \
  46                   grep docker0 | \
  47                   awk -F: '{ print $3 }';`
  48 fi
  49
  50
  51 usage () {
  52   echo "Usage: launcher COMMAND CONFIG [--skip-prereqs]"
  53   echo "Commands:"
  54   echo "    start:      Start/initialize a container"
  55   echo "    stop:       Stop a running container"
  56   echo "    restart:    Restart a container"
  57   echo "    destroy:    Stop and remove a container"
  58   echo "    enter:      Use nsenter to enter a container"
  59   echo "    ssh:        Start a bash shell in a running container"
  60   echo "    logs:       Docker logs for container"
  61   echo "    bootstrap:  Bootstrap a container for the config based on a template"
  62   echo "    rebuild:    Rebuild a container (destroy old, bootstrap, start new)"
  63   echo "    cleanup:    Remove all containers that have stopped for > 24 hours"
  64   echo
  65   echo "Options:"
  66   echo "    --skip-prereqs   Don't check prerequisites"
  67   echo "    --docker-args    Extra arguments to pass when running docker"
  68   exit 1
  69 }
  70
  71 compare_version() {
  72     declare -a ver_a
  73     declare -a ver_b
  74     IFS=. read -a ver_a <<< "$1"
  75     IFS=. read -a ver_b <<< "$2"
  76
  77     while [[ -n $ver_a ]]; do
  78         if (( ver_a > ver_b )); then
  79             return 0
  80         elif (( ver_b > ver_a )); then
  81             return 1
  82         else
  83             unset ver_a[0]
  84             ver_a=("${ver_a[@]}")
  85             unset ver_b[0]
  86             ver_b=("${ver_b[@]}")
  87         fi
  88     done
  89     return 1  # They are equal
  90 }
  91
  92 prereqs() {
  93
  94   # 1. docker daemon running?
  95   # we send stderr to /dev/null cause we don't care about warnings,
  96   # it usually complains about swap which does not matter
  97   test=`$docker_path info 2> /dev/null`
  98
  99   if [[ $? -ne 0 ]] ; then
 100     echo "Cannot connect to the docker daemon - verify it is running and you have access"
 101     exit 1
 102   fi
 103
 104   # 2. running aufs or btrfs
 105   test=`$docker_path info 2> /dev/null | grep 'Driver: '`
 106   if [[ "$test" =~ [aufs|btrfs|zfs|overlay] ]] ; then : ; else
 107     echo "Your Docker installation is not using a supported filesystem if we were to proceed you may have a broken install."
 108     echo "aufs is the recommended filesystem you should be using (zfs/btrfs and overlay may work as well)"
 109     echo "You can tell what filesystem you are using by running \"docker info\" and looking at the driver"
 110     echo ""
 111     echo "If you wish to continue anyway using your existing unsupported filesystem"
 112     echo "read the source code of launcher and figure out how to bypass this."
 113     exit 1
 114   fi
 115
 116   # 3. running recommended docker version
 117   test=($($docker_path --version))  # Get docker version string
 118   test=${test[2]//,/}  # Get version alone and strip comma if exists
 119
 120   [[ "$test" =~ "0.12.0" ]] && echo "You are running a broken version of Docker, please upgrade ASAP. See: https://meta.discourse.org/t/the-installation-stopped-in-the-middle/16311/ for more details." && exit 1
 121
 122   # At least minimum version
 123   if compare_version "${docker_min_version}" "${test}"; then
 124     echo "ERROR: Docker version ${test} not supported, please upgrade to at least ${docker_min_version}, or recommended ${docker_rec_version}"
 125     exit 1
 126   fi
 127
 128   # Recommend best version
 129   if compare_version "${docker_rec_version}" "${test}"; then
 130     echo "WARNING: Docker version ${test} deprecated, recommend upgrade to ${docker_rec_version} or newer."
 131   fi
 132
 133   # 4. discourse docker image is downloaded
 134   test=`$docker_path images | awk '{print $1 ":" $2 }' | grep "$image"`
 135
 136   if [ -z "$test" ]; then
 137     echo
 138     echo "WARNING: We are about to start downloading the Discourse base image"
 139     echo "This process may take anywhere between a few minutes to an hour, depending on your network speed"
 140     echo
 141     echo "Please be patient"
 142     echo
 143
 144   fi
 145
 146   # 5. able to attach stderr / out / tty
 147   test=`$docker_path run $user_args -i --rm -a stdout -a stderr $image echo working`
 148   if [[ "$test" =~ "working" ]] ; then : ; else
 149     echo "Your Docker installation is not working correctly"
 150     echo
 151     echo "See: https://meta.discourse.org/t/docker-error-on-bootstrap/13657/18?u=sam"
 152     exit 1
 153   fi
 154
 155 }
 156
 157 check_resources() {
 158   # Memory
 159   resources="ok"
 160   avail_mem="$(LANG=C free -m | grep '^Mem:' | awk '{print $2}')"
 161   if [ "$avail_mem" -lt 900 ]; then
 162     resources="insufficient"
 163     echo "WARNING: You do not appear to have sufficient memory to run Discourse."
 164     echo
 165     echo "Your system may not work properly, or future upgrades of Discourse may"
 166     echo "not complete successfully."
 167     echo
 168     echo "See https://github.com/discourse/discourse/blob/master/docs/INSTALL-cloud.md#create-new-cloud-server"
 169   elif [ "$avail_mem" -lt 1800 ]; then
 170     total_swap="$(LANG=C free -m | grep ^Swap: | awk '{print $2}')"
 171     if [ "$total_swap" -lt 1000 ]; then
 172       resources="insufficient"
 173       echo "WARNING: You must have at least 1GB of swap when running with less"
 174       echo "than 2GB of RAM."
 175       echo
 176       echo "Your system may not work properly, or future upgrades of Discourse may"
 177       echo "not complete successfully."
 178       echo
 179       echo "See https://github.com/discourse/discourse/blob/master/docs/INSTALL-cloud.md#set-up-swap-if-needed"
 180     fi
 181   fi
 182
 183   # Disk space
 184   free_disk="$(df /var | tail -n 1 | awk '{print $4}')"
 185   if [ "$free_disk" -lt 5000 ]; then
 186     resources="insufficient"
 187     echo "WARNING: You must have at least 5GB of *free* disk space to run Discourse."
 188     echo
 189     echo "Insufficient disk space may result in problems running your site, and may"
 190     echo "not even allow Discourse installation to complete successfully."
 191     echo
 192     echo "Please free up some space, or expand your disk, before continuing."
 193     exit 1
 194   fi
 195
 196   if [ -t 0 ] && [ "$resources" != "ok" ]; then
 197     echo
 198     read -p "Press ENTER to continue, or Ctrl-C to exit and give your system more resources"
 199   fi
 200 }
 201
 202 if [ "$opt" != "--skip-prereqs" ] ; then
 203   prereqs
 204 fi
 205
 206 if [ "$opt" == "--docker-args" ] ; then
 207   user_args=$4
 208 else
 209   user_args=""
 210 fi
 211
 212 get_ssh_pub_key() {
 213   local ${ssh_key_locations}
 214   ssh_key_locations=(
 215     ~/.ssh/id_ed25519.pub
 216     ~/.ssh/id_ecdsa.pub
 217     ~/.ssh/id_rsa.pub
 218     ~/.ssh/id_dsa.pub
 219     ~core/.ssh/authorized_keys
 220   )
 221
 222   local $keyfile
 223   for keyfile in "${ssh_key_locations[@]}"; do
 224     if [[ -e ${keyfile} ]] ; then
 225       ssh_pub_key="$(cat ${keyfile})"
 226       return 0
 227     fi
 228   done
 229
 230   return 0
 231 }
 232
 233
 234 install_docker() {
 235
 236   echo "Docker is not installed, you will need to install Docker in order to run Discourse"
 237   echo "Please visit https://docs.docker.com/installation/ for instructions on how to do this for your system"
 238   echo
 239   echo "If you are running Ubuntu Trusty or later, you can try the following:"
 240   echo
 241
 242   echo "sudo apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D"
 243   echo "sudo sh -c \"echo deb https://apt.dockerproject.org/repo ubuntu-$(lsb_release -sc) main > /etc/apt/sources.list.d/docker.list\""
 244   echo "sudo apt-get update"
 245   echo "sudo apt-get install docker-engine"
 246
 247   exit 1
 248 }
 249
 250 host_run() {
 251   read -r -d '' env_ruby << 'RUBY'
 252   require 'yaml'
 253
 254   input = STDIN.readlines.join
 255   yaml = YAML.load(input)
 256
 257   if host_run = yaml['host_run']
 258     params = yaml['params'] || {}
 259     host_run.each do |run|
 260       params.each do |k,v|
 261         run = run.gsub("$#{k}", v)
 262       end
 263       STDOUT.write "#{run}--SEP--"
 264     end
 265   end
 266 RUBY
 267
 268   host_run=`cat $config_file | $docker_path run $user_args --rm -i -a stdout -a stdin $image ruby -e "$env_ruby"`
 269
 270   while [ "$host_run" ] ; do
 271     iter=${host_run%%--SEP--*}
 272     echo
 273     echo "Host run: $iter"
 274     $iter || exit 1
 275     echo
 276     host_run="${host_run#*--SEP--}"
 277   done
 278 }
 279
 280
 281 set_volumes() {
 282   volumes=`cat $config_file | $docker_path run $user_args --rm -i -a stdout -a stdin $image ruby -e \
 283         "require 'yaml'; puts YAML.load(STDIN.readlines.join)['volumes'].map{|v| '-v ' << v['volume']['host'] << ':' << v['volume']['guest'] << ' '}.join"`
 284 }
 285
 286 set_links() {
 287     links=`cat $config_file | $docker_path run $user_args --rm -i -a stdout -a stdin $image ruby -e \
 288         "require 'yaml'; puts YAML.load(STDIN.readlines.join)['links'].map{|l| '--link ' << l['link']['name'] << ':' << l['link']['alias'] << ' '}.join"`
 289 }
 290
 291 set_template_info() {
 292
 293     templates=`cat $config_file | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \
 294       "require 'yaml'; puts YAML.load(STDIN.readlines.join)['templates']"`
 295
 296
 297     arrTemplates=(${templates// / })
 298     config_data=$(cat $config_file)
 299
 300     input="hack: true"
 301
 302
 303     for template in "${arrTemplates[@]}"
 304     do
 305       [ ! -z $template ] && {
 306         input="$input _FILE_SEPERATOR_ $(cat $template)"
 307       }
 308     done
 309
 310     # we always want our config file last so it takes priority
 311     input="$input _FILE_SEPERATOR_ $config_data"
 312
 313     read -r -d '' env_ruby << 'RUBY'
 314     require 'yaml'
 315
 316     input=STDIN.readlines.join
 317     # default to UTF-8 for the dbs sake
 318     env = {'LANG' => 'en_US.UTF-8'}
 319     input.split('_FILE_SEPERATOR_').each do |yml|
 320        yml.strip!
 321        begin
 322          env.merge!(YAML.load(yml)['env'] || {})
 323        rescue Psych::SyntaxError => e
 324         puts e
 325         puts "*ERROR."
 326        rescue => e
 327         puts yml
 328         p e
 329        end
 330     end
 331     puts env.map{|k,v| "-e\n#{k}=#{v}" }.join("\n")
 332 RUBY
 333
 334     raw=`exec echo "$input" | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e "$env_ruby"`
 335
 336     env=()
 337     ok=1
 338     while read i; do
 339       if [ "$i" == "*ERROR." ]; then
 340         ok=0
 341       elif [ -n "$i" ]; then
 342         env[${#env[@]}]=$i
 343       fi
 344     done <<< "$raw"
 345
 346     if [ "$ok" -ne 1 ]; then
 347       echo "${env[@]}"
 348       echo "YAML syntax error. Please check your /var/discourse/containers/*.yml config files."
 349       exit 1
 350     fi
 351 }
 352
 353 [ -z $docker_path ] && {
 354   install_docker
 355 }
 356
 357 [ "$command" == "cleanup" ] && {
 358   echo
 359   echo "The following command will"
 360   echo "- Delete all docker images for old containers"
 361   echo "- Delete all stopped and orphan containers"
 362   echo
 363   read -p "Are you sure (Y/n): " -n 1 -r && echo
 364   if [[ $REPLY =~ ^[Yy]$ || ! $REPLY ]]
 365     then
 366       space=$(df /var/lib/docker | awk '{ print $4 }' | grep -v Available)
 367       echo "Starting Cleanup (bytes free $space)"
 368
 369       STATE_DIR=./.gc-state scripts/docker-gc
 370
 371       space=$(df /var/lib/docker | awk '{ print $4 }' | grep -v Available)
 372       echo "Finished Cleanup (bytes free $space)"
 373
 374     else
 375       exit 1
 376   fi
 377   exit 0
 378 }
 379
 380 [ $# -lt 2 ] && {
 381   usage
 382 }
 383
 384 if [ ! -e $config_file ]
 385   then
 386     echo "Config file was not found, ensure $config_file exists"
 387     echo ""
 388     echo "Available configs ( `cd containers && ls -dm *.yml | tr -s '\n' ' ' | awk '{ gsub(/\.yml/, ""); print }'`)"
 389     exit 1
 390 fi
 391
 392
 393 docker_version=($($docker_path --version))
 394 docker_version=${test[2]//,/}
 395
 396 if compare_version "1.2.0" "$docker_version"; then
 397   echo "We recommend you upgrade docker, the version you are running has no restart policies, on reboot your container may not start up"
 398   restart_policy=""
 399 else
 400   restart_policy=${restart_policy:---restart=always}
 401 fi
 402
 403 set_existing_container(){
 404   existing=`$docker_path ps -a | awk '{ print $1, $(NF) }' | grep " $config$" | awk '{ print $1 }'`
 405 }
 406
 407 run_stop(){
 408
 409   set_existing_container
 410
 411   if [ ! -z $existing ]
 412      then
 413        (
 414         set -x
 415         $docker_path stop -t 10 $config
 416        )
 417      else
 418        echo "$config was not started !"
 419        exit 1
 420   fi
 421 }
 422
 423 run_start(){
 424
 425    existing=`$docker_path ps | awk '{ print $1, $(NF) }' | grep " $config$" | awk '{ print $1 }'`
 426    echo $existing
 427    if [ ! -z $existing ]
 428    then
 429      echo "Nothing to do, your container has already started!"
 430      exit 1
 431    fi
 432
 433    existing=`$docker_path ps -a | awk '{ print $1, $(NF) }' | grep " $config$" | awk '{ print $1 }'`
 434    if [ ! -z $existing ]
 435    then
 436      echo "starting up existing container"
 437      (
 438        set -x
 439        $docker_path start $config
 440      )
 441      exit 0
 442    fi
 443
 444    host_run
 445    ports=`cat $config_file | $docker_path run $user_args --rm -i -a stdout -a stdin $image ruby -e \
 446           "require 'yaml'; puts YAML.load(STDIN.readlines.join)['expose'].map{|p| '-p ' << p.to_s << ' '}.join"`
 447
 448    docker_args=`cat $config_file | $docker_path run $user_args --rm -i -a stdout -a stdin $image ruby -e \
 449           "require 'yaml'; puts YAML.load(STDIN.readlines.join)['docker_args']"`
 450
 451    set_template_info
 452    set_volumes
 453    set_links
 454
 455    (
 456      hostname=`hostname`
 457      set -x
 458      $docker_path run $user_args $links $attach_on_run $restart_policy "${env[@]}" -h "$hostname-$config" \
 459         -e DOCKER_HOST_IP=$docker_ip --name $config -t $ports $volumes $docker_args $local_discourse/$config \
 460         /sbin/boot
 461
 462    )
 463    exit 0
 464
 465 }
 466
 467 run_bootstrap(){
 468
 469   check_resources
 470
 471   host_run
 472
 473   get_ssh_pub_key
 474
 475   # Is the image available?
 476   # If not, pull it here so the user is aware what's happening.
 477   $docker_path history $image >/dev/null 2>&1 || $docker_path pull $image
 478
 479   set_template_info
 480
 481   base_image=`cat $config_file | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \
 482     "require 'yaml'; puts YAML.load(STDIN.readlines.join)['base_image']"`
 483
 484   update_pups=`cat $config_file | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \
 485     "require 'yaml'; puts YAML.load(STDIN.readlines.join)['update_pups']"`
 486
 487   if [[ ! X"" = X"$base_image" ]]; then
 488     image=$base_image
 489   fi
 490
 491   set_volumes
 492   set_links
 493
 494   rm -f $cidbootstrap
 495
 496   run_command="cd /pups &&"
 497   if [[ ! "false" =  $update_pups ]]; then
 498     run_command="$run_command git pull &&"
 499   fi
 500   run_command="$run_command /pups/bin/pups --stdin"
 501
 502   echo $run_command
 503
 504   env=("${env[@]}" "-e" "SSH_PUB_KEY=$ssh_pub_key")
 505
 506   (exec echo "$input" | $docker_path run $user_args $links "${env[@]}" -e DOCKER_HOST_IP=$docker_ip --cidfile $cidbootstrap -i -a stdin -a stdout -a stderr $volumes $image \
 507      /bin/bash -c "$run_command") \
 508      || ($docker_path rm `cat $cidbootstrap` && rm $cidbootstrap)
 509
 510   [ ! -e $cidbootstrap ] && echo "** FAILED TO BOOTSTRAP ** please scroll up and look for earlier error messages, there may be more than one" && exit 1
 511
 512   sleep 5
 513
 514   $docker_path commit `cat $cidbootstrap` $local_discourse/$config || echo 'FAILED TO COMMIT'
 515   $docker_path rm `cat $cidbootstrap` && rm $cidbootstrap
 516 }
 517
 518 case "$command" in
 519   bootstrap)
 520       run_bootstrap
 521       echo "Successfully bootstrapped, to startup use ./launcher start $config"
 522       exit 0
 523       ;;
 524
 525   enter)
 526       exec $docker_path exec -it $config /bin/bash
 527       ;;
 528
 529   ssh)
 530        existing=`$docker_path ps | awk '{ print $1, $(NF) }' | grep " $config$" | awk '{ print $1 }'`
 531
 532        if [[ ! -z $existing ]]; then
 533          address="`$docker_path port $config 22`"
 534          split=(${address//:/ })
 535          exec ssh -o StrictHostKeyChecking=no root@${split[0]} -p ${split[1]}
 536        else
 537          echo "$config is not running!"
 538          exit 1
 539        fi
 540       ;;
 541
 542   stop)
 543       run_stop
 544       exit 0
 545       ;;
 546
 547   logs)
 548
 549       $docker_path logs $config
 550       exit 0
 551       ;;
 552
 553   restart)
 554       run_stop
 555       run_start
 556       exit 0
 557       ;;
 558
 559   start)
 560       run_start
 561       exit 0
 562       ;;
 563
 564   rebuild)
 565       if [ "$(git symbolic-ref --short HEAD)" == "master" ]; then
 566         echo "Ensuring discourse docker is up to date"
 567
 568         git remote update
 569
 570         LOCAL=$(git rev-parse @)
 571         REMOTE=$(git rev-parse @{u})
 572         BASE=$(git merge-base @ @{u})
 573
 574         if [ $LOCAL = $REMOTE ]; then
 575           echo "Discourse Docker is up-to-date"
 576
 577         elif [ $LOCAL = $BASE ]; then
 578           echo "Updating Discourse Docker"
 579           git pull || (echo 'failed to update' && exit 1)
 580           exec /bin/bash $0 $@
 581
 582         elif [ $REMOTE = $BASE ]; then
 583           echo "Your version of Discourse Docker is ahead of origin"
 584
 585         else
 586           echo "Discourse Docker has diverged source, this is only expected in Dev mode"
 587         fi
 588
 589       fi
 590
 591       set_existing_container
 592
 593       if [ ! -z $existing ]
 594         then
 595           echo "Stopping old container"
 596           (
 597             set -x
 598             $docker_path stop -t 10 $config
 599           )
 600       fi
 601
 602       run_bootstrap
 603
 604       if [ ! -z $existing ]
 605         then
 606           echo "Removing old container"
 607           (
 608             set -x
 609             $docker_path rm $config
 610           )
 611       fi
 612
 613       run_start
 614       exit 0
 615       ;;
 616
 617
 618   destroy)
 619       (set -x; $docker_path stop -t 10 $config && $docker_path rm $config) || (echo "$config was not found" && exit 0)
 620       exit 0
 621       ;;
 622 esac
 623
 624 usage