launcher

   1 #!/bin/bash
   2
   3 command=$1
   4 config=$2
   5 opt=$3
   6
   7 # Docker doesn't like uppercase characters, spaces or special characters, catch it now before we build everything out and then find out
   8 re='[A-Z/ !@#$%^&*()+~`=]'
   9 if [[ $config =~ $re ]];
  10   then
  11     echo
  12     echo "ERROR: Config name must not contain upper case characters, spaces or special characters. Correct config name and rerun $0."
  13     echo
  14     exit 1
  15 fi
  16
  17 cd "$(dirname "$0")"
  18
  19 docker_min_version='1.2.0'
  20 docker_rec_version='1.2.0'
  21
  22 config_file=containers/"$config".yml
  23 cidbootstrap=cids/"$config"_bootstrap.cid
  24 local_discourse=local_discourse
  25 image=samsaffron/discourse:1.0.12
  26 docker_path=`which docker.io || which docker`
  27
  28 if [ "${SUPERVISED}" = "true" ]; then
  29   restart_policy="--restart=no"
  30   attach_on_start="-a"
  31   attach_on_run="-a stdout -a stderr"
  32 else
  33   attach_on_run="-d"
  34 fi
  35
  36 if [ -n "$DOCKER_HOST" ]; then
  37   docker_ip=`sed -e 's/^tcp:\/\/\(.*\):.*$/\1/' <<< "$DOCKER_HOST"`
  38 elif [ -x "$(which ip 2>/dev/null)" ]; then
  39   docker_ip=`ip addr show docker0 | \
  40                   grep 'inet ' | \
  41                   awk '{ split($2,a,"/"); print a[1] }';`
  42 else
  43   docker_ip=`ifconfig | \
  44                   grep -B1 "inet addr" | \
  45                   awk '{ if ( $1 == "inet" ) { print $2 } else if ( $2 == "Link" ) { printf "%s:" ,$1 } }' | \
  46                   grep docker0 | \
  47                   awk -F: '{ print $3 }';`
  48 fi
  49
  50
  51 usage () {
  52   echo "Usage: launcher COMMAND CONFIG [--skip-prereqs]"
  53   echo "Commands:"
  54   echo "    start:      Start/initialize a container"
  55   echo "    stop:       Stop a running container"
  56   echo "    restart:    Restart a container"
  57   echo "    destroy:    Stop and remove a container"
  58   echo "    enter:      Use nsenter to enter a container"
  59   echo "    ssh:        Start a bash shell in a running container"
  60   echo "    logs:       Docker logs for container"
  61   echo "    bootstrap:  Bootstrap a container for the config based on a template"
  62   echo "    rebuild:    Rebuild a container (destroy old, bootstrap, start new)"
  63   echo "    cleanup:    Remove all containers that have stopped for > 24 hours"
  64   echo
  65   echo "Options:"
  66   echo "    --skip-prereqs   Don't check prerequisites"
  67   echo "    --docker-args    Extra arguments to pass when running docker"
  68   exit 1
  69 }
  70
  71 compare_version() {
  72     declare -a ver_a
  73     declare -a ver_b
  74     IFS=. read -a ver_a <<< "$1"
  75     IFS=. read -a ver_b <<< "$2"
  76
  77     while [[ -n $ver_a ]]; do
  78         if (( ver_a > ver_b )); then
  79             return 0
  80         elif (( ver_b > ver_a )); then
  81             return 1
  82         else
  83             unset ver_a[0]
  84             ver_a=("${ver_a[@]}")
  85             unset ver_b[0]
  86             ver_b=("${ver_b[@]}")
  87         fi
  88     done
  89     return 1  # They are equal
  90 }
  91
  92 prereqs() {
  93
  94   # 1. docker daemon running?
  95   # we send stderr to /dev/null cause we don't care about warnings,
  96   # it usually complains about swap which does not matter
  97   test=`$docker_path info 2> /dev/null`
  98
  99   if [[ $? -ne 0 ]] ; then
 100     echo "Cannot connect to the docker daemon - verify it is running and you have access"
 101     exit 1
 102   fi
 103
 104   # 2. running aufs or btrfs
 105   test=`$docker_path info 2> /dev/null | grep 'Driver: '`
 106   if [[ "$test" =~ [aufs|btrfs|zfs|overlay] ]] ; then : ; else
 107     echo "Your Docker installation is not using a supported filesystem if we were to proceed you may have a broken install."
 108     echo "aufs is the recommended filesystem you should be using (zfs/btrfs and overlay may work as well)"
 109     echo "You can tell what filesystem you are using by running \"docker info\" and looking at the driver"
 110     echo ""
 111     echo "If you wish to continue anyway using your existing unsupported filesystem"
 112     echo "read the source code of launcher and figure out how to bypass this."
 113     exit 1
 114   fi
 115
 116   # 3. running recommended docker version
 117   test=($($docker_path --version))  # Get docker version string
 118   test=${test[2]//,/}  # Get version alone and strip comma if exists
 119
 120   [[ "$test" =~ "0.12.0" ]] && echo "You are running a broken version of Docker, please upgrade ASAP. See: https://meta.discourse.org/t/the-installation-stopped-in-the-middle/16311/ for more details." && exit 1
 121
 122   # At least minimum version
 123   if compare_version "${docker_min_version}" "${test}"; then
 124     echo "ERROR: Docker version ${test} not supported, please upgrade to at least ${docker_min_version}, or recommended ${docker_rec_version}"
 125     exit 1
 126   fi
 127
 128   # Recommend best version
 129   if compare_version "${docker_rec_version}" "${test}"; then
 130     echo "WARNING: Docker version ${test} deprecated, recommend upgrade to ${docker_rec_version} or newer."
 131   fi
 132
 133   # 4. discourse docker image is downloaded
 134   test=`$docker_path images | awk '{print $1 ":" $2 }' | grep "$image"`
 135
 136   if [ -z "$test" ]; then
 137     echo
 138     echo "WARNING: We are about to start downloading the Discourse base image"
 139     echo "This process may take anywhere between a few minutes to an hour, depending on your network speed"
 140     echo
 141     echo "Please be patient"
 142     echo
 143
 144   fi
 145
 146   # 5. able to attach stderr / out / tty
 147   test=`$docker_path run $user_args -i --rm -a stdout -a stderr $image echo working`
 148   if [[ "$test" =~ "working" ]] ; then : ; else
 149     echo "Your Docker installation is not working correctly"
 150     echo
 151     echo "See: https://meta.discourse.org/t/docker-error-on-bootstrap/13657/18?u=sam"
 152     exit 1
 153   fi
 154
 155   # 6. sufficient available resources
 156   # 6a. Memory
 157   #
 158   resources="ok"
 159   avail_mem="$(LANG=C free -m | grep '^Mem:' | awk '{print $2}')"
 160   if [ "$avail_mem" -lt 900 ]; then
 161     resources="insufficient"
 162     echo "WARNING: You do not appear to have sufficient memory to run Discourse."
 163     echo
 164     echo "Your system may not work properly, or future upgrades of Discourse may"
 165     echo "not complete successfully."
 166     echo
 167     echo "See https://github.com/discourse/discourse/blob/master/docs/INSTALL-cloud.md#create-new-cloud-server"
 168   elif [ "$avail_mem" -lt 1800 ]; then
 169     resources="insufficient"
 170     total_swap="$(LANG=C free -m | grep ^Swap: | awk '{print $2}')"
 171     if [ "$total_swap" -lt 1000 ]; then
 172       echo "WARNING: You must have at least 1GB of swap when running with less"
 173       echo "than 2GB of RAM."
 174       echo
 175       echo "Your system may not work properly, or future upgrades of Discourse may"
 176       echo "not complete successfully."
 177       echo
 178       echo "See https://github.com/discourse/discourse/blob/master/docs/INSTALL-cloud.md#set-up-swap-if-needed"
 179     fi
 180   fi
 181
 182   # 6b. Disk space
 183   free_disk="$(df /var | tail -n 1 | awk '{print $4}')"
 184   if [ "$free_disk" -lt 5000 ]; then
 185     resources="insufficient"
 186     echo "WARNING: You must have at least 5GB of *free* disk space to run Discourse."
 187     echo
 188     echo "Insufficient disk space may result in problems running your site, and may"
 189     echo "not even allow Discourse installation to complete successfully."
 190     echo
 191     echo "Please free up some space, or expand your disk, before continuing."
 192     exit 1
 193   fi
 194
 195   if [ -t 0 ] && [ "$resources" != "ok" ]; then
 196     echo
 197     read -p "Press ENTER to continue, or Ctrl-C to exit and give your system more resources"
 198   fi
 199 }
 200
 201 if [ "$opt" != "--skip-prereqs" ] ; then
 202   prereqs
 203 fi
 204
 205 if [ "$opt" == "--docker-args" ] ; then
 206   user_args=$4
 207 else
 208   user_args=""
 209 fi
 210
 211 get_ssh_pub_key() {
 212   local ${ssh_key_locations}
 213   ssh_key_locations=(
 214     ~/.ssh/id_ed25519.pub
 215     ~/.ssh/id_ecdsa.pub
 216     ~/.ssh/id_rsa.pub
 217     ~/.ssh/id_dsa.pub
 218     ~core/.ssh/authorized_keys
 219   )
 220
 221   local $keyfile
 222   for keyfile in "${ssh_key_locations[@]}"; do
 223     if [[ -e ${keyfile} ]] ; then
 224       ssh_pub_key="$(cat ${keyfile})"
 225       return 0
 226     fi
 227   done
 228
 229   return 0
 230 }
 231
 232
 233 install_docker() {
 234
 235   echo "Docker is not installed, you will need to install Docker in order to run Discourse"
 236   echo "Please visit https://docs.docker.com/installation/ for instructions on how to do this for your system"
 237   echo
 238   echo "If you are running Ubuntu Trusty or later, you can try the following:"
 239   echo
 240
 241   echo "sudo apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D"
 242   echo "sudo sh -c \"echo deb https://apt.dockerproject.org/repo ubuntu-$(lsb_release -sc) main > /etc/apt/sources.list.d/docker.list\""
 243   echo "sudo apt-get update"
 244   echo "sudo apt-get install docker-engine"
 245
 246   exit 1
 247 }
 248
 249 host_run() {
 250   read -r -d '' env_ruby << 'RUBY'
 251   require 'yaml'
 252
 253   input = STDIN.readlines.join
 254   yaml = YAML.load(input)
 255
 256   if host_run = yaml['host_run']
 257     params = yaml['params'] || {}
 258     host_run.each do |run|
 259       params.each do |k,v|
 260         run = run.gsub("$#{k}", v)
 261       end
 262       STDOUT.write "#{run}--SEP--"
 263     end
 264   end
 265 RUBY
 266
 267   host_run=`cat $config_file | $docker_path run $user_args --rm -i -a stdout -a stdin $image ruby -e "$env_ruby"`
 268
 269   while [ "$host_run" ] ; do
 270     iter=${host_run%%--SEP--*}
 271     echo
 272     echo "Host run: $iter"
 273     $iter || exit 1
 274     echo
 275     host_run="${host_run#*--SEP--}"
 276   done
 277 }
 278
 279
 280 set_volumes() {
 281   volumes=`cat $config_file | $docker_path run $user_args --rm -i -a stdout -a stdin $image ruby -e \
 282         "require 'yaml'; puts YAML.load(STDIN.readlines.join)['volumes'].map{|v| '-v ' << v['volume']['host'] << ':' << v['volume']['guest'] << ' '}.join"`
 283 }
 284
 285 set_links() {
 286     links=`cat $config_file | $docker_path run $user_args --rm -i -a stdout -a stdin $image ruby -e \
 287         "require 'yaml'; puts YAML.load(STDIN.readlines.join)['links'].map{|l| '--link ' << l['link']['name'] << ':' << l['link']['alias'] << ' '}.join"`
 288 }
 289
 290 set_template_info() {
 291
 292     templates=`cat $config_file | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \
 293       "require 'yaml'; puts YAML.load(STDIN.readlines.join)['templates']"`
 294
 295
 296     arrTemplates=(${templates// / })
 297     config_data=$(cat $config_file)
 298
 299     input="hack: true"
 300
 301
 302     for template in "${arrTemplates[@]}"
 303     do
 304       [ ! -z $template ] && {
 305         input="$input _FILE_SEPERATOR_ $(cat $template)"
 306       }
 307     done
 308
 309     # we always want our config file last so it takes priority
 310     input="$input _FILE_SEPERATOR_ $config_data"
 311
 312     read -r -d '' env_ruby << 'RUBY'
 313     require 'yaml'
 314
 315     input=STDIN.readlines.join
 316     # default to UTF-8 for the dbs sake
 317     env = {'LANG' => 'en_US.UTF-8'}
 318     input.split('_FILE_SEPERATOR_').each do |yml|
 319        yml.strip!
 320        begin
 321          env.merge!(YAML.load(yml)['env'] || {})
 322        rescue Psych::SyntaxError => e
 323         puts e
 324         puts "*ERROR."
 325        rescue => e
 326         puts yml
 327         p e
 328        end
 329     end
 330     puts env.map{|k,v| "-e\n#{k}=#{v}" }.join("\n")
 331 RUBY
 332
 333     raw=`exec echo "$input" | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e "$env_ruby"`
 334
 335     env=()
 336     ok=1
 337     while read i; do
 338       if [ "$i" == "*ERROR." ]; then
 339         ok=0
 340       elif [ -n "$i" ]; then
 341         env[${#env[@]}]=$i
 342       fi
 343     done <<< "$raw"
 344
 345     if [ "$ok" -ne 1 ]; then
 346       echo "${env[@]}"
 347       echo "YAML syntax error. Please check your /var/discourse/containers/*.yml config files."
 348       exit 1
 349     fi
 350 }
 351
 352 [ -z $docker_path ] && {
 353   install_docker
 354 }
 355
 356 [ "$command" == "cleanup" ] && {
 357   echo
 358   echo "The following command will"
 359   echo "- Delete all docker images for old containers"
 360   echo "- Delete all stopped and orphan containers"
 361   echo
 362   read -p "Are you sure (Y/n): " -n 1 -r && echo
 363   if [[ $REPLY =~ ^[Yy]$ || ! $REPLY ]]
 364     then
 365       space=$(df /var/lib/docker | awk '{ print $4 }' | grep -v Available)
 366       echo "Starting Cleanup (bytes free $space)"
 367
 368       STATE_DIR=./.gc-state scripts/docker-gc
 369
 370       space=$(df /var/lib/docker | awk '{ print $4 }' | grep -v Available)
 371       echo "Finished Cleanup (bytes free $space)"
 372
 373     else
 374       exit 1
 375   fi
 376   exit 0
 377 }
 378
 379 [ $# -lt 2 ] && {
 380   usage
 381 }
 382
 383 if [ ! -e $config_file ]
 384   then
 385     echo "Config file was not found, ensure $config_file exists"
 386     echo ""
 387     echo "Available configs ( `cd containers && ls -dm *.yml | tr -s '\n' ' ' | awk '{ gsub(/\.yml/, ""); print }'`)"
 388     exit 1
 389 fi
 390
 391
 392 docker_version=($($docker_path --version))
 393 docker_version=${test[2]//,/}
 394
 395 if compare_version "1.2.0" "$docker_version"; then
 396   echo "We recommend you upgrade docker, the version you are running has no restart policies, on reboot your container may not start up"
 397   restart_policy=""
 398 else
 399   restart_policy=${restart_policy:---restart=always}
 400 fi
 401
 402 set_existing_container(){
 403   existing=`$docker_path ps -a | awk '{ print $1, $(NF) }' | grep " $config$" | awk '{ print $1 }'`
 404 }
 405
 406 run_stop(){
 407
 408   set_existing_container
 409
 410   if [ ! -z $existing ]
 411      then
 412        (
 413         set -x
 414         $docker_path stop -t 10 $config
 415        )
 416      else
 417        echo "$config was not started !"
 418        exit 1
 419   fi
 420 }
 421
 422 run_start(){
 423
 424    existing=`$docker_path ps | awk '{ print $1, $(NF) }' | grep " $config$" | awk '{ print $1 }'`
 425    echo $existing
 426    if [ ! -z $existing ]
 427    then
 428      echo "Nothing to do, your container has already started!"
 429      exit 1
 430    fi
 431
 432    existing=`$docker_path ps -a | awk '{ print $1, $(NF) }' | grep " $config$" | awk '{ print $1 }'`
 433    if [ ! -z $existing ]
 434    then
 435      echo "starting up existing container"
 436      (
 437        set -x
 438        $docker_path start $config
 439      )
 440      exit 0
 441    fi
 442
 443    host_run
 444    ports=`cat $config_file | $docker_path run $user_args --rm -i -a stdout -a stdin $image ruby -e \
 445           "require 'yaml'; puts YAML.load(STDIN.readlines.join)['expose'].map{|p| '-p ' << p.to_s << ' '}.join"`
 446
 447    docker_args=`cat $config_file | $docker_path run $user_args --rm -i -a stdout -a stdin $image ruby -e \
 448           "require 'yaml'; puts YAML.load(STDIN.readlines.join)['docker_args']"`
 449
 450    set_template_info
 451    set_volumes
 452    set_links
 453
 454    (
 455      hostname=`hostname`
 456      set -x
 457      $docker_path run $user_args $links $attach_on_run $restart_policy "${env[@]}" -h "$hostname-$config" \
 458         -e DOCKER_HOST_IP=$docker_ip --name $config -t $ports $volumes $docker_args $local_discourse/$config \
 459         /sbin/boot
 460
 461    )
 462    exit 0
 463
 464 }
 465
 466 run_bootstrap(){
 467
 468   host_run
 469
 470   get_ssh_pub_key
 471
 472   # Is the image available?
 473   # If not, pull it here so the user is aware what's happening.
 474   $docker_path history $image >/dev/null 2>&1 || $docker_path pull $image
 475
 476   set_template_info
 477
 478   base_image=`cat $config_file | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \
 479     "require 'yaml'; puts YAML.load(STDIN.readlines.join)['base_image']"`
 480
 481   update_pups=`cat $config_file | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \
 482     "require 'yaml'; puts YAML.load(STDIN.readlines.join)['update_pups']"`
 483
 484   if [[ ! X"" = X"$base_image" ]]; then
 485     image=$base_image
 486   fi
 487
 488   set_volumes
 489   set_links
 490
 491   rm -f $cidbootstrap
 492
 493   run_command="cd /pups &&"
 494   if [[ ! "false" =  $update_pups ]]; then
 495     run_command="$run_command git pull &&"
 496   fi
 497   run_command="$run_command /pups/bin/pups --stdin"
 498
 499   echo $run_command
 500
 501   env=("${env[@]}" "-e" "SSH_PUB_KEY=$ssh_pub_key")
 502
 503   (exec echo "$input" | $docker_path run $user_args $links "${env[@]}" -e DOCKER_HOST_IP=$docker_ip --cidfile $cidbootstrap -i -a stdin -a stdout -a stderr $volumes $image \
 504      /bin/bash -c "$run_command") \
 505      || ($docker_path rm `cat $cidbootstrap` && rm $cidbootstrap)
 506
 507   [ ! -e $cidbootstrap ] && echo "** FAILED TO BOOTSTRAP ** please scroll up and look for earlier error messages, there may be more than one" && exit 1
 508
 509   sleep 5
 510
 511   $docker_path commit `cat $cidbootstrap` $local_discourse/$config || echo 'FAILED TO COMMIT'
 512   $docker_path rm `cat $cidbootstrap` && rm $cidbootstrap
 513 }
 514
 515 case "$command" in
 516   bootstrap)
 517       run_bootstrap
 518       echo "Successfully bootstrapped, to startup use ./launcher start $config"
 519       exit 0
 520       ;;
 521
 522   enter)
 523       exec $docker_path exec -it $config /bin/bash
 524       ;;
 525
 526   ssh)
 527        existing=`$docker_path ps | awk '{ print $1, $(NF) }' | grep " $config$" | awk '{ print $1 }'`
 528
 529        if [[ ! -z $existing ]]; then
 530          address="`$docker_path port $config 22`"
 531          split=(${address//:/ })
 532          exec ssh -o StrictHostKeyChecking=no root@${split[0]} -p ${split[1]}
 533        else
 534          echo "$config is not running!"
 535          exit 1
 536        fi
 537       ;;
 538
 539   stop)
 540       run_stop
 541       exit 0
 542       ;;
 543
 544   logs)
 545
 546       $docker_path logs $config
 547       exit 0
 548       ;;
 549
 550   restart)
 551       run_stop
 552       run_start
 553       exit 0
 554       ;;
 555
 556   start)
 557       run_start
 558       exit 0
 559       ;;
 560
 561   rebuild)
 562       if [ "$(git symbolic-ref --short HEAD)" == "master" ]; then
 563         echo "Ensuring discourse docker is up to date"
 564
 565         git remote update
 566
 567         LOCAL=$(git rev-parse @)
 568         REMOTE=$(git rev-parse @{u})
 569         BASE=$(git merge-base @ @{u})
 570
 571         if [ $LOCAL = $REMOTE ]; then
 572           echo "Discourse Docker is up-to-date"
 573
 574         elif [ $LOCAL = $BASE ]; then
 575           echo "Updating Discourse Docker"
 576           git pull || (echo 'failed to update' && exit 1)
 577           exec /bin/bash $0 $@
 578
 579         elif [ $REMOTE = $BASE ]; then
 580           echo "Your version of Discourse Docker is ahead of origin"
 581
 582         else
 583           echo "Discourse Docker has diverged source, this is only expected in Dev mode"
 584         fi
 585
 586       fi
 587
 588       set_existing_container
 589
 590       if [ ! -z $existing ]
 591         then
 592           echo "Stopping old container"
 593           (
 594             set -x
 595             $docker_path stop -t 10 $config
 596           )
 597       fi
 598
 599       run_bootstrap
 600
 601       if [ ! -z $existing ]
 602         then
 603           echo "Removing old container"
 604           (
 605             set -x
 606             $docker_path rm $config
 607           )
 608       fi
 609
 610       run_start
 611       exit 0
 612       ;;
 613
 614
 615   destroy)
 616       (set -x; $docker_path stop -t 10 $config && $docker_path rm $config) || (echo "$config was not found" && exit 0)
 617       exit 0
 618       ;;
 619 esac
 620
 621 usage