launcher

   1 #!/bin/bash
   2
   3 command=$1
   4 config=$2
   5 opt=$3
   6
   7 # Docker doesn't like uppercase characters, spaces or special characters, catch it now before we build everything out and then find out
   8 re='[A-Z/ !@#$%^&*()+~`=]'
   9 if [[ $config =~ $re ]];
  10   then
  11     echo
  12     echo "ERROR: Config name must not contain upper case characters, spaces or special characters. Correct config name and rerun $0."
  13     echo
  14     exit 1
  15 fi
  16
  17 cd "$(dirname "$0")"
  18
  19 docker_min_version='1.2.0'
  20 docker_rec_version='1.2.0'
  21
  22 config_file=containers/"$config".yml
  23 cidbootstrap=cids/"$config"_bootstrap.cid
  24 local_discourse=local_discourse
  25 image=samsaffron/discourse:1.0.12
  26 docker_path=`which docker.io || which docker`
  27
  28 if [ "${SUPERVISED}" = "true" ]; then
  29   restart_policy="--restart=no"
  30   attach_on_start="-a"
  31   attach_on_run="-a stdout -a stderr"
  32 else
  33   attach_on_run="-d"
  34 fi
  35
  36 if [ -n "$DOCKER_HOST" ]; then
  37   docker_ip=`sed -e 's/^tcp:\/\/\(.*\):.*$/\1/' <<< "$DOCKER_HOST"`
  38 elif [ -x "$(which ip 2>/dev/null)" ]; then
  39   docker_ip=`ip addr show docker0 | \
  40                   grep 'inet ' | \
  41                   awk '{ split($2,a,"/"); print a[1] }';`
  42 else
  43   docker_ip=`ifconfig | \
  44                   grep -B1 "inet addr" | \
  45                   awk '{ if ( $1 == "inet" ) { print $2 } else if ( $2 == "Link" ) { printf "%s:" ,$1 } }' | \
  46                   grep docker0 | \
  47                   awk -F: '{ print $3 }';`
  48 fi
  49
  50
  51 usage () {
  52   echo "Usage: launcher COMMAND CONFIG [--skip-prereqs]"
  53   echo "Commands:"
  54   echo "    start:      Start/initialize a container"
  55   echo "    stop:       Stop a running container"
  56   echo "    restart:    Restart a container"
  57   echo "    destroy:    Stop and remove a container"
  58   echo "    enter:      Use nsenter to enter a container"
  59   echo "    ssh:        Start a bash shell in a running container"
  60   echo "    logs:       Docker logs for container"
  61   echo "    bootstrap:  Bootstrap a container for the config based on a template"
  62   echo "    rebuild:    Rebuild a container (destroy old, bootstrap, start new)"
  63   echo "    cleanup:    Remove all containers that have stopped for > 24 hours"
  64   echo
  65   echo "Options:"
  66   echo "    --skip-prereqs   Don't check prerequisites"
  67   echo "    --docker-args    Extra arguments to pass when running docker"
  68   exit 1
  69 }
  70
  71 compare_version() {
  72     declare -a ver_a
  73     declare -a ver_b
  74     IFS=. read -a ver_a <<< "$1"
  75     IFS=. read -a ver_b <<< "$2"
  76
  77     while [[ -n $ver_a ]]; do
  78         if (( ver_a > ver_b )); then
  79             return 0
  80         elif (( ver_b > ver_a )); then
  81             return 1
  82         else
  83             unset ver_a[0]
  84             ver_a=("${ver_a[@]}")
  85             unset ver_b[0]
  86             ver_b=("${ver_b[@]}")
  87         fi
  88     done
  89     return 1  # They are equal
  90 }
  91
  92 prereqs() {
  93
  94   # 1. docker daemon running?
  95   # we send stderr to /dev/null cause we don't care about warnings,
  96   # it usually complains about swap which does not matter
  97   test=`$docker_path info 2> /dev/null`
  98
  99   if [[ $? -ne 0 ]] ; then
 100     echo "Cannot connect to the docker daemon - verify it is running and you have access"
 101     exit 1
 102   fi
 103
 104   # 2. running aufs or btrfs
 105   test=`$docker_path info 2> /dev/null | grep 'Driver: '`
 106   if [[ "$test" =~ [aufs|btrfs|zfs|overlay] ]] ; then : ; else
 107     echo "Your Docker installation is not using a supported filesystem if we were to proceed you may have a broken install."
 108     echo "aufs is the recommended filesystem you should be using (zfs/btrfs and overlay may work as well)"
 109     echo "You can tell what filesystem you are using by running \"docker info\" and looking at the driver"
 110     echo ""
 111     echo "If you wish to continue anyway using your existing unsupported filesystem"
 112     echo "read the source code of launcher and figure out how to bypass this."
 113     exit 1
 114   fi
 115
 116   # 3. running recommended docker version
 117   test=($($docker_path --version))  # Get docker version string
 118   test=${test[2]//,/}  # Get version alone and strip comma if exists
 119
 120   [[ "$test" =~ "0.12.0" ]] && echo "You are running a broken version of Docker, please upgrade ASAP. See: https://meta.discourse.org/t/the-installation-stopped-in-the-middle/16311/ for more details." && exit 1
 121
 122   # At least minimum version
 123   if compare_version "${docker_min_version}" "${test}"; then
 124     echo "ERROR: Docker version ${test} not supported, please upgrade to at least ${docker_min_version}, or recommended ${docker_rec_version}"
 125     exit 1
 126   fi
 127
 128   # Recommend best version
 129   if compare_version "${docker_rec_version}" "${test}"; then
 130     echo "WARNING: Docker version ${test} deprecated, recommend upgrade to ${docker_rec_version} or newer."
 131   fi
 132
 133   # 4. discourse docker image is downloaded
 134   test=`$docker_path images | awk '{print $1 ":" $2 }' | grep "$image"`
 135
 136   if [ -z "$test" ]; then
 137     echo
 138     echo "WARNING: We are about to start downloading the Discourse base image"
 139     echo "This process may take anywhere between a few minutes to an hour, depending on your network speed"
 140     echo
 141     echo "Please be patient"
 142     echo
 143
 144   fi
 145
 146   # 5. able to attach stderr / out / tty
 147   test=`$docker_path run $user_args -i --rm -a stdout -a stderr $image echo working`
 148   if [[ "$test" =~ "working" ]] ; then : ; else
 149     echo "Your Docker installation is not working correctly"
 150     echo
 151     echo "See: https://meta.discourse.org/t/docker-error-on-bootstrap/13657/18?u=sam"
 152     exit 1
 153   fi
 154 }
 155
 156 if [ "$opt" != "--skip-prereqs" ] ; then
 157   prereqs
 158 fi
 159
 160 if [ "$opt" == "--docker-args" ] ; then
 161   user_args=$4
 162 else
 163   user_args=""
 164 fi
 165
 166 get_ssh_pub_key() {
 167   local ${ssh_key_locations}
 168   ssh_key_locations=(
 169     ~/.ssh/id_ed25519.pub
 170     ~/.ssh/id_ecdsa.pub
 171     ~/.ssh/id_rsa.pub
 172     ~/.ssh/id_dsa.pub
 173     ~core/.ssh/authorized_keys
 174   )
 175
 176   local $keyfile
 177   for keyfile in "${ssh_key_locations[@]}"; do
 178     if [[ -e ${keyfile} ]] ; then
 179       ssh_pub_key="$(cat ${keyfile})"
 180       return 0
 181     fi
 182   done
 183
 184   return 0
 185 }
 186
 187
 188 install_docker() {
 189
 190   echo "Docker is not installed, make sure you are running on the 3.8 kernel"
 191   echo "The best supported Docker release is Ubuntu 12.04.03 for it run the following"
 192   echo
 193   echo "sudo apt-get update"
 194   echo "sudo apt-get install linux-image-generic-lts-raring linux-headers-generic-lts-raring"
 195   echo "sudo reboot"
 196   echo
 197
 198   echo "sudo apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D"
 199   echo "sudo sh -c \"echo deb https://apt.dockerproject.org/repo ubuntu-precise main > /etc/apt/sources.list.d/docker.list\""
 200   echo "sudo apt-get update"
 201   echo "sudo apt-get install docker-engine"
 202
 203   exit 1
 204 }
 205
 206 host_run() {
 207   read -r -d '' env_ruby << 'RUBY'
 208   require 'yaml'
 209
 210   input = STDIN.readlines.join
 211   yaml = YAML.load(input)
 212
 213   if host_run = yaml['host_run']
 214     params = yaml['params'] || {}
 215     host_run.each do |run|
 216       params.each do |k,v|
 217         run = run.gsub("$#{k}", v)
 218       end
 219       STDOUT.write "#{run}--SEP--"
 220     end
 221   end
 222 RUBY
 223
 224   host_run=`cat $config_file | $docker_path run $user_args --rm -i -a stdout -a stdin $image ruby -e "$env_ruby"`
 225
 226   while [ "$host_run" ] ; do
 227     iter=${host_run%%--SEP--*}
 228     echo
 229     echo "Host run: $iter"
 230     $iter || exit 1
 231     echo
 232     host_run="${host_run#*--SEP--}"
 233   done
 234 }
 235
 236
 237 set_volumes() {
 238   volumes=`cat $config_file | $docker_path run $user_args --rm -i -a stdout -a stdin $image ruby -e \
 239         "require 'yaml'; puts YAML.load(STDIN.readlines.join)['volumes'].map{|v| '-v ' << v['volume']['host'] << ':' << v['volume']['guest'] << ' '}.join"`
 240 }
 241
 242 set_links() {
 243     links=`cat $config_file | $docker_path run $user_args --rm -i -a stdout -a stdin $image ruby -e \
 244         "require 'yaml'; puts YAML.load(STDIN.readlines.join)['links'].map{|l| '--link ' << l['link']['name'] << ':' << l['link']['alias'] << ' '}.join"`
 245 }
 246
 247 set_template_info() {
 248
 249     templates=`cat $config_file | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \
 250       "require 'yaml'; puts YAML.load(STDIN.readlines.join)['templates']"`
 251
 252
 253     arrTemplates=(${templates// / })
 254     config_data=$(cat $config_file)
 255
 256     input="hack: true"
 257
 258
 259     for template in "${arrTemplates[@]}"
 260     do
 261       [ ! -z $template ] && {
 262         input="$input _FILE_SEPERATOR_ $(cat $template)"
 263       }
 264     done
 265
 266     # we always want our config file last so it takes priority
 267     input="$input _FILE_SEPERATOR_ $config_data"
 268
 269     read -r -d '' env_ruby << 'RUBY'
 270     require 'yaml'
 271
 272     input=STDIN.readlines.join
 273     # default to UTF-8 for the dbs sake
 274     env = {'LANG' => 'en_US.UTF-8'}
 275     input.split('_FILE_SEPERATOR_').each do |yml|
 276        yml.strip!
 277        begin
 278          env.merge!(YAML.load(yml)['env'] || {})
 279        rescue Psych::SyntaxError => e
 280         puts e
 281         puts "*ERROR."
 282        rescue => e
 283         puts yml
 284         p e
 285        end
 286     end
 287     puts env.map{|k,v| "-e\n#{k}=#{v}" }.join("\n")
 288 RUBY
 289
 290     raw=`exec echo "$input" | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e "$env_ruby"`
 291
 292     env=()
 293     ok=1
 294     while read i; do
 295       if [ "$i" == "*ERROR." ]; then
 296         ok=0
 297       elif [ -n "$i" ]; then
 298         env[${#env[@]}]=$i
 299       fi
 300     done <<< "$raw"
 301
 302     if [ "$ok" -ne 1 ]; then
 303       echo "${env[@]}"
 304       echo "YAML syntax error. Please check your /var/discourse/containers/*.yml config files."
 305       exit 1
 306     fi
 307 }
 308
 309 [ -z $docker_path ] && {
 310   install_docker
 311 }
 312
 313 [ "$command" == "cleanup" ] && {
 314   echo
 315   echo "The following command will"
 316   echo "- Delete all docker images for old containers"
 317   echo "- Delete all stopped and orphan containers"
 318   echo
 319   read -p "Are you sure (Y/n): " -n 1 -r && echo
 320   if [[ $REPLY =~ ^[Yy]$ || ! $REPLY ]]
 321     then
 322       space=$(df /var/lib/docker | awk '{ print $4 }' | grep -v Available)
 323       echo "Starting Cleanup (bytes free $space)"
 324
 325       STATE_DIR=./.gc-state scripts/docker-gc
 326
 327       space=$(df /var/lib/docker | awk '{ print $4 }' | grep -v Available)
 328       echo "Finished Cleanup (bytes free $space)"
 329
 330     else
 331       exit 1
 332   fi
 333   exit 0
 334 }
 335
 336 [ $# -lt 2 ] && {
 337   usage
 338 }
 339
 340 if [ ! -e $config_file ]
 341   then
 342     echo "Config file was not found, ensure $config_file exists"
 343     echo ""
 344     echo "Available configs ( `cd containers && ls -dm *.yml | tr -s '\n' ' ' | awk '{ gsub(/\.yml/, ""); print }'`)"
 345     exit 1
 346 fi
 347
 348
 349 docker_version=($($docker_path --version))
 350 docker_version=${test[2]//,/}
 351
 352 if compare_version "1.2.0" "$docker_version"; then
 353   echo "We recommend you upgrade docker, the version you are running has no restart policies, on reboot your container may not start up"
 354   restart_policy=""
 355 else
 356   restart_policy=${restart_policy:---restart=always}
 357 fi
 358
 359 set_existing_container(){
 360   existing=`$docker_path ps -a | awk '{ print $1, $(NF) }' | grep " $config$" | awk '{ print $1 }'`
 361 }
 362
 363 run_stop(){
 364
 365   set_existing_container
 366
 367   if [ ! -z $existing ]
 368      then
 369        (
 370         set -x
 371         $docker_path stop -t 10 $config
 372        )
 373      else
 374        echo "$config was not started !"
 375        exit 1
 376   fi
 377 }
 378
 379 run_start(){
 380
 381    existing=`$docker_path ps | awk '{ print $1, $(NF) }' | grep " $config$" | awk '{ print $1 }'`
 382    echo $existing
 383    if [ ! -z $existing ]
 384    then
 385      echo "Nothing to do, your container has already started!"
 386      exit 1
 387    fi
 388
 389    existing=`$docker_path ps -a | awk '{ print $1, $(NF) }' | grep " $config$" | awk '{ print $1 }'`
 390    if [ ! -z $existing ]
 391    then
 392      echo "starting up existing container"
 393      (
 394        set -x
 395        $docker_path start $config
 396      )
 397      exit 0
 398    fi
 399
 400    host_run
 401    ports=`cat $config_file | $docker_path run $user_args --rm -i -a stdout -a stdin $image ruby -e \
 402           "require 'yaml'; puts YAML.load(STDIN.readlines.join)['expose'].map{|p| '-p ' << p.to_s << ' '}.join"`
 403
 404    docker_args=`cat $config_file | $docker_path run $user_args --rm -i -a stdout -a stdin $image ruby -e \
 405           "require 'yaml'; puts YAML.load(STDIN.readlines.join)['docker_args']"`
 406
 407    set_template_info
 408    set_volumes
 409    set_links
 410
 411    (
 412      hostname=`hostname`
 413      set -x
 414      $docker_path run $user_args $links $attach_on_run $restart_policy "${env[@]}" -h "$hostname-$config" \
 415         -e DOCKER_HOST_IP=$docker_ip --name $config -t $ports $volumes $docker_args $local_discourse/$config \
 416         /sbin/boot
 417
 418    )
 419    exit 0
 420
 421 }
 422
 423 run_bootstrap(){
 424
 425   host_run
 426
 427   get_ssh_pub_key
 428
 429   # Is the image available?
 430   # If not, pull it here so the user is aware what's happening.
 431   $docker_path history $image >/dev/null 2>&1 || $docker_path pull $image
 432
 433   set_template_info
 434
 435   base_image=`cat $config_file | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \
 436     "require 'yaml'; puts YAML.load(STDIN.readlines.join)['base_image']"`
 437
 438   update_pups=`cat $config_file | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \
 439     "require 'yaml'; puts YAML.load(STDIN.readlines.join)['update_pups']"`
 440
 441   if [[ ! X"" = X"$base_image" ]]; then
 442     image=$base_image
 443   fi
 444
 445   set_volumes
 446   set_links
 447
 448   rm -f $cidbootstrap
 449
 450   run_command="cd /pups &&"
 451   if [[ ! "false" =  $update_pups ]]; then
 452     run_command="$run_command git pull &&"
 453   fi
 454   run_command="$run_command /pups/bin/pups --stdin"
 455
 456   echo $run_command
 457
 458   env=("${env[@]}" "-e" "SSH_PUB_KEY=$ssh_pub_key")
 459
 460   (exec echo "$input" | $docker_path run $user_args $links "${env[@]}" -e DOCKER_HOST_IP=$docker_ip --cidfile $cidbootstrap -i -a stdin -a stdout -a stderr $volumes $image \
 461      /bin/bash -c "$run_command") \
 462      || ($docker_path rm `cat $cidbootstrap` && rm $cidbootstrap)
 463
 464   [ ! -e $cidbootstrap ] && echo "** FAILED TO BOOTSTRAP ** please scroll up and look for earlier error messages, there may be more than one" && exit 1
 465
 466   sleep 5
 467
 468   $docker_path commit `cat $cidbootstrap` $local_discourse/$config || echo 'FAILED TO COMMIT'
 469   $docker_path rm `cat $cidbootstrap` && rm $cidbootstrap
 470 }
 471
 472 case "$command" in
 473   bootstrap)
 474       run_bootstrap
 475       echo "Successfully bootstrapped, to startup use ./launcher start $config"
 476       exit 0
 477       ;;
 478
 479   enter)
 480       exec $docker_path exec -it $config /bin/bash
 481       ;;
 482
 483   ssh)
 484        existing=`$docker_path ps | awk '{ print $1, $(NF) }' | grep " $config$" | awk '{ print $1 }'`
 485
 486        if [[ ! -z $existing ]]; then
 487          address="`$docker_path port $config 22`"
 488          split=(${address//:/ })
 489          exec ssh -o StrictHostKeyChecking=no root@${split[0]} -p ${split[1]}
 490        else
 491          echo "$config is not running!"
 492          exit 1
 493        fi
 494       ;;
 495
 496   stop)
 497       run_stop
 498       exit 0
 499       ;;
 500
 501   logs)
 502
 503       $docker_path logs $config
 504       exit 0
 505       ;;
 506
 507   restart)
 508       run_stop
 509       run_start
 510       exit 0
 511       ;;
 512
 513   start)
 514       run_start
 515       exit 0
 516       ;;
 517
 518   rebuild)
 519       if [ "$(git symbolic-ref --short HEAD)" == "master" ]; then
 520         echo "Ensuring discourse docker is up to date"
 521
 522         git remote update
 523
 524         LOCAL=$(git rev-parse @)
 525         REMOTE=$(git rev-parse @{u})
 526         BASE=$(git merge-base @ @{u})
 527
 528         if [ $LOCAL = $REMOTE ]; then
 529           echo "Discourse Docker is up-to-date"
 530
 531         elif [ $LOCAL = $BASE ]; then
 532           echo "Updating Discourse Docker"
 533           git pull || (echo 'failed to update' && exit 1)
 534           exec /bin/bash $0 $@
 535
 536         elif [ $REMOTE = $BASE ]; then
 537           echo "Your version of Discourse Docker is ahead of origin"
 538
 539         else
 540           echo "Discourse Docker has diverged source, this is only expected in Dev mode"
 541         fi
 542
 543       fi
 544
 545       set_existing_container
 546
 547       if [ ! -z $existing ]
 548         then
 549           echo "Stopping old container"
 550           (
 551             set -x
 552             $docker_path stop -t 10 $config
 553           )
 554       fi
 555
 556       run_bootstrap
 557
 558       if [ ! -z $existing ]
 559         then
 560           echo "Removing old container"
 561           (
 562             set -x
 563             $docker_path rm $config
 564           )
 565       fi
 566
 567       run_start
 568       exit 0
 569       ;;
 570
 571
 572   destroy)
 573       (set -x; $docker_path stop -t 10 $config && $docker_path rm $config) || (echo "$config was not found" && exit 0)
 574       exit 0
 575       ;;
 576 esac
 577
 578 usage