launcher

   1 #!/bin/bash
   2
   3 usage () {
   4   echo "Usage: launcher COMMAND CONFIG [--skip-prereqs] [--docker-args STRING]"
   5   echo "Commands:"
   6   echo "    start:      Start/initialize a container"
   7   echo "    stop:       Stop a running container"
   8   echo "    restart:    Restart a container"
   9   echo "    destroy:    Stop and remove a container"
  10   echo "    enter:      Open a shell to run commands inside the container"
  11   echo "    logs:       View the Docker logs for a container"
  12   echo "    bootstrap:  Bootstrap a container for the config based on a template"
  13   echo "    rebuild:    Rebuild a container (destroy old, bootstrap, start new)"
  14   echo "    cleanup:    Remove all containers that have stopped for > 24 hours"
  15   echo
  16   echo "Options:"
  17   echo "    --skip-prereqs             Don't check launcher prerequisites"
  18   echo "    --docker-args              Extra arguments to pass when running docker"
  19   echo "    --skip-mac-address         Don't assign a mac address"
  20   exit 1
  21 }
  22
  23 command=$1
  24 config=$2
  25 user_args=""
  26
  27 while [ ${#} -gt 0 ]; do
  28   case "${1}" in
  29   --debug)
  30     DEBUG="1"
  31     ;;
  32   --skip-prereqs)
  33     SKIP_PREREQS="1"
  34     ;;
  35   --skip-mac-address)
  36     SKIP_MAC_ADDRESS="1"
  37     ;;
  38   --docker-args)
  39     user_args="$2"
  40     shift
  41     ;;
  42   esac
  43
  44   shift 1
  45 done
  46
  47 # Docker doesn't like uppercase characters, spaces or special characters, catch it now before we build everything out and then find out
  48 re='[A-Z/ !@#$%^&*()+~`=]'
  49 if [[ $config =~ $re ]];
  50   then
  51     echo
  52     echo "ERROR: Config name must not contain upper case characters, spaces or special characters. Correct config name and rerun $0."
  53     echo
  54     exit 1
  55 fi
  56
  57 cd "$(dirname "$0")"
  58
  59 docker_min_version='1.8.0'
  60 docker_rec_version='1.8.0'
  61 git_min_version='1.8.0'
  62 git_rec_version='1.8.0'
  63
  64 config_file=containers/"$config".yml
  65 cidbootstrap=cids/"$config"_bootstrap.cid
  66 local_discourse=local_discourse
  67 image=discourse/discourse:1.3.5
  68 docker_path=`which docker.io || which docker`
  69 git_path=`which git`
  70
  71 if [ "${SUPERVISED}" = "true" ]; then
  72   restart_policy="--restart=no"
  73   attach_on_start="-a"
  74   attach_on_run="-a stdout -a stderr"
  75 else
  76   attach_on_run="-d"
  77 fi
  78
  79 if [ -n "$DOCKER_HOST" ]; then
  80   docker_ip=`sed -e 's/^tcp:\/\/\(.*\):.*$/\1/' <<< "$DOCKER_HOST"`
  81 elif [ -x "$(which ip 2>/dev/null)" ]; then
  82   docker_ip=`ip addr show docker0 | \
  83                   grep 'inet ' | \
  84                   awk '{ split($2,a,"/"); print a[1] }';`
  85 else
  86   docker_ip=`ifconfig | \
  87                   grep -B1 "inet addr" | \
  88                   awk '{ if ( $1 == "inet" ) { print $2 } else if ( $2 == "Link" ) { printf "%s:" ,$1 } }' | \
  89                   grep docker0 | \
  90                   awk -F: '{ print $3 }';`
  91 fi
  92
  93 compare_version() {
  94     declare -a ver_a
  95     declare -a ver_b
  96     IFS=. read -a ver_a <<< "$1"
  97     IFS=. read -a ver_b <<< "$2"
  98
  99     while [[ -n $ver_a ]]; do
 100         if (( ver_a > ver_b )); then
 101             return 0
 102         elif (( ver_b > ver_a )); then
 103             return 1
 104         else
 105             unset ver_a[0]
 106             ver_a=("${ver_a[@]}")
 107             unset ver_b[0]
 108             ver_b=("${ver_b[@]}")
 109         fi
 110     done
 111     return 1  # They are equal
 112 }
 113
 114
 115 install_docker() {
 116   echo "Docker is not installed, you will need to install Docker in order to run Launcher"
 117   echo "See https://docs.docker.com/installation/"
 118   exit 1
 119 }
 120
 121 check_prereqs() {
 122
 123   if [ -z $docker_path ]; then
 124     install_docker
 125   fi
 126
 127   # 1. docker daemon running?
 128   # we send stderr to /dev/null cause we don't care about warnings,
 129   # it usually complains about swap which does not matter
 130   test=`$docker_path info 2> /dev/null`
 131   if [[ $? -ne 0 ]] ; then
 132     echo "Cannot connect to the docker daemon - verify it is running and you have access"
 133     exit 1
 134   fi
 135
 136   # 2. running aufs or btrfs
 137   test=`$docker_path info 2> /dev/null | grep 'Driver: '`
 138   if [[ "$test" =~ [aufs|btrfs|zfs|overlay] ]] ; then : ; else
 139     echo "Your Docker installation is not using a supported filesystem if we were to proceed you may have a broken install."
 140     echo "aufs is the recommended filesystem you should be using (zfs/btrfs and overlay may work as well)"
 141     echo "You can tell what filesystem you are using by running \"docker info\" and looking at the driver"
 142     echo
 143     echo "If you wish to continue anyway using your existing unsupported filesystem, "
 144     echo "read the source code of launcher and figure out how to bypass this."
 145     exit 1
 146   fi
 147
 148   # 3. running recommended docker version
 149   test=($($docker_path --version))  # Get docker version string
 150   test=${test[2]//,/}  # Get version alone and strip comma if exists
 151
 152   # At least minimum docker version
 153   if compare_version "${docker_min_version}" "${test}"; then
 154     echo "ERROR: Docker version ${test} not supported, please upgrade to at least ${docker_min_version}, or recommended ${docker_rec_version}"
 155     exit 1
 156   fi
 157
 158   # Recommend newer docker version
 159   if compare_version "${docker_rec_version}" "${test}"; then
 160     echo "WARNING: Docker version ${test} deprecated, recommend upgrade to ${docker_rec_version} or newer."
 161   fi
 162
 163   # 4. discourse docker image is downloaded
 164   test=`$docker_path images | awk '{print $1 ":" $2 }' | grep "$image"`
 165
 166   if [ -z "$test" ]; then
 167     echo
 168     echo "WARNING: We are about to start downloading the Discourse base image"
 169     echo "This process may take anywhere between a few minutes to an hour, depending on your network speed"
 170     echo
 171     echo "Please be patient"
 172     echo
 173   fi
 174
 175   # 5. running recommended git version
 176   test=($($git_path --version))  # Get git version string
 177   test=${test[2]//,/}  # Get version alone and strip comma if exists
 178
 179   # At least minimum version
 180   if compare_version "${git_min_version}" "${test}"; then
 181     echo "ERROR: Git version ${test} not supported, please upgrade to at least ${git_min_version}, or recommended ${git_rec_version}"
 182     exit 1
 183   fi
 184
 185   # Recommend best version
 186   if compare_version "${git_rec_version}" "${test}"; then
 187     echo "WARNING: Git version ${test} deprecated, recommend upgrade to ${git_rec_version} or newer."
 188   fi
 189
 190   # 6. able to attach stderr / out / tty
 191   test=`$docker_path run $user_args -i --rm -a stdout -a stderr $image echo working`
 192   if [[ "$test" =~ "working" ]] ; then : ; else
 193     echo "Your Docker installation is not working correctly"
 194     echo
 195     echo "See: https://meta.discourse.org/t/docker-error-on-bootstrap/13657/18?u=sam"
 196     exit 1
 197   fi
 198
 199 }
 200
 201
 202 if [ -z "$SKIP_PREREQS" ] ; then
 203   check_prereqs
 204 fi
 205
 206 host_run() {
 207   read -r -d '' env_ruby << 'RUBY'
 208   require 'yaml'
 209
 210   input = STDIN.readlines.join
 211   yaml = YAML.load(input)
 212
 213   if host_run = yaml['host_run']
 214     params = yaml['params'] || {}
 215     host_run.each do |run|
 216       params.each do |k,v|
 217         run = run.gsub("$#{k}", v)
 218       end
 219       STDOUT.write "#{run}--SEP--"
 220     end
 221   end
 222 RUBY
 223
 224   host_run=`cat $config_file | $docker_path run $user_args --rm -i -a stdout -a stdin $image ruby -e "$env_ruby"`
 225
 226   while [ "$host_run" ] ; do
 227     iter=${host_run%%--SEP--*}
 228     echo
 229     echo "Host run: $iter"
 230     $iter || exit 1
 231     echo
 232     host_run="${host_run#*--SEP--}"
 233   done
 234 }
 235
 236
 237 set_volumes() {
 238   volumes=`cat $config_file | $docker_path run $user_args --rm -i -a stdout -a stdin $image ruby -e \
 239         "require 'yaml'; puts YAML.load(STDIN.readlines.join)['volumes'].map{|v| '-v ' << v['volume']['host'] << ':' << v['volume']['guest'] << ' '}.join"`
 240 }
 241
 242 set_links() {
 243     links=`cat $config_file | $docker_path run $user_args --rm -i -a stdout -a stdin $image ruby -e \
 244         "require 'yaml'; puts YAML.load(STDIN.readlines.join)['links'].map{|l| '--link ' << l['link']['name'] << ':' << l['link']['alias'] << ' '}.join"`
 245 }
 246
 247 find_templates() {
 248     local templates=`cat $1 | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \
 249       "require 'yaml'; puts YAML.load(STDIN.readlines.join)['templates']"`
 250
 251     local arrTemplates=${templates// / }
 252
 253     if [ ! -z "$templates" ]; then
 254       for template in "${arrTemplates[@]}"
 255       do
 256         local nested_templates=$(find_templates $template)
 257
 258         if [ ! -z "$nested_templates" ]; then
 259           templates="$templates $nested_templates"
 260         fi
 261       done
 262
 263       echo $templates
 264     else
 265       echo ""
 266     fi
 267 }
 268
 269 set_template_info() {
 270     templates=$(find_templates $config_file)
 271
 272     arrTemplates=(${templates// / })
 273     config_data=$(cat $config_file)
 274
 275     input="hack: true"
 276
 277     for template in "${arrTemplates[@]}"
 278     do
 279       [ ! -z $template ] && {
 280         input="$input _FILE_SEPERATOR_ $(cat $template)"
 281       }
 282     done
 283
 284     # we always want our config file last so it takes priority
 285     input="$input _FILE_SEPERATOR_ $config_data"
 286
 287     read -r -d '' env_ruby << 'RUBY'
 288     require 'yaml'
 289
 290     input=STDIN.readlines.join
 291     # default to UTF-8 for the dbs sake
 292     env = {'LANG' => 'en_US.UTF-8'}
 293     input.split('_FILE_SEPERATOR_').each do |yml|
 294        yml.strip!
 295        begin
 296          env.merge!(YAML.load(yml)['env'] || {})
 297        rescue Psych::SyntaxError => e
 298         puts e
 299         puts "*ERROR."
 300        rescue => e
 301         puts yml
 302         p e
 303        end
 304     end
 305     puts env.map{|k,v| "-e\n#{k}=#{v}" }.join("\n")
 306 RUBY
 307
 308     raw=`exec echo "$input" | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e "$env_ruby"`
 309
 310     env=()
 311     ok=1
 312     while read i; do
 313       if [ "$i" == "*ERROR." ]; then
 314         ok=0
 315       elif [ -n "$i" ]; then
 316         env[${#env[@]}]=$i
 317       fi
 318     done <<< "$raw"
 319
 320     if [ "$ok" -ne 1 ]; then
 321       echo "${env[@]}"
 322       echo "YAML syntax error. Please check your containers/*.yml config files."
 323       exit 1
 324     fi
 325 }
 326
 327 if [ -z $docker_path ]; then
 328   install_docker
 329 fi
 330
 331 [ "$command" == "cleanup" ] && {
 332   echo
 333   echo "The following command will"
 334   echo "- Delete all docker images for old containers"
 335   echo "- Delete all stopped and orphan containers"
 336   echo
 337   read -p "Are you sure (Y/n): " -n 1 -r && echo
 338   if [[ $REPLY =~ ^[Yy]$ || ! $REPLY ]]
 339     then
 340       space=$(df /var/lib/docker | awk '{ print $4 }' | grep -v Available)
 341       echo "Starting Cleanup (bytes free $space)"
 342
 343       STATE_DIR=./.gc-state scripts/docker-gc
 344
 345       space=$(df /var/lib/docker | awk '{ print $4 }' | grep -v Available)
 346       echo "Finished Cleanup (bytes free $space)"
 347
 348     else
 349       exit 1
 350   fi
 351   exit 0
 352 }
 353
 354 if [ -z "$command" -a -z "$config" ]; then
 355   usage
 356 fi
 357
 358 if [ ! "$command" == "setup" ]; then
 359   if [[ ! -e $config_file ]]; then
 360     echo "Config file was not found, ensure $config_file exists"
 361     echo
 362     echo "Available configs ( `cd containers && ls -dm *.yml | tr -s '\n' ' ' | awk '{ gsub(/\.yml/, ""); print }'`)"
 363     exit 1
 364   fi
 365 fi
 366
 367 docker_version=($($docker_path --version))
 368 docker_version=${test[2]//,/}
 369 restart_policy=${restart_policy:---restart=always}
 370
 371 set_existing_container(){
 372   existing=`$docker_path ps -a | awk '{ print $1, $(NF) }' | grep " $config$" | awk '{ print $1 }'`
 373 }
 374
 375 run_stop() {
 376
 377   set_existing_container
 378
 379   if [ ! -z $existing ]
 380      then
 381        (
 382         set -x
 383         $docker_path stop -t 10 $config
 384        )
 385      else
 386        echo "$config was not started !"
 387        exit 1
 388   fi
 389 }
 390
 391 set_run_image() {
 392   run_image=`cat $config_file | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \
 393     "require 'yaml'; puts YAML.load(STDIN.readlines.join)['run_image']"`
 394
 395   if [ -z "$run_image" ]; then
 396     run_image="$local_discourse/$config"
 397   fi
 398 }
 399
 400 set_boot_command() {
 401   boot_command=`cat $config_file | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \
 402     "require 'yaml'; puts YAML.load(STDIN.readlines.join)['boot_command']"`
 403
 404   if [ -z "$boot_command" ]; then
 405
 406     no_boot_command=`cat $config_file | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \
 407       "require 'yaml'; puts YAML.load(STDIN.readlines.join)['no_boot_command']"`
 408
 409     if [ -z "$no_boot_command" ]; then
 410       boot_command="/sbin/boot"
 411     fi
 412   fi
 413 }
 414
 415 run_start() {
 416
 417    existing=`$docker_path ps | awk '{ print $1, $(NF) }' | grep " $config$" | awk '{ print $1 }'`
 418    echo $existing
 419    if [ ! -z $existing ]
 420    then
 421      echo "Nothing to do, your container has already started!"
 422      exit 0
 423    fi
 424
 425    existing=`$docker_path ps -a | awk '{ print $1, $(NF) }' | grep " $config$" | awk '{ print $1 }'`
 426    if [ ! -z $existing ]
 427    then
 428      echo "starting up existing container"
 429      (
 430        set -x
 431        $docker_path start $config
 432      )
 433      exit 0
 434    fi
 435
 436    host_run
 437
 438    ports=`cat $config_file | $docker_path run $user_args --rm -i -a stdout -a stdin $image ruby -e \
 439          "require 'yaml'; puts YAML.load(STDIN.readlines.join)['expose'].map{|p| \"-p #{p}\"}.join(' ')"`
 440
 441    docker_args=`cat $config_file | $docker_path run $user_args --rm -i -a stdout -a stdin $image ruby -e \
 442           "require 'yaml'; puts YAML.load(STDIN.readlines.join)['docker_args']"`
 443
 444    set_template_info
 445    set_volumes
 446    set_links
 447    set_run_image
 448    set_boot_command
 449
 450    # get hostname and settings from container configuration
 451    for envar in "${env[@]}"
 452    do
 453      if [[ $envar == DOCKER_USE_HOSTNAME* ]] || [[ $envar == DISCOURSE_HOSTNAME* ]]
 454      then
 455        # use as environment variable
 456        eval $envar
 457      fi
 458    done
 459
 460    (
 461      hostname=`hostname -s`
 462      # overwrite hostname
 463      if [ "$DOCKER_USE_HOSTNAME" = "true" ]
 464      then
 465        hostname=$DISCOURSE_HOSTNAME
 466      else
 467        hostname=$hostname-$config
 468      fi
 469
 470      # we got to normalize so we only have allowed strings, this is more comprehensive but lets see how bash does first
 471      # hostname=`$docker_path run $user_args --rm $image ruby -e 'print ARGV[0].gsub(/[^a-zA-Z-]/, "-")' $hostname`
 472      # docker added more hostname rules
 473      hostname=${hostname//_/-}
 474
 475
 476      if [ -z "$SKIP_MAC_ADDRESS" ] ; then
 477       mac_address="--mac-address $($docker_path run $user_args -i --rm -a stdout -a stderr $image /bin/sh -c "echo $hostname | md5sum | sed 's/^\(..\)\(..\)\(..\)\(..\)\(..\).*$/02:\1:\2:\3:\4:\5/'")"
 478      fi
 479
 480      set -x
 481      $docker_path run $user_args $links $attach_on_run $restart_policy "${env[@]}" -h "$hostname" \
 482         -e DOCKER_HOST_IP=$docker_ip --name $config -t $ports $volumes $mac_address $docker_args \
 483         $run_image $boot_command
 484
 485    )
 486    exit 0
 487
 488 }
 489
 490
 491 run_bootstrap() {
 492
 493   # I got no frigging clue what this does, ask Sam Saffron. It RUNS STUFF ON THE HOST I GUESS?
 494   host_run
 495
 496   # Is the image available?
 497   # If not, pull it here so the user is aware what's happening.
 498   $docker_path history $image >/dev/null 2>&1 || $docker_path pull $image
 499
 500   set_template_info
 501
 502   base_image=`cat $config_file | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \
 503     "require 'yaml'; puts YAML.load(STDIN.readlines.join)['base_image']"`
 504
 505   update_pups=`cat $config_file | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \
 506     "require 'yaml'; puts YAML.load(STDIN.readlines.join)['update_pups']"`
 507
 508   if [[ ! X"" = X"$base_image" ]]; then
 509     image=$base_image
 510   fi
 511
 512   set_volumes
 513   set_links
 514
 515   rm -f $cidbootstrap
 516
 517   run_command="cd /pups &&"
 518   if [[ ! "false" =  $update_pups ]]; then
 519     run_command="$run_command git pull &&"
 520   fi
 521   run_command="$run_command /pups/bin/pups --stdin"
 522
 523   echo $run_command
 524
 525   unset ERR
 526   (exec echo "$input" | $docker_path run $user_args $links "${env[@]}" -e DOCKER_HOST_IP=$docker_ip --cidfile $cidbootstrap -i -a stdin -a stdout -a stderr $volumes $image \
 527     /bin/bash -c "$run_command") || ERR=$?
 528
 529   unset FAILED
 530   # magic exit code that indicates a retry
 531   if [[ "$ERR" == 77 ]]; then
 532     $docker_path rm `cat $cidbootstrap`
 533     rm $cidbootstrap
 534     exit 77
 535   elif [[ "$ERR" > 0 ]]; then
 536     FAILED=TRUE
 537   fi
 538
 539   if [[ $FAILED = "TRUE" ]]; then
 540     if [[ ! -z "$DEBUG" ]]; then
 541       $docker_path commit `cat $cidbootstrap` $local_discourse/$config-debug || echo 'FAILED TO COMMIT'
 542       echo "** DEBUG ** Maintaining image for diagnostics $local_discourse/$config-debug"
 543     fi
 544
 545     $docker_path rm `cat $cidbootstrap`
 546     rm $cidbootstrap
 547     echo "** FAILED TO BOOTSTRAP ** please scroll up and look for earlier error messages, there may be more than one"
 548     exit 1
 549   fi
 550
 551   sleep 5
 552
 553   $docker_path commit `cat $cidbootstrap` $local_discourse/$config || echo 'FAILED TO COMMIT'
 554   $docker_path rm `cat $cidbootstrap` && rm $cidbootstrap
 555 }
 556
 557
 558
 559 case "$command" in
 560   bootstrap)
 561       run_bootstrap
 562       echo "Successfully bootstrapped, to startup use ./launcher start $config"
 563       exit 0
 564       ;;
 565
 566   enter)
 567       exec $docker_path exec -it $config /bin/bash --login
 568       ;;
 569
 570   stop)
 571       run_stop
 572       exit 0
 573       ;;
 574
 575   logs)
 576
 577       $docker_path logs $config
 578       exit 0
 579       ;;
 580
 581   restart)
 582       run_stop
 583       run_start
 584       exit 0
 585       ;;
 586
 587   start)
 588       run_start
 589       exit 0
 590       ;;
 591
 592   rebuild)
 593       if [ "$(git symbolic-ref --short HEAD)" == "master" ]; then
 594         echo "Ensuring launcher is up to date"
 595
 596         git remote update
 597
 598         LOCAL=$(git rev-parse @)
 599         REMOTE=$(git rev-parse @{u})
 600         BASE=$(git merge-base @ @{u})
 601
 602         if [ $LOCAL = $REMOTE ]; then
 603           echo "Launcher is up-to-date"
 604
 605         elif [ $LOCAL = $BASE ]; then
 606           echo "Updating Launcher"
 607           git pull || (echo 'failed to update' && exit 1)
 608           exec /bin/bash $0 $@
 609
 610         elif [ $REMOTE = $BASE ]; then
 611           echo "Your version of Launcher is ahead of origin"
 612
 613         else
 614           echo "Launcher has diverged source, this is only expected in Dev mode"
 615         fi
 616
 617       fi
 618
 619       set_existing_container
 620
 621       if [ ! -z $existing ]
 622         then
 623           echo "Stopping old container"
 624           (
 625             set -x
 626             $docker_path stop -t 10 $config
 627           )
 628       fi
 629
 630       run_bootstrap
 631
 632       if [ ! -z $existing ]
 633         then
 634           echo "Removing old container"
 635           (
 636             set -x
 637             $docker_path rm $config
 638           )
 639       fi
 640
 641       run_start
 642       exit 0
 643       ;;
 644
 645
 646   destroy)
 647       (set -x; $docker_path stop -t 10 $config && $docker_path rm $config) || (echo "$config was not found" && exit 0)
 648       exit 0
 649       ;;
 650 esac
 651
 652 usage