launcher

   1 #!/bin/bash
   2
   3 usage () {
   4   echo "Usage: launcher COMMAND CONFIG [--skip-prereqs] [--docker-args STRING]"
   5   echo "Commands:"
   6   echo "    start:      Start/initialize a container"
   7   echo "    stop:       Stop a running container"
   8   echo "    restart:    Restart a container"
   9   echo "    destroy:    Stop and remove a container"
  10   echo "    enter:      Open a shell to run commands inside the container"
  11   echo "    logs:       View the Docker logs for a container"
  12   echo "    bootstrap:  Bootstrap a container for the config based on a template"
  13   echo "    rebuild:    Rebuild a container (destroy old, bootstrap, start new)"
  14   echo "    cleanup:    Remove all containers that have stopped for > 24 hours"
  15   echo
  16   echo "Options:"
  17   echo "    --skip-prereqs             Don't check launcher prerequisites"
  18   echo "    --docker-args              Extra arguments to pass when running docker"
  19   echo "    --skip-mac-address         Don't assign a mac address"
  20   exit 1
  21 }
  22
  23 command=$1
  24 config=$2
  25 user_args=""
  26
  27 while [ ${#} -gt 0 ]; do
  28   case "${1}" in
  29   --debug)
  30     DEBUG="1"
  31     ;;
  32   --skip-prereqs)
  33     SKIP_PREREQS="1"
  34     ;;
  35   --skip-mac-address)
  36     SKIP_MAC_ADDRESS="1"
  37     ;;
  38   --docker-args)
  39     user_args="$2"
  40     shift
  41     ;;
  42   esac
  43
  44   shift 1
  45 done
  46
  47 # Docker doesn't like uppercase characters, spaces or special characters, catch it now before we build everything out and then find out
  48 re='[A-Z/ !@#$%^&*()+~`=]'
  49 if [[ $config =~ $re ]];
  50   then
  51     echo
  52     echo "ERROR: Config name must not contain upper case characters, spaces or special characters. Correct config name and rerun $0."
  53     echo
  54     exit 1
  55 fi
  56
  57 cd "$(dirname "$0")"
  58
  59 docker_min_version='1.8.0'
  60 docker_rec_version='1.8.0'
  61 git_min_version='1.8.0'
  62 git_rec_version='1.8.0'
  63
  64 config_file=containers/"$config".yml
  65 cidbootstrap=cids/"$config"_bootstrap.cid
  66 local_discourse=local_discourse
  67 image=discourse/discourse:1.3.5
  68 docker_path=`which docker.io || which docker`
  69 git_path=`which git`
  70
  71 if [ "${SUPERVISED}" = "true" ]; then
  72   restart_policy="--restart=no"
  73   attach_on_start="-a"
  74   attach_on_run="-a stdout -a stderr"
  75 else
  76   attach_on_run="-d"
  77 fi
  78
  79 if [ -n "$DOCKER_HOST" ]; then
  80   docker_ip=`sed -e 's/^tcp:\/\/\(.*\):.*$/\1/' <<< "$DOCKER_HOST"`
  81 elif [ -x "$(which ip 2>/dev/null)" ]; then
  82   docker_ip=`ip addr show docker0 | \
  83                   grep 'inet ' | \
  84                   awk '{ split($2,a,"/"); print a[1] }';`
  85 else
  86   docker_ip=`ifconfig | \
  87                   grep -B1 "inet addr" | \
  88                   awk '{ if ( $1 == "inet" ) { print $2 } else if ( $2 == "Link" ) { printf "%s:" ,$1 } }' | \
  89                   grep docker0 | \
  90                   awk -F: '{ print $3 }';`
  91 fi
  92
  93 compare_version() {
  94     declare -a ver_a
  95     declare -a ver_b
  96     IFS=. read -a ver_a <<< "$1"
  97     IFS=. read -a ver_b <<< "$2"
  98
  99     while [[ -n $ver_a ]]; do
 100         if (( ver_a > ver_b )); then
 101             return 0
 102         elif (( ver_b > ver_a )); then
 103             return 1
 104         else
 105             unset ver_a[0]
 106             ver_a=("${ver_a[@]}")
 107             unset ver_b[0]
 108             ver_b=("${ver_b[@]}")
 109         fi
 110     done
 111     return 1  # They are equal
 112 }
 113
 114
 115 install_docker() {
 116   echo "Docker is not installed, you will need to install Docker in order to run Launcher"
 117   echo "See https://docs.docker.com/installation/"
 118   exit 1
 119 }
 120
 121 check_prereqs() {
 122
 123   if [ -z $docker_path ]; then
 124     install_docker
 125   fi
 126
 127   # 1. docker daemon running?
 128   # we send stderr to /dev/null cause we don't care about warnings,
 129   # it usually complains about swap which does not matter
 130   test=`$docker_path info 2> /dev/null`
 131   if [[ $? -ne 0 ]] ; then
 132     echo "Cannot connect to the docker daemon - verify it is running and you have access"
 133     exit 1
 134   fi
 135
 136   # 2. running aufs or btrfs
 137   test=`$docker_path info 2> /dev/null | grep 'Driver: '`
 138   if [[ "$test" =~ [aufs|btrfs|zfs|overlay] ]] ; then : ; else
 139     echo "Your Docker installation is not using a supported filesystem if we were to proceed you may have a broken install."
 140     echo "aufs is the recommended filesystem you should be using (zfs/btrfs and overlay may work as well)"
 141     echo "You can tell what filesystem you are using by running \"docker info\" and looking at the driver"
 142     echo
 143     echo "If you wish to continue anyway using your existing unsupported filesystem, "
 144     echo "read the source code of launcher and figure out how to bypass this."
 145     exit 1
 146   fi
 147
 148   # 3. running recommended docker version
 149   test=($($docker_path --version))  # Get docker version string
 150   test=${test[2]//,/}  # Get version alone and strip comma if exists
 151
 152   # At least minimum docker version
 153   if compare_version "${docker_min_version}" "${test}"; then
 154     echo "ERROR: Docker version ${test} not supported, please upgrade to at least ${docker_min_version}, or recommended ${docker_rec_version}"
 155     exit 1
 156   fi
 157
 158   # Recommend newer docker version
 159   if compare_version "${docker_rec_version}" "${test}"; then
 160     echo "WARNING: Docker version ${test} deprecated, recommend upgrade to ${docker_rec_version} or newer."
 161   fi
 162
 163   # 4. discourse docker image is downloaded
 164   test=`$docker_path images | awk '{print $1 ":" $2 }' | grep "$image"`
 165
 166   if [ -z "$test" ]; then
 167     echo
 168     echo "WARNING: We are about to start downloading the Discourse base image"
 169     echo "This process may take anywhere between a few minutes to an hour, depending on your network speed"
 170     echo
 171     echo "Please be patient"
 172     echo
 173   fi
 174
 175   # 5. running recommended git version
 176   test=($($git_path --version))  # Get git version string
 177   test=${test[2]//,/}  # Get version alone and strip comma if exists
 178
 179   # At least minimum version
 180   if compare_version "${git_min_version}" "${test}"; then
 181     echo "ERROR: Git version ${test} not supported, please upgrade to at least ${git_min_version}, or recommended ${git_rec_version}"
 182     exit 1
 183   fi
 184
 185   # Recommend best version
 186   if compare_version "${git_rec_version}" "${test}"; then
 187     echo "WARNING: Git version ${test} deprecated, recommend upgrade to ${git_rec_version} or newer."
 188   fi
 189
 190   # 6. able to attach stderr / out / tty
 191   test=`$docker_path run $user_args -i --rm -a stdout -a stderr $image echo working`
 192   if [[ "$test" =~ "working" ]] ; then : ; else
 193     echo "Your Docker installation is not working correctly"
 194     echo
 195     echo "See: https://meta.discourse.org/t/docker-error-on-bootstrap/13657/18?u=sam"
 196     exit 1
 197   fi
 198
 199 }
 200
 201
 202 if [ -z "$SKIP_PREREQS" ] ; then
 203   check_prereqs
 204 fi
 205
 206 host_run() {
 207   read -r -d '' env_ruby << 'RUBY'
 208   require 'yaml'
 209
 210   input = STDIN.readlines.join
 211   yaml = YAML.load(input)
 212
 213   if host_run = yaml['host_run']
 214     params = yaml['params'] || {}
 215     host_run.each do |run|
 216       params.each do |k,v|
 217         run = run.gsub("$#{k}", v)
 218       end
 219       STDOUT.write "#{run}--SEP--"
 220     end
 221   end
 222 RUBY
 223
 224   host_run=`cat $config_file | $docker_path run $user_args --rm -i -a stdout -a stdin $image ruby -e "$env_ruby"`
 225
 226   while [ "$host_run" ] ; do
 227     iter=${host_run%%--SEP--*}
 228     echo
 229     echo "Host run: $iter"
 230     $iter || exit 1
 231     echo
 232     host_run="${host_run#*--SEP--}"
 233   done
 234 }
 235
 236
 237 set_volumes() {
 238   volumes=`cat $config_file | $docker_path run $user_args --rm -i -a stdout -a stdin $image ruby -e \
 239         "require 'yaml'; puts YAML.load(STDIN.readlines.join)['volumes'].map{|v| '-v ' << v['volume']['host'] << ':' << v['volume']['guest'] << ' '}.join"`
 240 }
 241
 242 set_links() {
 243     links=`cat $config_file | $docker_path run $user_args --rm -i -a stdout -a stdin $image ruby -e \
 244         "require 'yaml'; puts YAML.load(STDIN.readlines.join)['links'].map{|l| '--link ' << l['link']['name'] << ':' << l['link']['alias'] << ' '}.join"`
 245 }
 246
 247 find_templates() {
 248     local templates=`cat $1 | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \
 249       "require 'yaml'; puts YAML.load(STDIN.readlines.join)['templates']"`
 250
 251     local arrTemplates=${templates// / }
 252
 253     if [ ! -z "$templates" ]; then
 254       for template in "${arrTemplates[@]}"
 255       do
 256         local nested_templates=$(find_templates $template)
 257
 258         if [ ! -z "$nested_templates" ]; then
 259           templates="$templates $nested_templates"
 260         fi
 261       done
 262
 263       echo $templates
 264     else
 265       echo ""
 266     fi
 267 }
 268
 269 set_template_info() {
 270     templates=$(find_templates $config_file)
 271
 272     arrTemplates=(${templates// / })
 273     config_data=$(cat $config_file)
 274
 275     input="hack: true"
 276
 277     for template in "${arrTemplates[@]}"
 278     do
 279       [ ! -z $template ] && {
 280         input="$input _FILE_SEPERATOR_ $(cat $template)"
 281       }
 282     done
 283
 284     # we always want our config file last so it takes priority
 285     input="$input _FILE_SEPERATOR_ $config_data"
 286
 287     read -r -d '' env_ruby << 'RUBY'
 288     require 'yaml'
 289
 290     input=STDIN.readlines.join
 291     # default to UTF-8 for the dbs sake
 292     env = {'LANG' => 'en_US.UTF-8'}
 293     input.split('_FILE_SEPERATOR_').each do |yml|
 294        yml.strip!
 295        begin
 296          env.merge!(YAML.load(yml)['env'] || {})
 297        rescue Psych::SyntaxError => e
 298         puts e
 299         puts "*ERROR."
 300        rescue => e
 301         puts yml
 302         p e
 303        end
 304     end
 305     puts env.map{|k,v| "-e\n#{k}=#{v}" }.join("\n")
 306 RUBY
 307
 308     raw=`exec echo "$input" | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e "$env_ruby"`
 309
 310     env=()
 311     ok=1
 312     while read i; do
 313       if [ "$i" == "*ERROR." ]; then
 314         ok=0
 315       elif [ -n "$i" ]; then
 316         env[${#env[@]}]=$i
 317       fi
 318     done <<< "$raw"
 319
 320     if [ "$ok" -ne 1 ]; then
 321       echo "${env[@]}"
 322       echo "YAML syntax error. Please check your containers/*.yml config files."
 323       exit 1
 324     fi
 325
 326     read -r -d '' labels_ruby << 'RUBY'
 327     require 'yaml'
 328
 329     input=STDIN.readlines.join
 330     # default to UTF-8 for the dbs sake
 331     labels = {}
 332     input.split('_FILE_SEPERATOR_').each do |yml|
 333        yml.strip!
 334        begin
 335          labels.merge!(YAML.load(yml)['labels'] || {})
 336        rescue Psych::SyntaxError => e
 337         puts e
 338         puts "*ERROR."
 339        rescue => e
 340         puts yml
 341         p e
 342        end
 343     end
 344     puts labels.map{|k,v| "-l\n#{k}=#{v}" }.join("\n")
 345 RUBY
 346
 347     raw=`exec echo "$input" | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e "$labels_ruby"`
 348
 349     labels=()
 350     ok=1
 351     while read i; do
 352       if [ "$i" == "*ERROR." ]; then
 353         ok=0
 354       elif [ -n "$i" ]; then
 355         labels[${#labels[@]}]=$i
 356       fi
 357     done <<< "$raw"
 358
 359     if [ "$ok" -ne 1 ]; then
 360       echo "${labels[@]}"
 361       echo "YAML syntax error. Please check your containers/*.yml config files."
 362       exit 1
 363     fi
 364 }
 365
 366 if [ -z $docker_path ]; then
 367   install_docker
 368 fi
 369
 370 [ "$command" == "cleanup" ] && {
 371   echo
 372   echo "The following command will"
 373   echo "- Delete all docker images for old containers"
 374   echo "- Delete all stopped and orphan containers"
 375   echo
 376   read -p "Are you sure (Y/n): " -n 1 -r && echo
 377   if [[ $REPLY =~ ^[Yy]$ || ! $REPLY ]]
 378     then
 379       space=$(df /var/lib/docker | awk '{ print $4 }' | grep -v Available)
 380       echo "Starting Cleanup (bytes free $space)"
 381
 382       STATE_DIR=./.gc-state scripts/docker-gc
 383
 384       space=$(df /var/lib/docker | awk '{ print $4 }' | grep -v Available)
 385       echo "Finished Cleanup (bytes free $space)"
 386
 387     else
 388       exit 1
 389   fi
 390   exit 0
 391 }
 392
 393 if [ -z "$command" -a -z "$config" ]; then
 394   usage
 395 fi
 396
 397 if [ ! "$command" == "setup" ]; then
 398   if [[ ! -e $config_file ]]; then
 399     echo "Config file was not found, ensure $config_file exists"
 400     echo
 401     echo "Available configs ( `cd containers && ls -dm *.yml | tr -s '\n' ' ' | awk '{ gsub(/\.yml/, ""); print }'`)"
 402     exit 1
 403   fi
 404 fi
 405
 406 docker_version=($($docker_path --version))
 407 docker_version=${test[2]//,/}
 408 restart_policy=${restart_policy:---restart=always}
 409
 410 set_existing_container(){
 411   existing=`$docker_path ps -a | awk '{ print $1, $(NF) }' | grep " $config$" | awk '{ print $1 }'`
 412 }
 413
 414 run_stop() {
 415
 416   set_existing_container
 417
 418   if [ ! -z $existing ]
 419      then
 420        (
 421         set -x
 422         $docker_path stop -t 10 $config
 423        )
 424      else
 425        echo "$config was not started !"
 426        exit 1
 427   fi
 428 }
 429
 430 set_run_image() {
 431   run_image=`cat $config_file | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \
 432     "require 'yaml'; puts YAML.load(STDIN.readlines.join)['run_image']"`
 433
 434   if [ -z "$run_image" ]; then
 435     run_image="$local_discourse/$config"
 436   fi
 437 }
 438
 439 set_boot_command() {
 440   boot_command=`cat $config_file | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \
 441     "require 'yaml'; puts YAML.load(STDIN.readlines.join)['boot_command']"`
 442
 443   if [ -z "$boot_command" ]; then
 444
 445     no_boot_command=`cat $config_file | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \
 446       "require 'yaml'; puts YAML.load(STDIN.readlines.join)['no_boot_command']"`
 447
 448     if [ -z "$no_boot_command" ]; then
 449       boot_command="/sbin/boot"
 450     fi
 451   fi
 452 }
 453
 454 run_start() {
 455
 456    existing=`$docker_path ps | awk '{ print $1, $(NF) }' | grep " $config$" | awk '{ print $1 }'`
 457    echo $existing
 458    if [ ! -z $existing ]
 459    then
 460      echo "Nothing to do, your container has already started!"
 461      exit 0
 462    fi
 463
 464    existing=`$docker_path ps -a | awk '{ print $1, $(NF) }' | grep " $config$" | awk '{ print $1 }'`
 465    if [ ! -z $existing ]
 466    then
 467      echo "starting up existing container"
 468      (
 469        set -x
 470        $docker_path start $config
 471      )
 472      exit 0
 473    fi
 474
 475    host_run
 476
 477    ports=`cat $config_file | $docker_path run $user_args --rm -i -a stdout -a stdin $image ruby -e \
 478          "require 'yaml'; puts YAML.load(STDIN.readlines.join)['expose'].map{|p| \"-p #{p}\"}.join(' ')"`
 479
 480    docker_args=`cat $config_file | $docker_path run $user_args --rm -i -a stdout -a stdin $image ruby -e \
 481           "require 'yaml'; puts YAML.load(STDIN.readlines.join)['docker_args']"`
 482
 483    set_template_info
 484    set_volumes
 485    set_links
 486    set_run_image
 487    set_boot_command
 488
 489    # get hostname and settings from container configuration
 490    for envar in "${env[@]}"
 491    do
 492      if [[ $envar == DOCKER_USE_HOSTNAME* ]] || [[ $envar == DISCOURSE_HOSTNAME* ]]
 493      then
 494        # use as environment variable
 495        eval $envar
 496      fi
 497    done
 498
 499    (
 500      hostname=`hostname -s`
 501      # overwrite hostname
 502      if [ "$DOCKER_USE_HOSTNAME" = "true" ]
 503      then
 504        hostname=$DISCOURSE_HOSTNAME
 505      else
 506        hostname=$hostname-$config
 507      fi
 508
 509      # we got to normalize so we only have allowed strings, this is more comprehensive but lets see how bash does first
 510      # hostname=`$docker_path run $user_args --rm $image ruby -e 'print ARGV[0].gsub(/[^a-zA-Z-]/, "-")' $hostname`
 511      # docker added more hostname rules
 512      hostname=${hostname//_/-}
 513
 514
 515      if [ -z "$SKIP_MAC_ADDRESS" ] ; then
 516       mac_address="--mac-address $($docker_path run $user_args -i --rm -a stdout -a stderr $image /bin/sh -c "echo $hostname | md5sum | sed 's/^\(..\)\(..\)\(..\)\(..\)\(..\).*$/02:\1:\2:\3:\4:\5/'")"
 517      fi
 518
 519      set -x
 520      $docker_path run $user_args $links $attach_on_run $restart_policy "${env[@]}" "${labels[@]}" -h "$hostname" \
 521         -e DOCKER_HOST_IP=$docker_ip --name $config -t $ports $volumes $mac_address $docker_args \
 522         $run_image $boot_command
 523
 524    )
 525    exit 0
 526
 527 }
 528
 529
 530 run_bootstrap() {
 531
 532   # I got no frigging clue what this does, ask Sam Saffron. It RUNS STUFF ON THE HOST I GUESS?
 533   host_run
 534
 535   # Is the image available?
 536   # If not, pull it here so the user is aware what's happening.
 537   $docker_path history $image >/dev/null 2>&1 || $docker_path pull $image
 538
 539   set_template_info
 540
 541   base_image=`cat $config_file | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \
 542     "require 'yaml'; puts YAML.load(STDIN.readlines.join)['base_image']"`
 543
 544   update_pups=`cat $config_file | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \
 545     "require 'yaml'; puts YAML.load(STDIN.readlines.join)['update_pups']"`
 546
 547   if [[ ! X"" = X"$base_image" ]]; then
 548     image=$base_image
 549   fi
 550
 551   set_volumes
 552   set_links
 553
 554   rm -f $cidbootstrap
 555
 556   run_command="cd /pups &&"
 557   if [[ ! "false" =  $update_pups ]]; then
 558     run_command="$run_command git pull &&"
 559   fi
 560   run_command="$run_command /pups/bin/pups --stdin"
 561
 562   echo $run_command
 563
 564   unset ERR
 565   (exec echo "$input" | $docker_path run $user_args $links "${env[@]}" -e DOCKER_HOST_IP=$docker_ip --cidfile $cidbootstrap -i -a stdin -a stdout -a stderr $volumes $image \
 566     /bin/bash -c "$run_command") || ERR=$?
 567
 568   unset FAILED
 569   # magic exit code that indicates a retry
 570   if [[ "$ERR" == 77 ]]; then
 571     $docker_path rm `cat $cidbootstrap`
 572     rm $cidbootstrap
 573     exit 77
 574   elif [[ "$ERR" > 0 ]]; then
 575     FAILED=TRUE
 576   fi
 577
 578   if [[ $FAILED = "TRUE" ]]; then
 579     if [[ ! -z "$DEBUG" ]]; then
 580       $docker_path commit `cat $cidbootstrap` $local_discourse/$config-debug || echo 'FAILED TO COMMIT'
 581       echo "** DEBUG ** Maintaining image for diagnostics $local_discourse/$config-debug"
 582     fi
 583
 584     $docker_path rm `cat $cidbootstrap`
 585     rm $cidbootstrap
 586     echo "** FAILED TO BOOTSTRAP ** please scroll up and look for earlier error messages, there may be more than one"
 587     exit 1
 588   fi
 589
 590   sleep 5
 591
 592   $docker_path commit `cat $cidbootstrap` $local_discourse/$config || echo 'FAILED TO COMMIT'
 593   $docker_path rm `cat $cidbootstrap` && rm $cidbootstrap
 594 }
 595
 596
 597
 598 case "$command" in
 599   bootstrap)
 600       run_bootstrap
 601       echo "Successfully bootstrapped, to startup use ./launcher start $config"
 602       exit 0
 603       ;;
 604
 605   enter)
 606       exec $docker_path exec -it $config /bin/bash --login
 607       ;;
 608
 609   stop)
 610       run_stop
 611       exit 0
 612       ;;
 613
 614   logs)
 615
 616       $docker_path logs $config
 617       exit 0
 618       ;;
 619
 620   restart)
 621       run_stop
 622       run_start
 623       exit 0
 624       ;;
 625
 626   start)
 627       run_start
 628       exit 0
 629       ;;
 630
 631   rebuild)
 632       if [ "$(git symbolic-ref --short HEAD)" == "master" ]; then
 633         echo "Ensuring launcher is up to date"
 634
 635         git remote update
 636
 637         LOCAL=$(git rev-parse @)
 638         REMOTE=$(git rev-parse @{u})
 639         BASE=$(git merge-base @ @{u})
 640
 641         if [ $LOCAL = $REMOTE ]; then
 642           echo "Launcher is up-to-date"
 643
 644         elif [ $LOCAL = $BASE ]; then
 645           echo "Updating Launcher"
 646           git pull || (echo 'failed to update' && exit 1)
 647           exec /bin/bash $0 $@
 648
 649         elif [ $REMOTE = $BASE ]; then
 650           echo "Your version of Launcher is ahead of origin"
 651
 652         else
 653           echo "Launcher has diverged source, this is only expected in Dev mode"
 654         fi
 655
 656       fi
 657
 658       set_existing_container
 659
 660       if [ ! -z $existing ]
 661         then
 662           echo "Stopping old container"
 663           (
 664             set -x
 665             $docker_path stop -t 10 $config
 666           )
 667       fi
 668
 669       run_bootstrap
 670
 671       if [ ! -z $existing ]
 672         then
 673           echo "Removing old container"
 674           (
 675             set -x
 676             $docker_path rm $config
 677           )
 678       fi
 679
 680       run_start
 681       exit 0
 682       ;;
 683
 684
 685   destroy)
 686       (set -x; $docker_path stop -t 10 $config && $docker_path rm $config) || (echo "$config was not found" && exit 0)
 687       exit 0
 688       ;;
 689 esac
 690
 691 usage