launcher

   1 #!/bin/bash
   2
   3 command=$1
   4 config=$2
   5 opt=$3
   6
   7 # Docker doesn't like uppercase characters, spaces or special characters, catch it now before we build everything out and then find out
   8 re='[A-Z/ !@#$%^&*()+~`=]'
   9 if [[ $config =~ $re ]];
  10   then
  11     echo
  12     echo "ERROR: Config name must not contain upper case characters, spaces or special characters. Correct config name and rerun $0."
  13     echo
  14     exit 1
  15 fi
  16
  17 cd "$(dirname "$0")"
  18
  19 docker_min_version='1.6.0'
  20 docker_rec_version='1.6.0'
  21
  22 config_file=containers/"$config".yml
  23 cidbootstrap=cids/"$config"_bootstrap.cid
  24 local_discourse=local_discourse
  25 image=discourse/discourse:1.0.16
  26 docker_path=`which docker.io || which docker`
  27
  28
  29 if [ "${SUPERVISED}" = "true" ]; then
  30   restart_policy="--restart=no"
  31   attach_on_start="-a"
  32   attach_on_run="-a stdout -a stderr"
  33 else
  34   attach_on_run="-d"
  35 fi
  36
  37 if [ -n "$DOCKER_HOST" ]; then
  38   docker_ip=`sed -e 's/^tcp:\/\/\(.*\):.*$/\1/' <<< "$DOCKER_HOST"`
  39 elif [ -x "$(which ip 2>/dev/null)" ]; then
  40   docker_ip=`ip addr show docker0 | \
  41                   grep 'inet ' | \
  42                   awk '{ split($2,a,"/"); print a[1] }';`
  43 else
  44   docker_ip=`ifconfig | \
  45                   grep -B1 "inet addr" | \
  46                   awk '{ if ( $1 == "inet" ) { print $2 } else if ( $2 == "Link" ) { printf "%s:" ,$1 } }' | \
  47                   grep docker0 | \
  48                   awk -F: '{ print $3 }';`
  49 fi
  50
  51
  52 usage () {
  53   echo "Usage: launcher COMMAND CONFIG [--skip-prereqs]"
  54   echo "Commands:"
  55   echo "    start:      Start/initialize a container"
  56   echo "    stop:       Stop a running container"
  57   echo "    restart:    Restart a container"
  58   echo "    destroy:    Stop and remove a container"
  59   echo "    enter:      Use nsenter to enter a container"
  60   echo "    ssh:        Start a bash shell in a running container"
  61   echo "    logs:       Docker logs for container"
  62   echo "    bootstrap:  Bootstrap a container for the config based on a template"
  63   echo "    rebuild:    Rebuild a container (destroy old, bootstrap, start new)"
  64   echo "    cleanup:    Remove all containers that have stopped for > 24 hours"
  65   echo
  66   echo "Options:"
  67   echo "    --skip-prereqs   Don't check prerequisites or resource requirements"
  68   echo "    --docker-args    Extra arguments to pass when running docker"
  69   exit 1
  70 }
  71
  72 compare_version() {
  73     declare -a ver_a
  74     declare -a ver_b
  75     IFS=. read -a ver_a <<< "$1"
  76     IFS=. read -a ver_b <<< "$2"
  77
  78     while [[ -n $ver_a ]]; do
  79         if (( ver_a > ver_b )); then
  80             return 0
  81         elif (( ver_b > ver_a )); then
  82             return 1
  83         else
  84             unset ver_a[0]
  85             ver_a=("${ver_a[@]}")
  86             unset ver_b[0]
  87             ver_b=("${ver_b[@]}")
  88         fi
  89     done
  90     return 1  # They are equal
  91 }
  92
  93 prereqs() {
  94
  95   # 1. docker daemon running?
  96   # we send stderr to /dev/null cause we don't care about warnings,
  97   # it usually complains about swap which does not matter
  98   test=`$docker_path info 2> /dev/null`
  99
 100   if [[ $? -ne 0 ]] ; then
 101     echo "Cannot connect to the docker daemon - verify it is running and you have access"
 102     exit 1
 103   fi
 104
 105   # 2. running aufs or btrfs
 106   test=`$docker_path info 2> /dev/null | grep 'Driver: '`
 107   if [[ "$test" =~ [aufs|btrfs|zfs|overlay] ]] ; then : ; else
 108     echo "Your Docker installation is not using a supported filesystem if we were to proceed you may have a broken install."
 109     echo "aufs is the recommended filesystem you should be using (zfs/btrfs and overlay may work as well)"
 110     echo "You can tell what filesystem you are using by running \"docker info\" and looking at the driver"
 111     echo ""
 112     echo "If you wish to continue anyway using your existing unsupported filesystem"
 113     echo "read the source code of launcher and figure out how to bypass this."
 114     exit 1
 115   fi
 116
 117   # 3. running recommended docker version
 118   test=($($docker_path --version))  # Get docker version string
 119   test=${test[2]//,/}  # Get version alone and strip comma if exists
 120
 121   [[ "$test" =~ "0.12.0" ]] && echo "You are running a broken version of Docker, please upgrade ASAP. See: https://meta.discourse.org/t/the-installation-stopped-in-the-middle/16311/ for more details." && exit 1
 122
 123   # At least minimum version
 124   if compare_version "${docker_min_version}" "${test}"; then
 125     echo "ERROR: Docker version ${test} not supported, please upgrade to at least ${docker_min_version}, or recommended ${docker_rec_version}"
 126     exit 1
 127   fi
 128
 129   # Recommend best version
 130   if compare_version "${docker_rec_version}" "${test}"; then
 131     echo "WARNING: Docker version ${test} deprecated, recommend upgrade to ${docker_rec_version} or newer."
 132   fi
 133
 134   # 4. discourse docker image is downloaded
 135   test=`$docker_path images | awk '{print $1 ":" $2 }' | grep "$image"`
 136
 137   if [ -z "$test" ]; then
 138     echo
 139     echo "WARNING: We are about to start downloading the Discourse base image"
 140     echo "This process may take anywhere between a few minutes to an hour, depending on your network speed"
 141     echo
 142     echo "Please be patient"
 143     echo
 144
 145   fi
 146
 147   # 5. able to attach stderr / out / tty
 148   test=`$docker_path run $user_args -i --rm -a stdout -a stderr $image echo working`
 149   if [[ "$test" =~ "working" ]] ; then : ; else
 150     echo "Your Docker installation is not working correctly"
 151     echo
 152     echo "See: https://meta.discourse.org/t/docker-error-on-bootstrap/13657/18?u=sam"
 153     exit 1
 154   fi
 155
 156 }
 157
 158 check_resources() {
 159   # Memory
 160   resources="ok"
 161   avail_mem="$(LANG=C free -m | grep '^Mem:' | awk '{print $2}')"
 162   if [ "$avail_mem" -lt 900 ]; then
 163     resources="insufficient"
 164     echo "WARNING: You do not appear to have sufficient memory to run Discourse."
 165     echo
 166     echo "Your system may not work properly, or future upgrades of Discourse may"
 167     echo "not complete successfully."
 168     echo
 169     echo "See https://github.com/discourse/discourse/blob/master/docs/INSTALL-cloud.md#create-new-cloud-server"
 170   elif [ "$avail_mem" -lt 1800 ]; then
 171     total_swap="$(LANG=C free -m | grep ^Swap: | awk '{print $2}')"
 172     if [ "$total_swap" -lt 1000 ]; then
 173       resources="insufficient"
 174       echo "WARNING: You must have at least 1GB of swap when running with less"
 175       echo "than 2GB of RAM."
 176       echo
 177       echo "Your system may not work properly, or future upgrades of Discourse may"
 178       echo "not complete successfully."
 179       echo
 180       echo "See https://github.com/discourse/discourse/blob/master/docs/INSTALL-cloud.md#set-up-swap-if-needed"
 181     fi
 182   fi
 183
 184   # Disk space
 185   free_disk="$(df /var | tail -n 1 | awk '{print $4}')"
 186   if [ "$free_disk" -lt 5000 ]; then
 187     resources="insufficient"
 188     echo "WARNING: You must have at least 5GB of *free* disk space to run Discourse."
 189     echo
 190     echo "Insufficient disk space may result in problems running your site, and may"
 191     echo "not even allow Discourse installation to complete successfully."
 192     echo
 193     echo "Please free up some space, or expand your disk, before continuing."
 194     exit 1
 195   fi
 196
 197   if [ -t 0 ] && [ "$resources" != "ok" ]; then
 198     echo
 199     read -p "Press ENTER to continue, or Ctrl-C to exit and give your system more resources"
 200   fi
 201 }
 202
 203 check_ports() {
 204   local valid=$(netstat -tln | awk '{print $4}' | grep ":${1}\$")
 205
 206   if [ -n "$valid" ]; then
 207     echo "Launcher has detected that port ${1} is in use."
 208     echo ""
 209     echo "If you are trying to run Discourse simultaneously with another web server like Apache or nginx, you will need to bind to a different port."
 210     echo "See https://meta.discourse.org/t/17247 for help."
 211     exit 1
 212   fi
 213 }
 214
 215 if [ "$opt" != "--skip-prereqs" ] ; then
 216   prereqs
 217 fi
 218
 219 if [ "$opt" == "--docker-args" ] ; then
 220   user_args=$4
 221 else
 222   user_args=""
 223 fi
 224
 225 get_ssh_pub_key() {
 226   local ${ssh_key_locations}
 227   ssh_key_locations=(
 228     ~/.ssh/id_ed25519.pub
 229     ~/.ssh/id_ecdsa.pub
 230     ~/.ssh/id_rsa.pub
 231     ~/.ssh/id_dsa.pub
 232     ~core/.ssh/authorized_keys
 233   )
 234
 235   local $keyfile
 236   for keyfile in "${ssh_key_locations[@]}"; do
 237     if [[ -e ${keyfile} ]] ; then
 238       ssh_pub_key="$(cat ${keyfile})"
 239       return 0
 240     fi
 241   done
 242
 243   return 0
 244 }
 245
 246
 247 install_docker() {
 248
 249   echo "Docker is not installed, you will need to install Docker in order to run Discourse"
 250   echo "Please visit https://docs.docker.com/installation/ for instructions on how to do this for your system"
 251   echo
 252   echo "If you are running Ubuntu Trusty or later, you can try the following:"
 253   echo
 254
 255   echo "sudo apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D"
 256   echo "sudo sh -c \"echo deb https://apt.dockerproject.org/repo ubuntu-$(lsb_release -sc) main > /etc/apt/sources.list.d/docker.list\""
 257   echo "sudo apt-get update"
 258   echo "sudo apt-get install docker-engine"
 259
 260   exit 1
 261 }
 262
 263 host_run() {
 264   read -r -d '' env_ruby << 'RUBY'
 265   require 'yaml'
 266
 267   input = STDIN.readlines.join
 268   yaml = YAML.load(input)
 269
 270   if host_run = yaml['host_run']
 271     params = yaml['params'] || {}
 272     host_run.each do |run|
 273       params.each do |k,v|
 274         run = run.gsub("$#{k}", v)
 275       end
 276       STDOUT.write "#{run}--SEP--"
 277     end
 278   end
 279 RUBY
 280
 281   host_run=`cat $config_file | $docker_path run $user_args --rm -i -a stdout -a stdin $image ruby -e "$env_ruby"`
 282
 283   while [ "$host_run" ] ; do
 284     iter=${host_run%%--SEP--*}
 285     echo
 286     echo "Host run: $iter"
 287     $iter || exit 1
 288     echo
 289     host_run="${host_run#*--SEP--}"
 290   done
 291 }
 292
 293
 294 set_volumes() {
 295   volumes=`cat $config_file | $docker_path run $user_args --rm -i -a stdout -a stdin $image ruby -e \
 296         "require 'yaml'; puts YAML.load(STDIN.readlines.join)['volumes'].map{|v| '-v ' << v['volume']['host'] << ':' << v['volume']['guest'] << ' '}.join"`
 297 }
 298
 299 set_links() {
 300     links=`cat $config_file | $docker_path run $user_args --rm -i -a stdout -a stdin $image ruby -e \
 301         "require 'yaml'; puts YAML.load(STDIN.readlines.join)['links'].map{|l| '--link ' << l['link']['name'] << ':' << l['link']['alias'] << ' '}.join"`
 302 }
 303
 304 set_template_info() {
 305
 306     templates=`cat $config_file | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \
 307       "require 'yaml'; puts YAML.load(STDIN.readlines.join)['templates']"`
 308
 309
 310     arrTemplates=(${templates// / })
 311     config_data=$(cat $config_file)
 312
 313     input="hack: true"
 314
 315
 316     for template in "${arrTemplates[@]}"
 317     do
 318       [ ! -z $template ] && {
 319         input="$input _FILE_SEPERATOR_ $(cat $template)"
 320       }
 321     done
 322
 323     # we always want our config file last so it takes priority
 324     input="$input _FILE_SEPERATOR_ $config_data"
 325
 326     read -r -d '' env_ruby << 'RUBY'
 327     require 'yaml'
 328
 329     input=STDIN.readlines.join
 330     # default to UTF-8 for the dbs sake
 331     env = {'LANG' => 'en_US.UTF-8'}
 332     input.split('_FILE_SEPERATOR_').each do |yml|
 333        yml.strip!
 334        begin
 335          env.merge!(YAML.load(yml)['env'] || {})
 336        rescue Psych::SyntaxError => e
 337         puts e
 338         puts "*ERROR."
 339        rescue => e
 340         puts yml
 341         p e
 342        end
 343     end
 344     # strip out blank env
 345     env.delete_if{|k,v| "#{v}".empty?}
 346     puts env.map{|k,v| "-e\n#{k}=#{v}" }.join("\n")
 347 RUBY
 348
 349     raw=`exec echo "$input" | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e "$env_ruby"`
 350
 351     env=()
 352     ok=1
 353     while read i; do
 354       if [ "$i" == "*ERROR." ]; then
 355         ok=0
 356       elif [ -n "$i" ]; then
 357         env[${#env[@]}]=$i
 358       fi
 359     done <<< "$raw"
 360
 361     if [ "$ok" -ne 1 ]; then
 362       echo "${env[@]}"
 363       echo "YAML syntax error. Please check your /var/discourse/containers/*.yml config files."
 364       exit 1
 365     fi
 366 }
 367
 368 [ -z $docker_path ] && {
 369   install_docker
 370 }
 371
 372 [ "$command" == "cleanup" ] && {
 373   echo
 374   echo "The following command will"
 375   echo "- Delete all docker images for old containers"
 376   echo "- Delete all stopped and orphan containers"
 377   echo
 378   read -p "Are you sure (Y/n): " -n 1 -r && echo
 379   if [[ $REPLY =~ ^[Yy]$ || ! $REPLY ]]
 380     then
 381       space=$(df /var/lib/docker | awk '{ print $4 }' | grep -v Available)
 382       echo "Starting Cleanup (bytes free $space)"
 383
 384       STATE_DIR=./.gc-state scripts/docker-gc
 385
 386       space=$(df /var/lib/docker | awk '{ print $4 }' | grep -v Available)
 387       echo "Finished Cleanup (bytes free $space)"
 388
 389     else
 390       exit 1
 391   fi
 392   exit 0
 393 }
 394
 395 [ $# -lt 2 ] && {
 396   usage
 397 }
 398
 399 if [ ! -e $config_file ]
 400   then
 401     echo "Config file was not found, ensure $config_file exists"
 402     echo ""
 403     echo "Available configs ( `cd containers && ls -dm *.yml | tr -s '\n' ' ' | awk '{ gsub(/\.yml/, ""); print }'`)"
 404     exit 1
 405 fi
 406
 407
 408 docker_version=($($docker_path --version))
 409 docker_version=${test[2]//,/}
 410
 411 if compare_version "1.2.0" "$docker_version"; then
 412   echo "We recommend you upgrade docker, the version you are running has no restart policies, on reboot your container may not start up"
 413   restart_policy=""
 414 else
 415   restart_policy=${restart_policy:---restart=always}
 416 fi
 417
 418 set_existing_container(){
 419   existing=`$docker_path ps -a | awk '{ print $1, $(NF) }' | grep " $config$" | awk '{ print $1 }'`
 420 }
 421
 422 run_stop(){
 423
 424   set_existing_container
 425
 426   if [ ! -z $existing ]
 427      then
 428        (
 429         set -x
 430         $docker_path stop -t 10 $config
 431        )
 432      else
 433        echo "$config was not started !"
 434        exit 1
 435   fi
 436 }
 437
 438 set_run_image() {
 439   run_image=`cat $config_file | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \
 440     "require 'yaml'; puts YAML.load(STDIN.readlines.join)['run_image']"`
 441
 442   if [ -z "$run_image" ]; then
 443     run_image="$local_discourse/$config"
 444   fi
 445 }
 446
 447 set_boot_command() {
 448   boot_command=`cat $config_file | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \
 449     "require 'yaml'; puts YAML.load(STDIN.readlines.join)['boot_command']"`
 450
 451   if [ -z "$boot_command" ]; then
 452
 453     no_boot_command=`cat $config_file | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \
 454       "require 'yaml'; puts YAML.load(STDIN.readlines.join)['no_boot_command']"`
 455
 456     if [ -z "$no_boot_command" ]; then
 457       boot_command="/sbin/boot"
 458     fi
 459   fi
 460 }
 461
 462 run_start(){
 463
 464    existing=`$docker_path ps | awk '{ print $1, $(NF) }' | grep " $config$" | awk '{ print $1 }'`
 465    echo $existing
 466    if [ ! -z $existing ]
 467    then
 468      echo "Nothing to do, your container has already started!"
 469      exit 1
 470    fi
 471
 472    existing=`$docker_path ps -a | awk '{ print $1, $(NF) }' | grep " $config$" | awk '{ print $1 }'`
 473    if [ ! -z $existing ]
 474    then
 475      echo "starting up existing container"
 476      (
 477        set -x
 478        $docker_path start $config
 479      )
 480      exit 0
 481    fi
 482
 483    host_run
 484    ports=`cat $config_file | $docker_path run $user_args --rm -i -a stdout -a stdin $image ruby -e \
 485           "require 'yaml'; puts YAML.load(STDIN.readlines.join)['expose'].map{|p| \"-p #{p}\"}.join(' ')"`
 486
 487    IFS='-p ' read -a array <<< "$ports"
 488    for element in "${array[@]}"
 489    do
 490      IFS=':' read -a args <<< "$element"
 491      if [ "${#args[@]}" == "2" ]; then
 492        check_ports "${args[0]}"
 493      elif [ "${#args[@]}" == "3" ]; then
 494        check_ports "${args[1]}"
 495      fi
 496    done
 497
 498    docker_args=`cat $config_file | $docker_path run $user_args --rm -i -a stdout -a stdin $image ruby -e \
 499           "require 'yaml'; puts YAML.load(STDIN.readlines.join)['docker_args']"`
 500
 501    set_template_info
 502    set_volumes
 503    set_links
 504    set_run_image
 505    set_boot_command
 506
 507    (
 508      hostname=`hostname`
 509      set -x
 510      $docker_path run $user_args $links $attach_on_run $restart_policy "${env[@]}" -h "$hostname-$config" \
 511         -e DOCKER_HOST_IP=$docker_ip --name $config -t $ports $volumes $docker_args $run_image $boot_command
 512
 513    )
 514    exit 0
 515
 516 }
 517
 518 run_bootstrap(){
 519
 520   if [ "$opt" != "--skip-prereqs" ] ; then
 521     check_resources
 522   fi
 523
 524   host_run
 525
 526   get_ssh_pub_key
 527
 528   # Is the image available?
 529   # If not, pull it here so the user is aware what's happening.
 530   $docker_path history $image >/dev/null 2>&1 || $docker_path pull $image
 531
 532   set_template_info
 533
 534   base_image=`cat $config_file | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \
 535     "require 'yaml'; puts YAML.load(STDIN.readlines.join)['base_image']"`
 536
 537   update_pups=`cat $config_file | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \
 538     "require 'yaml'; puts YAML.load(STDIN.readlines.join)['update_pups']"`
 539
 540   if [[ ! X"" = X"$base_image" ]]; then
 541     image=$base_image
 542   fi
 543
 544   set_volumes
 545   set_links
 546
 547   rm -f $cidbootstrap
 548
 549   run_command="cd /pups &&"
 550   if [[ ! "false" =  $update_pups ]]; then
 551     run_command="$run_command git pull &&"
 552   fi
 553   run_command="$run_command /pups/bin/pups --stdin"
 554
 555   echo $run_command
 556
 557   env=("${env[@]}" "-e" "SSH_PUB_KEY=$ssh_pub_key")
 558
 559   (exec echo "$input" | $docker_path run $user_args $links "${env[@]}" -e DOCKER_HOST_IP=$docker_ip --cidfile $cidbootstrap -i -a stdin -a stdout -a stderr $volumes $image \
 560      /bin/bash -c "$run_command") \
 561      || ($docker_path rm `cat $cidbootstrap` && rm $cidbootstrap)
 562
 563   [ ! -e $cidbootstrap ] && echo "** FAILED TO BOOTSTRAP ** please scroll up and look for earlier error messages, there may be more than one" && exit 1
 564
 565   sleep 5
 566
 567   $docker_path commit `cat $cidbootstrap` $local_discourse/$config || echo 'FAILED TO COMMIT'
 568   $docker_path rm `cat $cidbootstrap` && rm $cidbootstrap
 569 }
 570
 571
 572
 573 case "$command" in
 574   bootstrap)
 575       run_bootstrap
 576       echo "Successfully bootstrapped, to startup use ./launcher start $config"
 577       exit 0
 578       ;;
 579
 580   enter)
 581       exec $docker_path exec -it $config /bin/bash
 582       ;;
 583
 584   ssh)
 585        existing=`$docker_path ps | awk '{ print $1, $(NF) }' | grep " $config$" | awk '{ print $1 }'`
 586
 587        if [[ ! -z $existing ]]; then
 588          address="`$docker_path port $config 22`"
 589          split=(${address//:/ })
 590          exec ssh -o StrictHostKeyChecking=no root@${split[0]} -p ${split[1]}
 591        else
 592          echo "$config is not running!"
 593          exit 1
 594        fi
 595       ;;
 596
 597   stop)
 598       run_stop
 599       exit 0
 600       ;;
 601
 602   logs)
 603
 604       $docker_path logs $config
 605       exit 0
 606       ;;
 607
 608   restart)
 609       run_stop
 610       run_start
 611       exit 0
 612       ;;
 613
 614   start)
 615       run_start
 616       exit 0
 617       ;;
 618
 619   rebuild)
 620       if [ "$(git symbolic-ref --short HEAD)" == "master" ]; then
 621         echo "Ensuring discourse docker is up to date"
 622
 623         git remote update
 624
 625         LOCAL=$(git rev-parse @)
 626         REMOTE=$(git rev-parse @{u})
 627         BASE=$(git merge-base @ @{u})
 628
 629         if [ $LOCAL = $REMOTE ]; then
 630           echo "Discourse Docker is up-to-date"
 631
 632         elif [ $LOCAL = $BASE ]; then
 633           echo "Updating Discourse Docker"
 634           git pull || (echo 'failed to update' && exit 1)
 635           exec /bin/bash $0 $@
 636
 637         elif [ $REMOTE = $BASE ]; then
 638           echo "Your version of Discourse Docker is ahead of origin"
 639
 640         else
 641           echo "Discourse Docker has diverged source, this is only expected in Dev mode"
 642         fi
 643
 644       fi
 645
 646       set_existing_container
 647
 648       if [ ! -z $existing ]
 649         then
 650           echo "Stopping old container"
 651           (
 652             set -x
 653             $docker_path stop -t 10 $config
 654           )
 655       fi
 656
 657       run_bootstrap
 658
 659       if [ ! -z $existing ]
 660         then
 661           echo "Removing old container"
 662           (
 663             set -x
 664             $docker_path rm $config
 665           )
 666       fi
 667
 668       run_start
 669       exit 0
 670       ;;
 671
 672
 673   destroy)
 674       (set -x; $docker_path stop -t 10 $config && $docker_path rm $config) || (echo "$config was not found" && exit 0)
 675       exit 0
 676       ;;
 677 esac
 678
 679 usage