launcher

   1 #!/bin/bash
   2
   3 command=$1
   4 config=$2
   5 opt=$3
   6
   7 # Docker doesn't like uppercase characters, spaces or special characters, catch it now before we build everything out and then find out
   8 re='[A-Z/ !@#$%^&*()+~`=]'
   9 if [[ $config =~ $re ]];
  10   then
  11     echo
  12     echo "ERROR: Config name must not contain upper case characters, spaces or special characters. Correct config name and rerun $0."
  13     echo
  14     exit 1
  15 fi
  16
  17 cd "$(dirname "$0")"
  18
  19 docker_min_version='1.6.0'
  20 docker_rec_version='1.6.0'
  21
  22 config_file=containers/"$config".yml
  23 cidbootstrap=cids/"$config"_bootstrap.cid
  24 local_discourse=local_discourse
  25 image=discourse/discourse:1.0.17
  26 docker_path=`which docker.io || which docker`
  27
  28
  29 if [ "${SUPERVISED}" = "true" ]; then
  30   restart_policy="--restart=no"
  31   attach_on_start="-a"
  32   attach_on_run="-a stdout -a stderr"
  33 else
  34   attach_on_run="-d"
  35 fi
  36
  37 if [ -n "$DOCKER_HOST" ]; then
  38   docker_ip=`sed -e 's/^tcp:\/\/\(.*\):.*$/\1/' <<< "$DOCKER_HOST"`
  39 elif [ -x "$(which ip 2>/dev/null)" ]; then
  40   docker_ip=`ip addr show docker0 | \
  41                   grep 'inet ' | \
  42                   awk '{ split($2,a,"/"); print a[1] }';`
  43 else
  44   docker_ip=`ifconfig | \
  45                   grep -B1 "inet addr" | \
  46                   awk '{ if ( $1 == "inet" ) { print $2 } else if ( $2 == "Link" ) { printf "%s:" ,$1 } }' | \
  47                   grep docker0 | \
  48                   awk -F: '{ print $3 }';`
  49 fi
  50
  51
  52 usage () {
  53   echo "Usage: launcher COMMAND CONFIG [--skip-prereqs]"
  54   echo "Commands:"
  55   echo "    start:      Start/initialize a container"
  56   echo "    stop:       Stop a running container"
  57   echo "    restart:    Restart a container"
  58   echo "    destroy:    Stop and remove a container"
  59   echo "    enter:      Use nsenter to enter a container"
  60   echo "    ssh:        Start a bash shell in a running container"
  61   echo "    logs:       Docker logs for container"
  62   echo "    bootstrap:  Bootstrap a container for the config based on a template"
  63   echo "    rebuild:    Rebuild a container (destroy old, bootstrap, start new)"
  64   echo "    cleanup:    Remove all containers that have stopped for > 24 hours"
  65   echo
  66   echo "Options:"
  67   echo "    --skip-prereqs   Don't check prerequisites or resource requirements"
  68   echo "    --docker-args    Extra arguments to pass when running docker"
  69   exit 1
  70 }
  71
  72 compare_version() {
  73     declare -a ver_a
  74     declare -a ver_b
  75     IFS=. read -a ver_a <<< "$1"
  76     IFS=. read -a ver_b <<< "$2"
  77
  78     while [[ -n $ver_a ]]; do
  79         if (( ver_a > ver_b )); then
  80             return 0
  81         elif (( ver_b > ver_a )); then
  82             return 1
  83         else
  84             unset ver_a[0]
  85             ver_a=("${ver_a[@]}")
  86             unset ver_b[0]
  87             ver_b=("${ver_b[@]}")
  88         fi
  89     done
  90     return 1  # They are equal
  91 }
  92
  93 prereqs() {
  94
  95   # 1. docker daemon running?
  96   # we send stderr to /dev/null cause we don't care about warnings,
  97   # it usually complains about swap which does not matter
  98   test=`$docker_path info 2> /dev/null`
  99
 100   if [[ $? -ne 0 ]] ; then
 101     echo "Cannot connect to the docker daemon - verify it is running and you have access"
 102     exit 1
 103   fi
 104
 105   # 2. running aufs or btrfs
 106   test=`$docker_path info 2> /dev/null | grep 'Driver: '`
 107   if [[ "$test" =~ [aufs|btrfs|zfs|overlay] ]] ; then : ; else
 108     echo "Your Docker installation is not using a supported filesystem if we were to proceed you may have a broken install."
 109     echo "aufs is the recommended filesystem you should be using (zfs/btrfs and overlay may work as well)"
 110     echo "You can tell what filesystem you are using by running \"docker info\" and looking at the driver"
 111     echo ""
 112     echo "If you wish to continue anyway using your existing unsupported filesystem"
 113     echo "read the source code of launcher and figure out how to bypass this."
 114     exit 1
 115   fi
 116
 117   # 3. running recommended docker version
 118   test=($($docker_path --version))  # Get docker version string
 119   test=${test[2]//,/}  # Get version alone and strip comma if exists
 120
 121   [[ "$test" =~ "0.12.0" ]] && echo "You are running a broken version of Docker, please upgrade ASAP. See: https://meta.discourse.org/t/the-installation-stopped-in-the-middle/16311/ for more details." && exit 1
 122
 123   # At least minimum version
 124   if compare_version "${docker_min_version}" "${test}"; then
 125     echo "ERROR: Docker version ${test} not supported, please upgrade to at least ${docker_min_version}, or recommended ${docker_rec_version}"
 126     exit 1
 127   fi
 128
 129   # Recommend best version
 130   if compare_version "${docker_rec_version}" "${test}"; then
 131     echo "WARNING: Docker version ${test} deprecated, recommend upgrade to ${docker_rec_version} or newer."
 132   fi
 133
 134   # 4. discourse docker image is downloaded
 135   test=`$docker_path images | awk '{print $1 ":" $2 }' | grep "$image"`
 136
 137   if [ -z "$test" ]; then
 138     echo
 139     echo "WARNING: We are about to start downloading the Discourse base image"
 140     echo "This process may take anywhere between a few minutes to an hour, depending on your network speed"
 141     echo
 142     echo "Please be patient"
 143     echo
 144
 145   fi
 146
 147   # 5. able to attach stderr / out / tty
 148   test=`$docker_path run $user_args -i --rm -a stdout -a stderr $image echo working`
 149   if [[ "$test" =~ "working" ]] ; then : ; else
 150     echo "Your Docker installation is not working correctly"
 151     echo
 152     echo "See: https://meta.discourse.org/t/docker-error-on-bootstrap/13657/18?u=sam"
 153     exit 1
 154   fi
 155
 156 }
 157
 158 check_resources() {
 159   # Memory
 160   resources="ok"
 161   avail_mem="$(LANG=C free -m | grep '^Mem:' | awk '{print $2}')"
 162   if [ "$avail_mem" -lt 900 ]; then
 163     resources="insufficient"
 164     echo "WARNING: You do not appear to have sufficient memory to run Discourse."
 165     echo
 166     echo "Your system may not work properly, or future upgrades of Discourse may"
 167     echo "not complete successfully."
 168     echo
 169     echo "See https://github.com/discourse/discourse/blob/master/docs/INSTALL-cloud.md#create-new-cloud-server"
 170   elif [ "$avail_mem" -lt 1800 ]; then
 171     total_swap="$(LANG=C free -m | grep ^Swap: | awk '{print $2}')"
 172     if [ "$total_swap" -lt 1000 ]; then
 173       resources="insufficient"
 174       echo "WARNING: You must have at least 1GB of swap when running with less"
 175       echo "than 2GB of RAM."
 176       echo
 177       echo "Your system may not work properly, or future upgrades of Discourse may"
 178       echo "not complete successfully."
 179       echo
 180       echo "See https://github.com/discourse/discourse/blob/master/docs/INSTALL-cloud.md#set-up-swap-if-needed"
 181     fi
 182   fi
 183
 184   # Disk space
 185   free_disk="$(df /var | tail -n 1 | awk '{print $4}')"
 186   if [ "$free_disk" -lt 5000 ]; then
 187     resources="insufficient"
 188     echo "WARNING: You must have at least 5GB of *free* disk space to run Discourse."
 189     echo
 190     echo "Insufficient disk space may result in problems running your site, and may"
 191     echo "not even allow Discourse installation to complete successfully."
 192     echo
 193     echo "Please free up some space, or expand your disk, before continuing."
 194     exit 1
 195   fi
 196
 197   if [ -t 0 ] && [ "$resources" != "ok" ]; then
 198     echo
 199     read -p "Press ENTER to continue, or Ctrl-C to exit and give your system more resources"
 200   fi
 201 }
 202
 203 check_ports() {
 204   local valid=$(netstat -tln | awk '{print $4}' | grep ":${1}\$")
 205
 206   if [ -n "$valid" ]; then
 207     echo "Launcher has detected that port ${1} is in use."
 208     echo ""
 209     echo "If you are trying to run Discourse simultaneously with another web server like Apache or nginx, you will need to bind to a different port."
 210     echo "See https://meta.discourse.org/t/17247 for help."
 211     exit 1
 212   fi
 213 }
 214
 215 if [ "$opt" != "--skip-prereqs" ] ; then
 216   prereqs
 217 fi
 218
 219 if [ "$opt" == "--docker-args" ] ; then
 220   user_args=$4
 221 else
 222   user_args=""
 223 fi
 224
 225 get_ssh_pub_key() {
 226   local ${ssh_key_locations}
 227   ssh_key_locations=(
 228     ~/.ssh/id_ed25519.pub
 229     ~/.ssh/id_ecdsa.pub
 230     ~/.ssh/id_rsa.pub
 231     ~/.ssh/id_dsa.pub
 232     ~core/.ssh/authorized_keys
 233   )
 234
 235   local $keyfile
 236   for keyfile in "${ssh_key_locations[@]}"; do
 237     if [[ -e ${keyfile} ]] ; then
 238       ssh_pub_key="$(cat ${keyfile})"
 239       return 0
 240     fi
 241   done
 242
 243   return 0
 244 }
 245
 246
 247 install_docker() {
 248
 249   echo "Docker is not installed, you will need to install Docker in order to run Discourse"
 250   echo "Please visit https://docs.docker.com/installation/ for instructions on how to do this for your system"
 251   echo
 252   echo "If you are running Ubuntu Trusty or later, you can try the following:"
 253   echo
 254
 255   echo "sudo apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D"
 256   echo "sudo sh -c \"echo deb https://apt.dockerproject.org/repo ubuntu-$(lsb_release -sc) main > /etc/apt/sources.list.d/docker.list\""
 257   echo "sudo apt-get update"
 258   echo "sudo apt-get install docker-engine"
 259
 260   exit 1
 261 }
 262
 263 host_run() {
 264   read -r -d '' env_ruby << 'RUBY'
 265   require 'yaml'
 266
 267   input = STDIN.readlines.join
 268   yaml = YAML.load(input)
 269
 270   if host_run = yaml['host_run']
 271     params = yaml['params'] || {}
 272     host_run.each do |run|
 273       params.each do |k,v|
 274         run = run.gsub("$#{k}", v)
 275       end
 276       STDOUT.write "#{run}--SEP--"
 277     end
 278   end
 279 RUBY
 280
 281   host_run=`cat $config_file | $docker_path run $user_args --rm -i -a stdout -a stdin $image ruby -e "$env_ruby"`
 282
 283   while [ "$host_run" ] ; do
 284     iter=${host_run%%--SEP--*}
 285     echo
 286     echo "Host run: $iter"
 287     $iter || exit 1
 288     echo
 289     host_run="${host_run#*--SEP--}"
 290   done
 291 }
 292
 293
 294 set_volumes() {
 295   volumes=`cat $config_file | $docker_path run $user_args --rm -i -a stdout -a stdin $image ruby -e \
 296         "require 'yaml'; puts YAML.load(STDIN.readlines.join)['volumes'].map{|v| '-v ' << v['volume']['host'] << ':' << v['volume']['guest'] << ' '}.join"`
 297 }
 298
 299 set_links() {
 300     links=`cat $config_file | $docker_path run $user_args --rm -i -a stdout -a stdin $image ruby -e \
 301         "require 'yaml'; puts YAML.load(STDIN.readlines.join)['links'].map{|l| '--link ' << l['link']['name'] << ':' << l['link']['alias'] << ' '}.join"`
 302 }
 303
 304 set_template_info() {
 305
 306     templates=`cat $config_file | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \
 307       "require 'yaml'; puts YAML.load(STDIN.readlines.join)['templates']"`
 308
 309
 310     arrTemplates=(${templates// / })
 311     config_data=$(cat $config_file)
 312
 313     input="hack: true"
 314
 315
 316     for template in "${arrTemplates[@]}"
 317     do
 318       [ ! -z $template ] && {
 319         input="$input _FILE_SEPERATOR_ $(cat $template)"
 320       }
 321     done
 322
 323     # we always want our config file last so it takes priority
 324     input="$input _FILE_SEPERATOR_ $config_data"
 325
 326     read -r -d '' env_ruby << 'RUBY'
 327     require 'yaml'
 328
 329     input=STDIN.readlines.join
 330     # default to UTF-8 for the dbs sake
 331     env = {'LANG' => 'en_US.UTF-8'}
 332     input.split('_FILE_SEPERATOR_').each do |yml|
 333        yml.strip!
 334        begin
 335          env.merge!(YAML.load(yml)['env'] || {})
 336        rescue Psych::SyntaxError => e
 337         puts e
 338         puts "*ERROR."
 339        rescue => e
 340         puts yml
 341         p e
 342        end
 343     end
 344     puts env.map{|k,v| "-e\n#{k}=#{v}" }.join("\n")
 345 RUBY
 346
 347     raw=`exec echo "$input" | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e "$env_ruby"`
 348
 349     env=()
 350     ok=1
 351     while read i; do
 352       if [ "$i" == "*ERROR." ]; then
 353         ok=0
 354       elif [ -n "$i" ]; then
 355         env[${#env[@]}]=$i
 356       fi
 357     done <<< "$raw"
 358
 359     if [ "$ok" -ne 1 ]; then
 360       echo "${env[@]}"
 361       echo "YAML syntax error. Please check your /var/discourse/containers/*.yml config files."
 362       exit 1
 363     fi
 364 }
 365
 366 [ -z $docker_path ] && {
 367   install_docker
 368 }
 369
 370 [ "$command" == "cleanup" ] && {
 371   echo
 372   echo "The following command will"
 373   echo "- Delete all docker images for old containers"
 374   echo "- Delete all stopped and orphan containers"
 375   echo
 376   read -p "Are you sure (Y/n): " -n 1 -r && echo
 377   if [[ $REPLY =~ ^[Yy]$ || ! $REPLY ]]
 378     then
 379       space=$(df /var/lib/docker | awk '{ print $4 }' | grep -v Available)
 380       echo "Starting Cleanup (bytes free $space)"
 381
 382       STATE_DIR=./.gc-state scripts/docker-gc
 383
 384       space=$(df /var/lib/docker | awk '{ print $4 }' | grep -v Available)
 385       echo "Finished Cleanup (bytes free $space)"
 386
 387     else
 388       exit 1
 389   fi
 390   exit 0
 391 }
 392
 393 [ $# -lt 2 ] && {
 394   usage
 395 }
 396
 397 if [ ! -e $config_file ]
 398   then
 399     echo "Config file was not found, ensure $config_file exists"
 400     echo ""
 401     echo "Available configs ( `cd containers && ls -dm *.yml | tr -s '\n' ' ' | awk '{ gsub(/\.yml/, ""); print }'`)"
 402     exit 1
 403 fi
 404
 405
 406 docker_version=($($docker_path --version))
 407 docker_version=${test[2]//,/}
 408
 409 if compare_version "1.2.0" "$docker_version"; then
 410   echo "We recommend you upgrade docker, the version you are running has no restart policies, on reboot your container may not start up"
 411   restart_policy=""
 412 else
 413   restart_policy=${restart_policy:---restart=always}
 414 fi
 415
 416 set_existing_container(){
 417   existing=`$docker_path ps -a | awk '{ print $1, $(NF) }' | grep " $config$" | awk '{ print $1 }'`
 418 }
 419
 420 run_stop(){
 421
 422   set_existing_container
 423
 424   if [ ! -z $existing ]
 425      then
 426        (
 427         set -x
 428         $docker_path stop -t 10 $config
 429        )
 430      else
 431        echo "$config was not started !"
 432        exit 1
 433   fi
 434 }
 435
 436 set_run_image() {
 437   run_image=`cat $config_file | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \
 438     "require 'yaml'; puts YAML.load(STDIN.readlines.join)['run_image']"`
 439
 440   if [ -z "$run_image" ]; then
 441     run_image="$local_discourse/$config"
 442   fi
 443 }
 444
 445 set_boot_command() {
 446   boot_command=`cat $config_file | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \
 447     "require 'yaml'; puts YAML.load(STDIN.readlines.join)['boot_command']"`
 448
 449   if [ -z "$boot_command" ]; then
 450
 451     no_boot_command=`cat $config_file | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \
 452       "require 'yaml'; puts YAML.load(STDIN.readlines.join)['no_boot_command']"`
 453
 454     if [ -z "$no_boot_command" ]; then
 455       boot_command="/sbin/boot"
 456     fi
 457   fi
 458 }
 459
 460 run_start(){
 461
 462    existing=`$docker_path ps | awk '{ print $1, $(NF) }' | grep " $config$" | awk '{ print $1 }'`
 463    echo $existing
 464    if [ ! -z $existing ]
 465    then
 466      echo "Nothing to do, your container has already started!"
 467      exit 1
 468    fi
 469
 470    existing=`$docker_path ps -a | awk '{ print $1, $(NF) }' | grep " $config$" | awk '{ print $1 }'`
 471    if [ ! -z $existing ]
 472    then
 473      echo "starting up existing container"
 474      (
 475        set -x
 476        $docker_path start $config
 477      )
 478      exit 0
 479    fi
 480
 481    host_run
 482    ports=`cat $config_file | $docker_path run $user_args --rm -i -a stdout -a stdin $image ruby -e \
 483           "require 'yaml'; puts YAML.load(STDIN.readlines.join)['expose'].map{|p| \"-p #{p}\"}.join(' ')"`
 484
 485    IFS='-p ' read -a array <<< "$ports"
 486    for element in "${array[@]}"
 487    do
 488      IFS=':' read -a args <<< "$element"
 489      if [ "${#args[@]}" == "2" ]; then
 490        check_ports "${args[0]}"
 491      elif [ "${#args[@]}" == "3" ]; then
 492        check_ports "${args[1]}"
 493      fi
 494    done
 495
 496    docker_args=`cat $config_file | $docker_path run $user_args --rm -i -a stdout -a stdin $image ruby -e \
 497           "require 'yaml'; puts YAML.load(STDIN.readlines.join)['docker_args']"`
 498
 499    set_template_info
 500    set_volumes
 501    set_links
 502    set_run_image
 503    set_boot_command
 504
 505    (
 506      hostname=`hostname`
 507      set -x
 508      $docker_path run $user_args $links $attach_on_run $restart_policy "${env[@]}" -h "$hostname-$config" \
 509         -e DOCKER_HOST_IP=$docker_ip --name $config -t $ports $volumes $docker_args $run_image $boot_command
 510
 511    )
 512    exit 0
 513
 514 }
 515
 516 run_bootstrap(){
 517
 518   if [ "$opt" != "--skip-prereqs" ] ; then
 519     check_resources
 520   fi
 521
 522   host_run
 523
 524   get_ssh_pub_key
 525
 526   # Is the image available?
 527   # If not, pull it here so the user is aware what's happening.
 528   $docker_path history $image >/dev/null 2>&1 || $docker_path pull $image
 529
 530   set_template_info
 531
 532   base_image=`cat $config_file | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \
 533     "require 'yaml'; puts YAML.load(STDIN.readlines.join)['base_image']"`
 534
 535   update_pups=`cat $config_file | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \
 536     "require 'yaml'; puts YAML.load(STDIN.readlines.join)['update_pups']"`
 537
 538   if [[ ! X"" = X"$base_image" ]]; then
 539     image=$base_image
 540   fi
 541
 542   set_volumes
 543   set_links
 544
 545   rm -f $cidbootstrap
 546
 547   run_command="cd /pups &&"
 548   if [[ ! "false" =  $update_pups ]]; then
 549     run_command="$run_command git pull &&"
 550   fi
 551   run_command="$run_command /pups/bin/pups --stdin"
 552
 553   echo $run_command
 554
 555   env=("${env[@]}" "-e" "SSH_PUB_KEY=$ssh_pub_key")
 556
 557   (exec echo "$input" | $docker_path run $user_args $links "${env[@]}" -e DOCKER_HOST_IP=$docker_ip --cidfile $cidbootstrap -i -a stdin -a stdout -a stderr $volumes $image \
 558      /bin/bash -c "$run_command") \
 559      || ($docker_path rm `cat $cidbootstrap` && rm $cidbootstrap)
 560
 561   [ ! -e $cidbootstrap ] && echo "** FAILED TO BOOTSTRAP ** please scroll up and look for earlier error messages, there may be more than one" && exit 1
 562
 563   sleep 5
 564
 565   $docker_path commit `cat $cidbootstrap` $local_discourse/$config || echo 'FAILED TO COMMIT'
 566   $docker_path rm `cat $cidbootstrap` && rm $cidbootstrap
 567 }
 568
 569
 570
 571 case "$command" in
 572   bootstrap)
 573       run_bootstrap
 574       echo "Successfully bootstrapped, to startup use ./launcher start $config"
 575       exit 0
 576       ;;
 577
 578   enter)
 579       exec $docker_path exec -it $config /bin/bash --login
 580       ;;
 581
 582   ssh)
 583        existing=`$docker_path ps | awk '{ print $1, $(NF) }' | grep " $config$" | awk '{ print $1 }'`
 584
 585        if [[ ! -z $existing ]]; then
 586          address="`$docker_path port $config 22`"
 587          split=(${address//:/ })
 588          exec ssh -o StrictHostKeyChecking=no root@${split[0]} -p ${split[1]}
 589        else
 590          echo "$config is not running!"
 591          exit 1
 592        fi
 593       ;;
 594
 595   stop)
 596       run_stop
 597       exit 0
 598       ;;
 599
 600   logs)
 601
 602       $docker_path logs $config
 603       exit 0
 604       ;;
 605
 606   restart)
 607       run_stop
 608       run_start
 609       exit 0
 610       ;;
 611
 612   start)
 613       run_start
 614       exit 0
 615       ;;
 616
 617   rebuild)
 618       if [ "$(git symbolic-ref --short HEAD)" == "master" ]; then
 619         echo "Ensuring discourse docker is up to date"
 620
 621         git remote update
 622
 623         LOCAL=$(git rev-parse @)
 624         REMOTE=$(git rev-parse @{u})
 625         BASE=$(git merge-base @ @{u})
 626
 627         if [ $LOCAL = $REMOTE ]; then
 628           echo "Discourse Docker is up-to-date"
 629
 630         elif [ $LOCAL = $BASE ]; then
 631           echo "Updating Discourse Docker"
 632           git pull || (echo 'failed to update' && exit 1)
 633           exec /bin/bash $0 $@
 634
 635         elif [ $REMOTE = $BASE ]; then
 636           echo "Your version of Discourse Docker is ahead of origin"
 637
 638         else
 639           echo "Discourse Docker has diverged source, this is only expected in Dev mode"
 640         fi
 641
 642       fi
 643
 644       set_existing_container
 645
 646       if [ ! -z $existing ]
 647         then
 648           echo "Stopping old container"
 649           (
 650             set -x
 651             $docker_path stop -t 10 $config
 652           )
 653       fi
 654
 655       run_bootstrap
 656
 657       if [ ! -z $existing ]
 658         then
 659           echo "Removing old container"
 660           (
 661             set -x
 662             $docker_path rm $config
 663           )
 664       fi
 665
 666       run_start
 667       exit 0
 668       ;;
 669
 670
 671   destroy)
 672       (set -x; $docker_path stop -t 10 $config && $docker_path rm $config) || (echo "$config was not found" && exit 0)
 673       exit 0
 674       ;;
 675 esac
 676
 677 usage