#!/usr/bin/env bash usage () { echo "Usage: launcher COMMAND CONFIG [--skip-prereqs] [--docker-args STRING]" echo "Commands:" echo " start: Start/initialize a container" echo " stop: Stop a running container" echo " restart: Restart a container" echo " destroy: Stop and remove a container" echo " enter: Open a shell to run commands inside the container" echo " logs: View the Docker logs for a container" echo " bootstrap: Bootstrap a container for the config based on a template" echo " run: Run the given command with the config in the context of the last bootstrapped image" echo " rebuild: Rebuild a container (destroy old, bootstrap, start new)" echo " cleanup: Remove all containers that have stopped for > 24 hours" echo " start-cmd: Generate docker command used to start container" echo echo "Options:" echo " --skip-prereqs Don't check launcher prerequisites" echo " --docker-args Extra arguments to pass when running docker" echo " --skip-mac-address Don't assign a mac address" echo " --run-image Override the image used for running the container" exit 1 } command=$1 config=$2 # user_args_argv is assigned once when the argument vector is parsed. user_args_argv="" # user_args is mutable: its value may change when templates are parsed. # Superset of user_args_argv. user_args="" user_run_image="" if [[ $command == "run" ]]; then run_command=$3 fi while [ ${#} -gt 0 ]; do case "${1}" in --debug) DEBUG="1" ;; --skip-prereqs) SKIP_PREREQS="1" ;; --skip-mac-address) SKIP_MAC_ADDRESS="1" ;; --docker-args) user_args_argv="$2" user_args="$user_args_argv" shift ;; --run-image) user_run_image="$2" shift ;; esac shift 1 done if [ -z "$command" -o -z "$config" -a "$command" != "cleanup" ]; then usage exit 1 fi # Docker doesn't like uppercase characters, spaces or special characters, catch it now before we build everything out and then find out re='[[:upper:]/ !@#$%^&*()+~`=]' if [[ $config =~ $re ]]; then echo echo "ERROR: Config name '$config' must not contain upper case characters, spaces or special characters. Correct config name and rerun $0." echo exit 1 fi cd "$(dirname "$0")" docker_min_version='17.03.1' docker_rec_version='17.06.2' git_min_version='1.8.0' git_rec_version='1.8.0' config_file=containers/"$config".yml cidbootstrap=cids/"$config"_bootstrap.cid local_discourse=local_discourse image="discourse/base:2.0.20190901-2315" docker_path=`which docker.io 2> /dev/null || which docker` git_path=`which git` if [ "${SUPERVISED}" = "true" ]; then restart_policy="--restart=no" attach_on_start="-a" attach_on_run="-a stdout -a stderr" else attach_on_run="-d" fi if [ -n "$DOCKER_HOST" ]; then docker_ip=`sed -e 's/^tcp:\/\/\(.*\):.*$/\1/' <<< "$DOCKER_HOST"` elif [ -x "$(which ip 2>/dev/null)" ]; then docker_ip=`ip addr show docker0 | \ grep 'inet ' | \ awk '{ split($2,a,"/"); print a[1] }';` else docker_ip=`ifconfig | \ grep -B1 "inet addr" | \ awk '{ if ( $1 == "inet" ) { print $2 } else if ( $2 == "Link" ) { printf "%s:" ,$1 } }' | \ grep docker0 | \ awk -F: '{ print $3 }';` fi # From https://stackoverflow.com/a/44660519/702738 compare_version() { if [[ $1 == $2 ]]; then return 1 fi local IFS=. local i a=(${1%%[^0-9.]*}) b=(${2%%[^0-9.]*}) local arem=${1#${1%%[^0-9.]*}} brem=${2#${2%%[^0-9.]*}} for ((i=0; i<${#a[@]} || i<${#b[@]}; i++)); do if ((10#${a[i]:-0} < 10#${b[i]:-0})); then return 1 elif ((10#${a[i]:-0} > 10#${b[i]:-0})); then return 0 fi done if [ "$arem" '<' "$brem" ]; then return 1 elif [ "$arem" '>' "$brem" ]; then return 0 fi return 1 } install_docker() { echo "Docker is not installed, you will need to install Docker in order to run Launcher" echo "See https://docs.docker.com/installation/" exit 1 } check_prereqs() { if [ -z $docker_path ]; then install_docker fi # 1. docker daemon running? # we send stderr to /dev/null cause we don't care about warnings, # it usually complains about swap which does not matter test=`$docker_path info 2> /dev/null` if [[ $? -ne 0 ]] ; then echo "Cannot connect to the docker daemon - verify it is running and you have access" exit 1 fi # 2. running an approved storage driver? if ! $docker_path info 2> /dev/null | egrep -q 'Storage Driver: (aufs|zfs|overlay2)$'; then echo "Your Docker installation is not using a supported storage driver. If we were to proceed you may have a broken install." echo "aufs is the recommended storage driver, although zfs and overlay2 may work as well." echo "Other storage drivers are known to be problematic." echo "You can tell what filesystem you are using by running \"docker info\" and looking at the 'Storage Driver' line." echo echo "If you wish to continue anyway using your existing unsupported storage driver," echo "read the source code of launcher and figure out how to bypass this check." exit 1 fi # 3. running recommended docker version test=($($docker_path --version)) # Get docker version string test=${test[2]//,/} # Get version alone and strip comma if exists # At least minimum docker version if compare_version "${docker_min_version}" "${test}"; then echo "ERROR: Docker version ${test} not supported, please upgrade to at least ${docker_min_version}, or recommended ${docker_rec_version}" exit 1 fi # Recommend newer docker version if compare_version "${docker_rec_version}" "${test}"; then echo "WARNING: Docker version ${test} deprecated, recommend upgrade to ${docker_rec_version} or newer." fi # 4. discourse docker image is downloaded test=`$docker_path images | awk '{print $1 ":" $2 }' | grep "$image"` if [ -z "$test" ]; then echo echo "WARNING: We are about to start downloading the Discourse base image" echo "This process may take anywhere between a few minutes to an hour, depending on your network speed" echo echo "Please be patient" echo fi # 5. running recommended git version test=($($git_path --version)) # Get git version string test=${test[2]//,/} # Get version alone and strip comma if exists # At least minimum version if compare_version "${git_min_version}" "${test}"; then echo "ERROR: Git version ${test} not supported, please upgrade to at least ${git_min_version}, or recommended ${git_rec_version}" exit 1 fi # Recommend best version if compare_version "${git_rec_version}" "${test}"; then echo "WARNING: Git version ${test} deprecated, recommend upgrade to ${git_rec_version} or newer." fi # 6. able to attach stderr / out / tty test=`$docker_path run $user_args -i --rm -a stdout -a stderr $image echo working` if [[ "$test" =~ "working" ]] ; then : ; else echo "Your Docker installation is not working correctly" echo echo "See: https://meta.discourse.org/t/docker-error-on-bootstrap/13657/18?u=sam" exit 1 fi # 7. enough space for the bootstrap on docker folder folder=`$docker_path info --format '{{.DockerRootDir}}'` safe_folder=${folder:-/var/lib/docker} test=$(($(stat -f --format="%a*%S" $safe_folder)/1024**3 < 5)) if [[ $test -ne 0 ]] ; then echo "You have less than 5GB of free space on the disk where $safe_folder is located. You will need more space to continue" df -h $safe_folder echo if tty >/dev/null; then read -p "Would you like to attempt to recover space by cleaning docker images and containers in the system?(y/N)" -n 1 -r echo if [[ $REPLY =~ ^[Yy]$ ]] then $docker_path system prune -af echo "If the cleanup was successful, you may try again now" fi fi exit 1 fi } if [ -z "$SKIP_PREREQS" ] && [ "$command" != "cleanup" ]; then check_prereqs fi host_run() { read -r -d '' env_ruby << 'RUBY' require 'yaml' input = STDIN.readlines.join yaml = YAML.load(input) if host_run = yaml['host_run'] params = yaml['params'] || {} host_run.each do |run| params.each do |k,v| run = run.gsub("$#{k}", v) end STDOUT.write "#{run}--SEP--" end end RUBY host_run=`cat $config_file | $docker_path run $user_args --rm -i -a stdout -a stdin $image ruby -e "$env_ruby"` while [ "$host_run" ] ; do iter=${host_run%%--SEP--*} echo echo "Host run: $iter" $iter || exit 1 echo host_run="${host_run#*--SEP--}" done } set_volumes() { volumes=`cat $config_file | $docker_path run $user_args --rm -i -a stdout -a stdin $image ruby -e \ "require 'yaml'; puts YAML.load(STDIN.readlines.join)['volumes'].map{|v| '-v ' << v['volume']['host'] << ':' << v['volume']['guest'] << ' '}.join"` } set_links() { links=`cat $config_file | $docker_path run $user_args --rm -i -a stdout -a stdin $image ruby -e \ "require 'yaml'; puts YAML.load(STDIN.readlines.join)['links'].map{|l| '--link ' << l['link']['name'] << ':' << l['link']['alias'] << ' '}.join"` } find_templates() { local templates=`cat $1 | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \ "require 'yaml'; puts YAML.load(STDIN.readlines.join)['templates']"` local arrTemplates=${templates// / } if [ ! -z "$templates" ]; then for template in "${arrTemplates[@]}" do local nested_templates=$(find_templates $template) if [ ! -z "$nested_templates" ]; then templates="$templates $nested_templates" fi done echo $templates else echo "" fi } set_template_info() { templates=$(find_templates $config_file) arrTemplates=(${templates// / }) config_data=$(cat $config_file) input="hack: true" for template in "${arrTemplates[@]}" do [ ! -z $template ] && { input="$input _FILE_SEPERATOR_ $(cat $template)" } done # we always want our config file last so it takes priority input="$input _FILE_SEPERATOR_ $config_data" read -r -d '' env_ruby << 'RUBY' require 'yaml' input=STDIN.readlines.join # default to UTF-8 for the dbs sake env = {'LANG' => 'en_US.UTF-8'} input.split('_FILE_SEPERATOR_').each do |yml| yml.strip! begin env.merge!(YAML.load(yml)['env'] || {}) rescue Psych::SyntaxError => e puts e puts "*ERROR." rescue => e puts yml p e end end puts env.map{|k,v| "-e\n#{k}=#{v}" }.join("\n") RUBY tmp_input_file=$(mktemp) echo "$input" > "$tmp_input_file" raw=`exec cat "$tmp_input_file" | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e "$env_ruby"` rm -f "$tmp_input_file" env=() ok=1 while read i; do if [ "$i" == "*ERROR." ]; then ok=0 elif [ -n "$i" ]; then env[${#env[@]}]="${i//\{\{config\}\}/${config}}" fi done <<< "$raw" if [ "$ok" -ne 1 ]; then echo "${env[@]}" echo "YAML syntax error. Please check your containers/*.yml config files." exit 1 fi # labels read -r -d '' labels_ruby << 'RUBY' require 'yaml' input=STDIN.readlines.join labels = {} input.split('_FILE_SEPERATOR_').each do |yml| yml.strip! begin labels.merge!(YAML.load(yml)['labels'] || {}) rescue Psych::SyntaxError => e puts e puts "*ERROR." rescue => e puts yml p e end end puts labels.map{|k,v| "-l\n#{k}=#{v}" }.join("\n") RUBY tmp_input_file=$(mktemp) echo "$input" > "$tmp_input_file" raw=`exec cat "$tmp_input_file" | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e "$labels_ruby"` rm -f "$tmp_input_file" labels=() ok=1 while read i; do if [ "$i" == "*ERROR." ]; then ok=0 elif [ -n "$i" ]; then labels[${#labels[@]}]=$(echo $i | sed s/{{config}}/${config}/g) fi done <<< "$raw" if [ "$ok" -ne 1 ]; then echo "${labels[@]}" echo "YAML syntax error. Please check your containers/*.yml config files." exit 1 fi # expose read -r -d '' ports_ruby << 'RUBY' require 'yaml' input=STDIN.readlines.join ports = [] input.split('_FILE_SEPERATOR_').each do |yml| yml.strip! begin ports += (YAML.load(yml)['expose'] || []) rescue Psych::SyntaxError => e puts e puts "*ERROR." rescue => e puts yml p e end end puts ports.map { |p| p.to_s.include?(':') ? "-p\n#{p}" : "--expose\n#{p}" }.join("\n") RUBY tmp_input_file=$(mktemp) echo "$input" > "$tmp_input_file" raw=`exec cat "$tmp_input_file" | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e "$ports_ruby"` rm -f "$tmp_input_file" ports=() ok=1 while read i; do if [ "$i" == "*ERROR." ]; then ok=0 elif [ -n "$i" ]; then ports[${#ports[@]}]=$i fi done <<< "$raw" if [ "$ok" -ne 1 ]; then echo "${ports[@]}" echo "YAML syntax error. Please check your containers/*.yml config files." exit 1 fi merge_user_args } if [ -z $docker_path ]; then install_docker fi [ "$command" == "cleanup" ] && { $docker_path system prune -a if [ -d /var/discourse/shared/standalone/postgres_data_old ]; then echo echo "Old PostgreSQL backup data cluster detected taking up $(du -hs /var/discourse/shared/standalone/postgres_data_old | awk '{print $1}') detected" read -p "Would you like to remove it? (Y/n): " -n 1 -r && echo if [[ $REPLY =~ ^[Yy]$ ]]; then echo "removing old PostgreSQL data cluster at /var/discourse/shared/standalone/postgres_data_old..." rm -rf /var/discourse/shared/standalone/postgres_data_old else exit 1 fi fi exit 0 } if [ ! "$command" == "setup" ]; then if [[ ! -e $config_file ]]; then echo "Config file was not found, ensure $config_file exists" echo echo "Available configs ( `cd containers && ls -dm *.yml | tr -s '\n' ' ' | awk '{ gsub(/\.yml/, ""); print }'`)" exit 1 fi fi docker_version=($($docker_path --version)) docker_version=${test[2]//,/} restart_policy=${restart_policy:---restart=always} set_existing_container(){ existing=`$docker_path ps -a | awk '{ print $1, $(NF) }' | grep " $config$" | awk '{ print $1 }'` } run_stop() { set_existing_container if [ ! -z $existing ] then ( set -x $docker_path stop -t 10 $config ) else echo "$config was not started !" echo "./discourse-doctor may help diagnose the problem." exit 1 fi } set_run_image() { run_image=`cat $config_file | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \ "require 'yaml'; puts YAML.load(STDIN.readlines.join)['run_image']"` if [ -n "$user_run_image" ]; then run_image=$user_run_image elif [ -z "$run_image" ]; then run_image="$local_discourse/$config" fi } set_boot_command() { boot_command=`cat $config_file | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \ "require 'yaml'; puts YAML.load(STDIN.readlines.join)['boot_command']"` if [ -z "$boot_command" ]; then no_boot_command=`cat $config_file | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \ "require 'yaml'; puts YAML.load(STDIN.readlines.join)['no_boot_command']"` if [ -z "$no_boot_command" ]; then boot_command="/sbin/boot" fi fi } merge_user_args() { local docker_args docker_args=`cat $config_file | $docker_path run $user_args --rm -i -a stdout -a stdin $image ruby -e \ "require 'yaml'; puts YAML.load(STDIN.readlines.join)['docker_args']"` if [[ -n "$docker_args" ]]; then user_args="$user_args_argv $docker_args" fi } run_start() { if [ -z "$START_CMD_ONLY" ] then existing=`$docker_path ps | awk '{ print $1, $(NF) }' | grep " $config$" | awk '{ print $1 }'` echo $existing if [ ! -z $existing ] then echo "Nothing to do, your container has already started!" exit 0 fi existing=`$docker_path ps -a | awk '{ print $1, $(NF) }' | grep " $config$" | awk '{ print $1 }'` if [ ! -z $existing ] then echo "starting up existing container" ( set -x $docker_path start $config ) exit 0 fi fi host_run set_template_info set_volumes set_links set_run_image set_boot_command # get hostname and settings from container configuration for envar in "${env[@]}" do if [[ $envar == DOCKER_USE_HOSTNAME* ]] || [[ $envar == DISCOURSE_HOSTNAME* ]] then # use as environment variable eval $envar fi done ( hostname=`hostname -s` # overwrite hostname if [ "$DOCKER_USE_HOSTNAME" = "true" ] then hostname=$DISCOURSE_HOSTNAME else hostname=$hostname-$config fi # we got to normalize so we only have allowed strings, this is more comprehensive but lets see how bash does first # hostname=`$docker_path run $user_args --rm $image ruby -e 'print ARGV[0].gsub(/[^a-zA-Z-]/, "-")' $hostname` # docker added more hostname rules hostname=${hostname//_/-} if [ -z "$SKIP_MAC_ADDRESS" ] ; then mac_address="--mac-address $($docker_path run $user_args -i --rm -a stdout -a stderr $image /bin/sh -c "echo $hostname | md5sum | sed 's/^\(..\)\(..\)\(..\)\(..\)\(..\).*$/02:\1:\2:\3:\4:\5/'")" fi if [ ! -z "$START_CMD_ONLY" ] ; then docker_path="true" fi set -x $docker_path run --shm-size=512m $links $attach_on_run $restart_policy "${env[@]}" "${labels[@]}" -h "$hostname" \ -e DOCKER_HOST_IP="$docker_ip" --name $config -t "${ports[@]}" $volumes $mac_address $user_args \ $run_image $boot_command ) exit 0 } run_run() { set_template_info set_volumes set_links set_run_image unset ERR (exec $docker_path run --rm --shm-size=512m $user_args $links "${env[@]}" -e DOCKER_HOST_IP="$docker_ip" -i -a stdin -a stdout -a stderr $volumes $run_image \ /bin/bash -c "$run_command") || ERR=$? if [[ $ERR > 0 ]]; then exit 1 fi } run_bootstrap() { host_run # Is the image available? # If not, pull it here so the user is aware what's happening. $docker_path history $image >/dev/null 2>&1 || $docker_path pull $image set_template_info base_image=`cat $config_file | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \ "require 'yaml'; puts YAML.load(STDIN.readlines.join)['base_image']"` update_pups=`cat $config_file | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \ "require 'yaml'; puts YAML.load(STDIN.readlines.join)['update_pups']"` if [[ ! X"" = X"$base_image" ]]; then image=$base_image fi set_volumes set_links rm -f $cidbootstrap run_command="cd /pups &&" if [[ ! "false" = $update_pups ]]; then run_command="$run_command git pull &&" fi run_command="$run_command /pups/bin/pups --stdin" echo $run_command unset ERR tmp_input_file=$(mktemp) echo "$input" > "$tmp_input_file" (exec cat "$tmp_input_file" | $docker_path run --shm-size=512m $user_args $links "${env[@]}" -e DOCKER_HOST_IP="$docker_ip" --cidfile $cidbootstrap -i -a stdin -a stdout -a stderr $volumes $image \ /bin/bash -c "$run_command") || ERR=$? rm -f "$tmp_input_file" unset FAILED # magic exit code that indicates a retry if [[ "$ERR" == 77 ]]; then $docker_path rm `cat $cidbootstrap` rm $cidbootstrap exit 77 elif [[ "$ERR" > 0 ]]; then FAILED=TRUE fi if [[ $FAILED = "TRUE" ]]; then if [[ ! -z "$DEBUG" ]]; then $docker_path commit `cat $cidbootstrap` $local_discourse/$config-debug || echo 'FAILED TO COMMIT' echo "** DEBUG ** Maintaining image for diagnostics $local_discourse/$config-debug" fi $docker_path rm `cat $cidbootstrap` rm $cidbootstrap echo "** FAILED TO BOOTSTRAP ** please scroll up and look for earlier error messages, there may be more than one." echo "./discourse-doctor may help diagnose the problem." exit 1 fi sleep 5 $docker_path commit `cat $cidbootstrap` $local_discourse/$config || echo 'FAILED TO COMMIT' $docker_path rm `cat $cidbootstrap` && rm $cidbootstrap } case "$command" in bootstrap) run_bootstrap echo "Successfully bootstrapped, to startup use ./launcher start $config" exit 0 ;; run) run_run exit 0 ;; enter) exec $docker_path exec -it $config /bin/bash --login ;; stop) run_stop exit 0 ;; logs) $docker_path logs $config exit 0 ;; restart) run_stop run_start exit 0 ;; start-cmd) START_CMD_ONLY="1" run_start exit 0; ;; start) run_start exit 0 ;; rebuild) if [ "$(git symbolic-ref --short HEAD)" == "master" ]; then echo "Ensuring launcher is up to date" git remote update LOCAL=$(git rev-parse HEAD) REMOTE=$(git rev-parse @{u}) BASE=$(git merge-base HEAD @{u}) if [ $LOCAL = $REMOTE ]; then echo "Launcher is up-to-date" elif [ $LOCAL = $BASE ]; then echo "Updating Launcher" git pull || (echo 'failed to update' && exit 1) for (( i=${#BASH_ARGV[@]}-1,j=0; i>=0,j<${#BASH_ARGV[@]}; i--,j++ )) do args[$j]=${BASH_ARGV[$i]} done exec /bin/bash $0 "${args[@]}" # $@ is empty, because of shift at the beginning. Use BASH_ARGV instead. elif [ $REMOTE = $BASE ]; then echo "Your version of Launcher is ahead of origin" else echo "Launcher has diverged source, this is only expected in Dev mode" fi fi set_existing_container if [ ! -z $existing ] then echo "Stopping old container" ( set -x $docker_path stop -t 10 $config ) fi run_bootstrap if [ ! -z $existing ] then echo "Removing old container" ( set -x $docker_path rm $config ) fi run_start exit 0 ;; destroy) (set -x; $docker_path stop -t 10 $config && $docker_path rm $config) || (echo "$config was not found" && exit 0) exit 0 ;; esac usage