X-Git-Url: https://vcs.fsf.org/?a=blobdiff_plain;f=launcher;h=4e615927503187c33d970fc27142d7ad0238e080;hb=2f7a204e0ed810526dc792c1ac00d508766e03a0;hp=88d174d0098b5076a1219bb21047be4bae39d295;hpb=889fb825d4f7fbcf11ce8b9f831549c4817c2cec;p=discourse_docker.git diff --git a/launcher b/launcher index 88d174d..4e61592 100755 --- a/launcher +++ b/launcher @@ -3,25 +3,34 @@ usage () { echo "Usage: launcher COMMAND CONFIG [--skip-prereqs] [--docker-args STRING]" echo "Commands:" - echo " start: Start/initialize a container" - echo " stop: Stop a running container" - echo " restart: Restart a container" - echo " destroy: Stop and remove a container" - echo " enter: Use nsenter to get a shell into a container" - echo " logs: View the Docker logs for a container" - echo " bootstrap: Bootstrap a container for the config based on a template" - echo " rebuild: Rebuild a container (destroy old, bootstrap, start new)" - echo " cleanup: Remove all containers that have stopped for > 24 hours" + echo " start: Start/initialize a container" + echo " stop: Stop a running container" + echo " restart: Restart a container" + echo " destroy: Stop and remove a container" + echo " enter: Open a shell to run commands inside the container" + echo " logs: View the Docker logs for a container" + echo " bootstrap: Bootstrap a container for the config based on a template" + echo " run: Run the given command with the config in the context of the last bootstrapped image" + echo " rebuild: Rebuild a container (destroy old, bootstrap, start new)" + echo " cleanup: Remove all containers that have stopped for > 24 hours" + echo " start-cmd: Generate docker command used to start container" echo echo "Options:" echo " --skip-prereqs Don't check launcher prerequisites" echo " --docker-args Extra arguments to pass when running docker" + echo " --skip-mac-address Don't assign a mac address" + echo " --run-image Override the image used for running the container" exit 1 } command=$1 config=$2 user_args="" +user_run_image="" + +if [[ $command == "run" ]]; then + run_command=$3 +fi while [ ${#} -gt 0 ]; do case "${1}" in @@ -29,39 +38,51 @@ while [ ${#} -gt 0 ]; do DEBUG="1" ;; --skip-prereqs) - SKIP_PREREQ="1" + SKIP_PREREQS="1" + ;; + --skip-mac-address) + SKIP_MAC_ADDRESS="1" ;; --docker-args) user_args="$2" shift ;; + --run-image) + user_run_image="$2" + shift + ;; esac shift 1 done +if [ -z "$command" -o -z "$config" -a "$command" != "cleanup" ]; then + usage + exit 1 +fi + # Docker doesn't like uppercase characters, spaces or special characters, catch it now before we build everything out and then find out -re='[A-Z/ !@#$%^&*()+~`=]' +re='[[:upper:]/ !@#$%^&*()+~`=]' if [[ $config =~ $re ]]; then echo - echo "ERROR: Config name must not contain upper case characters, spaces or special characters. Correct config name and rerun $0." + echo "ERROR: Config name '$config' must not contain upper case characters, spaces or special characters. Correct config name and rerun $0." echo exit 1 fi cd "$(dirname "$0")" -docker_min_version='1.6.0' -docker_rec_version='1.6.0' +docker_min_version='17.03.1' +docker_rec_version='17.06.2' git_min_version='1.8.0' git_rec_version='1.8.0' config_file=containers/"$config".yml cidbootstrap=cids/"$config"_bootstrap.cid local_discourse=local_discourse -image=discourse/discourse:1.0.17 -docker_path=`which docker.io || which docker` +image="discourse/base:2.0.20190321-0122" +docker_path=`which docker.io 2> /dev/null || which docker` git_path=`which git` if [ "${SUPERVISED}" = "true" ]; then @@ -86,25 +107,27 @@ else awk -F: '{ print $3 }';` fi +# From https://stackoverflow.com/a/44660519/702738 compare_version() { - declare -a ver_a - declare -a ver_b - IFS=. read -a ver_a <<< "$1" - IFS=. read -a ver_b <<< "$2" - - while [[ -n $ver_a ]]; do - if (( ver_a > ver_b )); then - return 0 - elif (( ver_b > ver_a )); then + if [[ $1 == $2 ]]; then + return 1 + fi + local IFS=. + local i a=(${1%%[^0-9.]*}) b=(${2%%[^0-9.]*}) + local arem=${1#${1%%[^0-9.]*}} brem=${2#${2%%[^0-9.]*}} + for ((i=0; i<${#a[@]} || i<${#b[@]}; i++)); do + if ((10#${a[i]:-0} < 10#${b[i]:-0})); then return 1 - else - unset ver_a[0] - ver_a=("${ver_a[@]}") - unset ver_b[0] - ver_b=("${ver_b[@]}") + elif ((10#${a[i]:-0} > 10#${b[i]:-0})); then + return 0 fi done - return 1 # They are equal + if [ "$arem" '<' "$brem" ]; then + return 1 + elif [ "$arem" '>' "$brem" ]; then + return 0 + fi + return 1 } @@ -129,15 +152,15 @@ check_prereqs() { exit 1 fi - # 2. running aufs or btrfs - test=`$docker_path info 2> /dev/null | grep 'Driver: '` - if [[ "$test" =~ [aufs|btrfs|zfs|overlay] ]] ; then : ; else - echo "Your Docker installation is not using a supported filesystem if we were to proceed you may have a broken install." - echo "aufs is the recommended filesystem you should be using (zfs/btrfs and overlay may work as well)" - echo "You can tell what filesystem you are using by running \"docker info\" and looking at the driver" + # 2. running an approved storage driver? + if ! $docker_path info 2> /dev/null | egrep -q '^Storage Driver: (aufs|btrfs|zfs|overlay|overlay2)$'; then + echo "Your Docker installation is not using a supported storage driver. If we were to proceed you may have a broken install." + echo "aufs is the recommended storage driver, although zfs/btrfs/overlay and overlay2 may work as well." + echo "Other storage drivers are known to be problematic." + echo "You can tell what filesystem you are using by running \"docker info\" and looking at the 'Storage Driver' line." echo - echo "If you wish to continue anyway using your existing unsupported filesystem, " - echo "read the source code of launcher and figure out how to bypass this." + echo "If you wish to continue anyway using your existing unsupported storage driver," + echo "read the source code of launcher and figure out how to bypass this check." exit 1 fi @@ -192,10 +215,27 @@ check_prereqs() { exit 1 fi + # 7. enough space for the bootstrap on docker folder + folder=`$docker_path info --format '{{.DockerRootDir}}'` + safe_folder=${folder:-/var/lib/docker} + test=$(($(stat -f --format="%a*%S" $safe_folder)/1024**3 < 5)) + if [[ $test -ne 0 ]] ; then + echo "You have less than 5GB of free space on the disk where $safe_folder is located. You will need more space to continue" + df -h $safe_folder + echo + read -p "Would you like to attempt to recover space by cleaning docker images and containers in the system?(y/N)" -n 1 -r + echo + if [[ $REPLY =~ ^[Yy]$ ]] + then + $docker_path system prune -af + echo "If the cleanup was successful, you may try again now" + fi + exit 1 + fi } -if [ -z "$SKIP_PREREQS" ] ; then +if [ -z "$SKIP_PREREQS" ] && [ "$command" != "cleanup" ]; then check_prereqs fi @@ -240,11 +280,31 @@ set_links() { "require 'yaml'; puts YAML.load(STDIN.readlines.join)['links'].map{|l| '--link ' << l['link']['name'] << ':' << l['link']['alias'] << ' '}.join"` } -set_template_info() { - - templates=`cat $config_file | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \ +find_templates() { + local templates=`cat $1 | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \ "require 'yaml'; puts YAML.load(STDIN.readlines.join)['templates']"` + local arrTemplates=${templates// / } + + if [ ! -z "$templates" ]; then + for template in "${arrTemplates[@]}" + do + local nested_templates=$(find_templates $template) + + if [ ! -z "$nested_templates" ]; then + templates="$templates $nested_templates" + fi + done + + echo $templates + else + echo "" + fi +} + +set_template_info() { + templates=$(find_templates $config_file) + arrTemplates=(${templates// / }) config_data=$(cat $config_file) @@ -281,7 +341,12 @@ set_template_info() { puts env.map{|k,v| "-e\n#{k}=#{v}" }.join("\n") RUBY - raw=`exec echo "$input" | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e "$env_ruby"` + tmp_input_file=$(mktemp) + + echo "$input" > "$tmp_input_file" + raw=`exec cat "$tmp_input_file" | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e "$env_ruby"` + + rm -f "$tmp_input_file" env=() ok=1 @@ -289,7 +354,7 @@ RUBY if [ "$i" == "*ERROR." ]; then ok=0 elif [ -n "$i" ]; then - env[${#env[@]}]=$i + env[${#env[@]}]="${i//\{\{config\}\}/${config}}" fi done <<< "$raw" @@ -298,6 +363,96 @@ RUBY echo "YAML syntax error. Please check your containers/*.yml config files." exit 1 fi + + # labels + read -r -d '' labels_ruby << 'RUBY' + require 'yaml' + + input=STDIN.readlines.join + labels = {} + input.split('_FILE_SEPERATOR_').each do |yml| + yml.strip! + begin + labels.merge!(YAML.load(yml)['labels'] || {}) + rescue Psych::SyntaxError => e + puts e + puts "*ERROR." + rescue => e + puts yml + p e + end + end + puts labels.map{|k,v| "-l\n#{k}=#{v}" }.join("\n") +RUBY + + tmp_input_file=$(mktemp) + + echo "$input" > "$tmp_input_file" + raw=`exec cat "$tmp_input_file" | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e "$labels_ruby"` + + rm -f "$tmp_input_file" + + labels=() + ok=1 + while read i; do + if [ "$i" == "*ERROR." ]; then + ok=0 + elif [ -n "$i" ]; then + labels[${#labels[@]}]=$(echo $i | sed s/{{config}}/${config}/g) + fi + done <<< "$raw" + + if [ "$ok" -ne 1 ]; then + echo "${labels[@]}" + echo "YAML syntax error. Please check your containers/*.yml config files." + exit 1 + fi + + # expose + read -r -d '' ports_ruby << 'RUBY' + require 'yaml' + + input=STDIN.readlines.join + ports = [] + input.split('_FILE_SEPERATOR_').each do |yml| + yml.strip! + begin + ports += (YAML.load(yml)['expose'] || []) + rescue Psych::SyntaxError => e + puts e + puts "*ERROR." + rescue => e + puts yml + p e + end + end + puts ports.map { |p| p.to_s.include?(':') ? "-p\n#{p}" : "--expose\n#{p}" }.join("\n") +RUBY + + tmp_input_file=$(mktemp) + + echo "$input" > "$tmp_input_file" + raw=`exec cat "$tmp_input_file" | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e "$ports_ruby"` + + rm -f "$tmp_input_file" + + ports=() + ok=1 + while read i; do + if [ "$i" == "*ERROR." ]; then + ok=0 + elif [ -n "$i" ]; then + ports[${#ports[@]}]=$i + fi + done <<< "$raw" + + if [ "$ok" -ne 1 ]; then + echo "${ports[@]}" + echo "YAML syntax error. Please check your containers/*.yml config files." + exit 1 + fi + + merge_user_args } if [ -z $docker_path ]; then @@ -305,32 +460,24 @@ if [ -z $docker_path ]; then fi [ "$command" == "cleanup" ] && { - echo - echo "The following command will" - echo "- Delete all docker images for old containers" - echo "- Delete all stopped and orphan containers" - echo - read -p "Are you sure (Y/n): " -n 1 -r && echo - if [[ $REPLY =~ ^[Yy]$ || ! $REPLY ]] - then - space=$(df /var/lib/docker | awk '{ print $4 }' | grep -v Available) - echo "Starting Cleanup (bytes free $space)" + $docker_path system prune -a - STATE_DIR=./.gc-state scripts/docker-gc - - space=$(df /var/lib/docker | awk '{ print $4 }' | grep -v Available) - echo "Finished Cleanup (bytes free $space)" + if [ -d /var/discourse/shared/standalone/postgres_data_old ]; then + echo + echo "Old PostgreSQL backup data cluster detected taking up $(du -hs /var/discourse/shared/standalone/postgres_data_old | awk '{print $1}') detected" + read -p "Would you like to remove it? (Y/n): " -n 1 -r && echo + if [[ $REPLY =~ ^[Yy]$ ]]; then + echo "removing old PostgreSQL data cluster at /var/discourse/shared/standalone/postgres_data_old..." + rm -rf /var/discourse/shared/standalone/postgres_data_old else exit 1 + fi fi + exit 0 } -if [ -z "$command" -a -z "$config" ]; then - usage -fi - if [ ! "$command" == "setup" ]; then if [[ ! -e $config_file ]]; then echo "Config file was not found, ensure $config_file exists" @@ -368,7 +515,9 @@ set_run_image() { run_image=`cat $config_file | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \ "require 'yaml'; puts YAML.load(STDIN.readlines.join)['run_image']"` - if [ -z "$run_image" ]; then + if [ -n "$user_run_image" ]; then + run_image=$user_run_image + elif [ -z "$run_image" ]; then run_image="$local_discourse/$config" fi } @@ -388,35 +537,43 @@ set_boot_command() { fi } +merge_user_args() { + local docker_args + + docker_args=`cat $config_file | $docker_path run $user_args --rm -i -a stdout -a stdin $image ruby -e \ + "require 'yaml'; puts YAML.load(STDIN.readlines.join)['docker_args']"` + + if [[ -n "$docker_args" ]]; then + user_args="$user_args $docker_args" + fi +} + run_start() { - existing=`$docker_path ps | awk '{ print $1, $(NF) }' | grep " $config$" | awk '{ print $1 }'` - echo $existing - if [ ! -z $existing ] + if [ -z "$START_CMD_ONLY" ] then - echo "Nothing to do, your container has already started!" - exit 0 - fi + existing=`$docker_path ps | awk '{ print $1, $(NF) }' | grep " $config$" | awk '{ print $1 }'` + echo $existing + if [ ! -z $existing ] + then + echo "Nothing to do, your container has already started!" + exit 0 + fi - existing=`$docker_path ps -a | awk '{ print $1, $(NF) }' | grep " $config$" | awk '{ print $1 }'` - if [ ! -z $existing ] - then - echo "starting up existing container" - ( - set -x - $docker_path start $config - ) - exit 0 + existing=`$docker_path ps -a | awk '{ print $1, $(NF) }' | grep " $config$" | awk '{ print $1 }'` + if [ ! -z $existing ] + then + echo "starting up existing container" + ( + set -x + $docker_path start $config + ) + exit 0 + fi fi host_run - ports=`cat $config_file | $docker_path run $user_args --rm -i -a stdout -a stdin $image ruby -e \ - "require 'yaml'; puts YAML.load(STDIN.readlines.join)['expose'].map{|p| \"-p #{p}\"}.join(' ')"` - - docker_args=`cat $config_file | $docker_path run $user_args --rm -i -a stdout -a stdin $image ruby -e \ - "require 'yaml'; puts YAML.load(STDIN.readlines.join)['docker_args']"` - set_template_info set_volumes set_links @@ -448,11 +605,19 @@ run_start() { # docker added more hostname rules hostname=${hostname//_/-} - mac_address="--mac-address $($docker_path run $user_args -i --rm -a stdout -a stderr $image /bin/sh -c "echo $hostname | md5sum | sed 's/^\(..\)\(..\)\(..\)\(..\)\(..\).*$/02:\1:\2:\3:\4:\5/'")" + + if [ -z "$SKIP_MAC_ADDRESS" ] ; then + mac_address="--mac-address $($docker_path run $user_args -i --rm -a stdout -a stderr $image /bin/sh -c "echo $hostname | md5sum | sed 's/^\(..\)\(..\)\(..\)\(..\)\(..\).*$/02:\1:\2:\3:\4:\5/'")" + fi + + if [ ! -z "$START_CMD_ONLY" ] ; then + docker_path="true" + fi set -x - $docker_path run $user_args $links $attach_on_run $restart_policy "${env[@]}" -h "$hostname" \ - -e DOCKER_HOST_IP=$docker_ip --name $config -t $ports $volumes $mac_address $docker_args \ + + $docker_path run --shm-size=512m $links $attach_on_run $restart_policy "${env[@]}" "${labels[@]}" -h "$hostname" \ + -e DOCKER_HOST_IP="$docker_ip" --name $config -t "${ports[@]}" $volumes $mac_address $user_args \ $run_image $boot_command ) @@ -460,10 +625,22 @@ run_start() { } +run_run() { + set_template_info + set_volumes + set_links + set_run_image -run_bootstrap() { + unset ERR + (exec $docker_path run --rm --shm-size=512m $user_args $links "${env[@]}" -e DOCKER_HOST_IP="$docker_ip" -i -a stdin -a stdout -a stderr $volumes $run_image \ + /bin/bash -c "$run_command") || ERR=$? - # I got no frigging clue what this does, ask Sam Saffron. It RUNS STUFF ON THE HOST I GUESS? + if [[ $ERR > 0 ]]; then + exit 1 + fi +} + +run_bootstrap() { host_run # Is the image available? @@ -495,9 +672,25 @@ run_bootstrap() { echo $run_command + unset ERR + + tmp_input_file=$(mktemp) + + echo "$input" > "$tmp_input_file" + (exec cat "$tmp_input_file" | $docker_path run --shm-size=512m $user_args $links "${env[@]}" -e DOCKER_HOST_IP="$docker_ip" --cidfile $cidbootstrap -i -a stdin -a stdout -a stderr $volumes $image \ + /bin/bash -c "$run_command") || ERR=$? + + rm -f "$tmp_input_file" + unset FAILED - (exec echo "$input" | $docker_path run $user_args $links "${env[@]}" -e DOCKER_HOST_IP=$docker_ip --cidfile $cidbootstrap -i -a stdin -a stdout -a stderr $volumes $image \ - /bin/bash -c "$run_command") || FAILED="TRUE" + # magic exit code that indicates a retry + if [[ "$ERR" == 77 ]]; then + $docker_path rm `cat $cidbootstrap` + rm $cidbootstrap + exit 77 + elif [[ "$ERR" > 0 ]]; then + FAILED=TRUE + fi if [[ $FAILED = "TRUE" ]]; then if [[ ! -z "$DEBUG" ]]; then @@ -517,8 +710,6 @@ run_bootstrap() { $docker_path rm `cat $cidbootstrap` && rm $cidbootstrap } - - case "$command" in bootstrap) run_bootstrap @@ -526,6 +717,11 @@ case "$command" in exit 0 ;; + run) + run_run + exit 0 + ;; + enter) exec $docker_path exec -it $config /bin/bash --login ;; @@ -547,6 +743,12 @@ case "$command" in exit 0 ;; + start-cmd) + START_CMD_ONLY="1" + run_start + exit 0; + ;; + start) run_start exit 0 @@ -568,7 +770,12 @@ case "$command" in elif [ $LOCAL = $BASE ]; then echo "Updating Launcher" git pull || (echo 'failed to update' && exit 1) - exec /bin/bash $0 $@ + + for (( i=${#BASH_ARGV[@]}-1,j=0; i>=0,j<${#BASH_ARGV[@]}; i--,j++ )) + do + args[$j]=${BASH_ARGV[$i]} + done + exec /bin/bash $0 "${args[@]}" # $@ is empty, because of shift at the beginning. Use BASH_ARGV instead. elif [ $REMOTE = $BASE ]; then echo "Your version of Launcher is ahead of origin"