From 9fb5f2d34fd4827f9c4cfa349307e0e30f2e023c Mon Sep 17 00:00:00 2001 From: Sam Saffron Date: Tue, 26 Nov 2013 23:31:50 -0800 Subject: [PATCH] split up templates --- launcher | 24 ++- standalone.template.yml | 263 -------------------------------- templates/cron.template.yml | 13 ++ templates/postgres.template.yml | 93 +++++++++++ templates/redis.template.yml | 51 +++++++ templates/sshd.template.yml | 19 +++ templates/web.template.yml | 116 ++++++++++++++ 7 files changed, 310 insertions(+), 269 deletions(-) delete mode 100644 standalone.template.yml create mode 100644 templates/cron.template.yml create mode 100644 templates/postgres.template.yml create mode 100644 templates/redis.template.yml create mode 100644 templates/sshd.template.yml create mode 100644 templates/web.template.yml diff --git a/launcher b/launcher index 7b7369a..a8f1d36 100755 --- a/launcher +++ b/launcher @@ -63,17 +63,29 @@ fi case "$command" in bootstrap) - template=`cat $config_file | docker run -rm -i -a stdin -a stdout $image ruby -e \ - "require 'yaml'; puts YAML.load(STDIN.readlines.join)['template']"` + templates=`cat $config_file | docker run -rm -i -a stdin -a stdout $image ruby -e \ + "require 'yaml'; puts YAML.load(STDIN.readlines.join)['templates']"` + arrTemplates=(${templates// / }) input=$(cat $config_file) - [ ! -z $template ] && { - input="$input _FILE_SEPERATOR_ $(cat $template)" - } + + + for template in "${arrTemplates[@]}" + do + [ ! -z $template ] && { + input="$input _FILE_SEPERATOR_ $(cat $template)" + } + done set_volumes - exec echo "$input" | docker run -cidfile $cidbootstrap -i -a stdin -a stdout -a stderr $volumes $image /pups/bin/pups --stdin + docker run -cidfile $cidbootstrap -i -a stdin -a stdout -a stderr $image /bin/bash -c 'cd /pups && git pull' + docker commit `cat $cidbootstrap` $local_discourse/up_to_date_pups + docker rm `cat $cidbootstrap` && rm $cidbootstrap + + (exec echo "$input" | docker run -cidfile $cidbootstrap -i -a stdin -a stdout -a stderr $volumes $local_discourse/up_to_date_pups /pups/bin/pups --stdin) \ + || exit 1 && docker rm `cat $cidbootstrap` && rm $cidbootstrap && exit 1 + sleep 10 docker commit `cat $cidbootstrap` $local_discourse/$config || echo 'FAILED TO COMMIT' docker rm `cat $cidbootstrap` && rm $cidbootstrap diff --git a/standalone.template.yml b/standalone.template.yml deleted file mode 100644 index e6f39fc..0000000 --- a/standalone.template.yml +++ /dev/null @@ -1,263 +0,0 @@ -# expose: - # expose public port 80 to map to private docker port 80 (web) - # - "80:80" - # expose public port 2222 to map to private docker port 22 (ssh) - # - "2222:22" - -# ENV is baked in to the image, any changes here require ./launcher bootstrap to run -env: - # Comma delimited list of emails, required if you want admin access for first account - DEVELOPER_EMAILS: 'YOUR_EMAIL@EMAIL.COM1' - # You can have redis on a different box - # REDIS_PROVIDER_URL: 'redis://l.discourse:6379' - RAILS_ENV: 'production' - UNICORN_WORKERS: 3 - # slightly less aggressive than "recommendation" but works fine with oobgc - RUBY_GC_MALLOC_LIMIT: 40000000 - # this ensures we have enough heap space to handle a big pile of small reqs - RUBY_HEAP_MIN_SLOTS: 800000 - -params: - # SSH key is required for remote access into the container - ssh_key: YOUR_SSH_KEY_HERE - version: HEAD - - home: /var/www/discourse - - # You can connect to any DB you wish to - database_yml: - production: - database: discourse - username: discourse - socket: /var/run/postgresql - password: - host: - host_names: - - YOUR_HOSTNAME_HERE - -run: - - file: - path: /etc/service/cron/run - chmod: "+x" - contents: | - #!/bin/bash - exec 2>&1 - cd / - exec cron - - - file: - path: /var/lib/postgresql/take-database-backup - chown: postgres:postgres - chmod: "+x" - contents: | - #!/bin/bash - ID=db-$(date +%F_%T) - FILENAME=/shared/postgres_backup/$ID.tar.gz - pg_basebackup --format=tar --pgdata=- --xlog --gzip --label=$ID > $FILENAME - echo $FILENAME - - - file: - path: /var/spool/cron/crontabs/postgres - contents: | - # m h dom mon dow command - #MAILTO=? - 0 */4 * * * /var/lib/postgresql/take-database-backup - - - file: - path: /etc/service/unicorn/run - chmod: "+x" - contents: | - #!/bin/bash - exec 2>&1 - $env - sv start redis || exit 1 - sv start postgres || exit 1 - cd $home - exec sudo -E -u discourse LD_PRELOAD=/usr/lib/libjemalloc.so.1 bundle exec config/unicorn_launcher -E production -c config/unicorn.conf.rb - - - file: - path: /etc/service/sidekiq/run - chmod: "+x" - contents: | - #!/bin/bash - exec 2>&1 - $env - sv start redis || exit 1 - sv start postgres || exit 1 - cd $home - exec sudo -E -u discourse LD_PRELOAD=/usr/lib/libjemalloc.so.1 bundle exec sidekiq - - - file: - path: /etc/service/sshd/run - chmod: "+x" - contents: | - #!/bin/sh - exec 2>&1 - exec /usr/sbin/sshd -D -e - - - file: - path: /etc/service/redis/run - chmod: "+x" - contents: | - #!/bin/sh - exec 2>&1 - exec sudo -u redis /usr/bin/redis-server /etc/redis/redis.conf - - - file: - path: /etc/service/nginx/run - chmod: "+x" - contents: | - #!/bin/sh - exec 2>&1 - exec /usr/sbin/nginx - - - file: - path: /etc/service/postgres/run - chmod: "+x" - contents: | - #!/bin/sh - exec 2>&1 - exec sudo -u postgres /usr/lib/postgresql/9.2/bin/postmaster -D /etc/postgresql/9.2/main - - - exec: - cmd: - - mkdir -p /var/run/sshd - - mkdir -p /root/.ssh - - echo $ssh_key >> /root/.ssh/authorized_keys - - - exec: - cd: $home - hook: code - cmd: - - git reset --hard - - git clean -f - - git pull - - git checkout $head - - cp config/database.yml.production-sample config/database.yml - - cp config/redis.yml.sample config/redis.yml - - cp config/environments/production.rb.sample config/environments/production.rb - - mkdir -p tmp/pids - - mkdir -p tmp/sockets - - touch tmp/.gitkeep - - mkdir -p /shared/log/rails - - rm -r log - - ln -s /shared/log/rails $home/log - - mkdir -p /shared/uploads - - ln -s /shared/uploads $home/public/uploads - - chown -R discourse /shared/uploads - - chown -R discourse /shared/log/rails - - exec: - cmd: - - "cp $home/config/nginx.sample.conf /etc/nginx/conf.d/discourse.conf" - - "rm /etc/nginx/sites-enabled/default" - - - replace: - filename: /etc/nginx/nginx.conf - from: pid /run/nginx.pid; - to: daemon off; - - - replace: - filename: "/etc/nginx/conf.d/discourse.conf" - from: /upstream[^\}]+\}/m - to: "upstream discourse { - server 127.0.0.1:3000; - }" - - - replace: - filename: "/etc/nginx/conf.d/discourse.conf" - from: /server_name.+$/ - to: server_name _ ; - - - replace: - filename: "/etc/redis/redis.conf" - from: "daemonize yes" - to: "" - - replace: - filename: "/etc/redis/redis.conf" - from: /^pidfile.*$/ - to: "" - - - exec: - cmd: - - install -d -m 0755 -o redis -g redis /shared/redis_data - - - replace: - filename: "/etc/redis/redis.conf" - from: /^logfile.*$/ - to: "logfile \"\"" - - - replace: - filename: "/etc/redis/redis.conf" - from: /^dir .*$/ - to: "dir /shared/redis_data" - - # we can not migrate without redis - - exec: - background: true - cmd: "sudo -u redis /usr/bin/redis-server /etc/redis/redis.conf" - - - merge: $home/config/database.yml $database_yml - - - exec: - cmd: - - chown -R root /var/lib/postgresql/9.2/main - - "[ ! -e /shared/postgres_data ] && cp -r /var/lib/postgresql/9.2/main /shared/postgres_data || exit 0" - - chown -R postgres:postgres /shared/postgres_data - - - replace: - filename: "/etc/postgresql/9.2/main/postgresql.conf" - from: "data_directory = '/var/lib/postgresql/9.2/main'" - to: "data_directory = '/shared/postgres_data'" - - # Necessary to enable backups - - exec: - cmd: - - install -d -m 0755 -o postgres -g postgres /shared/postgres_backup - - - replace: - filename: "/etc/postgresql/9.2/main/postgresql.conf" - from: /#?max_wal_senders *=.*/ - to: "max_wal_senders = 4" - - - replace: - filename: "/etc/postgresql/9.2/main/postgresql.conf" - from: /#?wal_level *=.*/ - to: "wal_level = hot_standby" - - - replace: - filename: "/etc/postgresql/9.2/main/pg_hba.conf" - from: /^#local +replication +postgres +peer$/ - to: "local replication postgres peer" - - - exec: - background: true - cmd: "sudo -u postgres /usr/lib/postgresql/9.2/bin/postmaster -D /etc/postgresql/9.2/main" - - # give db a few secs to start up - - exec: "sleep 5" - - - exec: sudo -u postgres createdb discourse || exit 0 - - exec: - stdin: | - create user discourse; - cmd: sudo -u postgres psql discourse - raise_on_fail: false - - - exec: - stdin: | - grant all privileges on database discourse to discourse; - cmd: sudo -u postgres psql discourse - raise_on_fail: false - - - exec: /bin/bash -c 'sudo -u postgres psql discourse <<< "create extension if not exists hstore;"' - - exec: /bin/bash -c 'sudo -u postgres psql discourse <<< "create extension if not exists pg_trgm;"' - - - - exec: - cd: $home - cmd: - - chown -R discourse $home - - sudo -E -u discourse bundle install --deployment --without test --without development - - sudo -E -u discourse bundle exec rake db:migrate - - sudo -E -u discourse bundle exec rake assets:precompile - diff --git a/templates/cron.template.yml b/templates/cron.template.yml new file mode 100644 index 0000000..86d47a6 --- /dev/null +++ b/templates/cron.template.yml @@ -0,0 +1,13 @@ +run: + - file: + path: /etc/service/cron/run + chmod: "+x" + contents: | + #!/bin/bash + exec 2>&1 + cd / + exec cron + - exec: + hook: cron + cmd: echo cron installed + diff --git a/templates/postgres.template.yml b/templates/postgres.template.yml new file mode 100644 index 0000000..b42145c --- /dev/null +++ b/templates/postgres.template.yml @@ -0,0 +1,93 @@ +hooks: + after_cron: + - file: + path: /var/lib/postgresql/take-database-backup + chown: postgres:postgres + chmod: "+x" + contents: | + #!/bin/bash + ID=db-$(date +%F_%T) + FILENAME=/shared/postgres_backup/$ID.tar.gz + pg_basebackup --format=tar --pgdata=- --xlog --gzip --label=$ID > $FILENAME + echo $FILENAME + + - file: + path: /var/spool/cron/crontabs/postgres + contents: | + # m h dom mon dow command + #MAILTO=? + 0 */4 * * * /var/lib/postgresql/take-database-backup + + before_code: + - exec: + background: true + cmd: "sudo -u postgres /usr/lib/postgresql/9.2/bin/postmaster -D /etc/postgresql/9.2/main" + + # give db a few secs to start up + - exec: "sleep 5" + + - exec: sudo -u postgres createdb discourse || exit 0 + - exec: + stdin: | + create user discourse; + cmd: sudo -u postgres psql discourse + raise_on_fail: false + + - exec: + stdin: | + grant all privileges on database discourse to discourse; + cmd: sudo -u postgres psql discourse + raise_on_fail: false + + - exec: /bin/bash -c 'sudo -u postgres psql discourse <<< "create extension if not exists hstore;"' + - exec: /bin/bash -c 'sudo -u postgres psql discourse <<< "create extension if not exists pg_trgm;"' + - replace: + filename: /etc/service/sidekiq/run + from: "# postgres" + to: sv start postgres || exit 1 + - replace: + filename: /etc/service/unicorn/run + from: "# postgres" + to: sv start postgres || exit 1 + + +run: + + - file: + path: /etc/service/postgres/run + chmod: "+x" + contents: | + #!/bin/sh + exec 2>&1 + exec sudo -u postgres /usr/lib/postgresql/9.2/bin/postmaster -D /etc/postgresql/9.2/main + + - exec: + cmd: + - chown -R root /var/lib/postgresql/9.2/main + - "[ ! -e /shared/postgres_data ] && cp -r /var/lib/postgresql/9.2/main /shared/postgres_data || exit 0" + - chown -R postgres:postgres /shared/postgres_data + + - replace: + filename: "/etc/postgresql/9.2/main/postgresql.conf" + from: "data_directory = '/var/lib/postgresql/9.2/main'" + to: "data_directory = '/shared/postgres_data'" + + # Necessary to enable backups + - exec: + cmd: + - install -d -m 0755 -o postgres -g postgres /shared/postgres_backup + + - replace: + filename: "/etc/postgresql/9.2/main/postgresql.conf" + from: /#?max_wal_senders *=.*/ + to: "max_wal_senders = 4" + + - replace: + filename: "/etc/postgresql/9.2/main/postgresql.conf" + from: /#?wal_level *=.*/ + to: "wal_level = hot_standby" + + - replace: + filename: "/etc/postgresql/9.2/main/pg_hba.conf" + from: /^#local +replication +postgres +peer$/ + to: "local replication postgres peer" diff --git a/templates/redis.template.yml b/templates/redis.template.yml new file mode 100644 index 0000000..4866a57 --- /dev/null +++ b/templates/redis.template.yml @@ -0,0 +1,51 @@ +run: + - file: + path: /etc/service/redis/run + chmod: "+x" + contents: | + #!/bin/sh + exec 2>&1 + exec sudo -u redis /usr/bin/redis-server /etc/redis/redis.conf + + - replace: + filename: "/etc/redis/redis.conf" + from: "daemonize yes" + to: "" + - replace: + filename: "/etc/redis/redis.conf" + from: /^pidfile.*$/ + to: "" + + - exec: + cmd: + - install -d -m 0755 -o redis -g redis /shared/redis_data + + - replace: + filename: "/etc/redis/redis.conf" + from: /^logfile.*$/ + to: "logfile \"\"" + + - replace: + filename: "/etc/redis/redis.conf" + from: /^dir .*$/ + to: "dir /shared/redis_data" + + - exec: + cmd: echo redis installed + hook: redis + +# we can not migrate without redis, launch it if needed +hooks: + before_code: + - exec: + background: true + cmd: "sudo -u redis /usr/bin/redis-server /etc/redis/redis.conf" + after_code: + - replace: + filename: /etc/service/sidekiq/run + from: "# redis" + to: sv start redis || exit 1 + - replace: + filename: /etc/service/unicorn/run + from: "# redis" + to: sv start redis || exit 1 diff --git a/templates/sshd.template.yml b/templates/sshd.template.yml new file mode 100644 index 0000000..7437625 --- /dev/null +++ b/templates/sshd.template.yml @@ -0,0 +1,19 @@ +params: + # SSH key is required for remote access into the container + ssh_key: YOUR_SSH_KEY_HERE + +run: + + - file: + path: /etc/service/sshd/run + chmod: "+x" + contents: | + #!/bin/sh + exec 2>&1 + exec /usr/sbin/sshd -D -e + + - exec: + cmd: + - mkdir -p /var/run/sshd + - mkdir -p /root/.ssh + - echo $ssh_key >> /root/.ssh/authorized_keys diff --git a/templates/web.template.yml b/templates/web.template.yml new file mode 100644 index 0000000..595c926 --- /dev/null +++ b/templates/web.template.yml @@ -0,0 +1,116 @@ +env: + # Comma delimited list of emails, required if you want admin access for first account + DEVELOPER_EMAILS: 'YOUR_EMAIL@EMAIL.COM1' + # You can have redis on a different box + # REDIS_PROVIDER_URL: 'redis://l.discourse:6379' + RAILS_ENV: 'production' + UNICORN_WORKERS: 3 + # slightly less aggressive than "recommendation" but works fine with oobgc + RUBY_GC_MALLOC_LIMIT: 40000000 + # this ensures we have enough heap space to handle a big pile of small reqs + RUBY_HEAP_MIN_SLOTS: 800000 + +params: + # SSH key is required for remote access into the container + version: HEAD + + home: /var/www/discourse + + # You can connect to any DB you wish to + database_yml: + production: + database: discourse + username: discourse + socket: /var/run/postgresql + password: + host: + host_names: + - YOUR_HOSTNAME_HERE + +run: + + - file: + path: /etc/service/unicorn/run + chmod: "+x" + contents: | + #!/bin/bash + exec 2>&1 + $env + # redis + # postgres + cd $home + exec sudo -E -u discourse LD_PRELOAD=/usr/lib/libjemalloc.so.1 bundle exec config/unicorn_launcher -E production -c config/unicorn.conf.rb + + - file: + path: /etc/service/sidekiq/run + chmod: "+x" + contents: | + #!/bin/bash + exec 2>&1 + $env + # redis + # postgres + cd $home + exec sudo -E -u discourse LD_PRELOAD=/usr/lib/libjemalloc.so.1 bundle exec sidekiq + + - file: + path: /etc/service/nginx/run + chmod: "+x" + contents: | + #!/bin/sh + exec 2>&1 + exec /usr/sbin/nginx + + - exec: + cd: $home + hook: code + cmd: + - git reset --hard + - git clean -f + - git pull + - git checkout $version + - cp config/database.yml.production-sample config/database.yml + - cp config/redis.yml.sample config/redis.yml + - cp config/environments/production.rb.sample config/environments/production.rb + - mkdir -p tmp/pids + - mkdir -p tmp/sockets + - touch tmp/.gitkeep + - mkdir -p /shared/log/rails + - rm -r log + - ln -s /shared/log/rails $home/log + - mkdir -p /shared/uploads + - ln -s /shared/uploads $home/public/uploads + - chown -R discourse /shared/uploads + - chown -R discourse /shared/log/rails + - exec: + cmd: + - "cp $home/config/nginx.sample.conf /etc/nginx/conf.d/discourse.conf" + - "rm /etc/nginx/sites-enabled/default" + + - replace: + filename: /etc/nginx/nginx.conf + from: pid /run/nginx.pid; + to: daemon off; + + - replace: + filename: "/etc/nginx/conf.d/discourse.conf" + from: /upstream[^\}]+\}/m + to: "upstream discourse { + server 127.0.0.1:3000; + }" + + - replace: + filename: "/etc/nginx/conf.d/discourse.conf" + from: /server_name.+$/ + to: server_name _ ; + + - merge: $home/config/database.yml $database_yml + + - exec: + cd: $home + cmd: + - chown -R discourse $home + - sudo -E -u discourse bundle install --deployment --verbose --without test --without development + - sudo -E -u discourse bundle exec rake db:migrate + - sudo -E -u discourse bundle exec rake assets:precompile + -- 2.25.1