Update base image and default to postgres 13. (#493)

FEATURE: update to PostgreSQL 13.

* postgres.template.yml: update to new major version of 13.

This changes the default postgres version to 13 for the postgres
template.

* images/base: bump postgres to version 13.
* postgres.13.template.yml: add postgres 13 template.
* postgres.12.template: add step to remove pg 13 install.
* launcher: increase timeout allowed for docker stop.

With large databases Postgres may take some time to stop gracefully, so
increase the allowed timeout.

* launcher: use temporary pg13 image while build is running.
Note this will be updated once the image change lands in master.

diff --git a/image/base/Dockerfile b/image/base/Dockerfile
index 724025f204204e46dee18e6dfafac26520d5c90b..dae8bf0670f436c4d65616bf2ca5f31dea1113ac 100644 (file)
--- a/image/base/Dockerfile
+++ b/image/base/Dockerfile
@@ -2,7 +2,7 @@
 # VERSION:  release
 FROM debian:buster-slim
 
-ENV PG_MAJOR 12
+ENV PG_MAJOR 13
 ENV RUBY_ALLOCATOR /usr/lib/libjemalloc.so.1
 ENV RAILS_ENV production
 

diff --git a/launcher b/launcher
index bfcd5af7e07b96cff84b9e00e0daa500950a6fca..4328fd10b5e84f959d411f3e453626772a46de2e 100755 (executable)
--- a/launcher
+++ b/launcher
@@ -90,7 +90,7 @@ git_rec_version='1.8.0'
 config_file=containers/"$config".yml
 cidbootstrap=cids/"$config"_bootstrap.cid
 local_discourse=local_discourse
-image="discourse/base:2.0.20201125-2246"
+image="discourse/base:pg13"
 docker_path=`which docker.io 2> /dev/null || which docker`
 git_path=`which git`
 
@@ -524,7 +524,7 @@ run_stop() {
      then
        (
         set -x
-        $docker_path stop -t 10 $config
+        $docker_path stop -t 30 $config
        )
      else
        echo "$config was not started !"

diff --git a/templates/postgres.12.template.yml b/templates/postgres.12.template.yml
index 7a29faaa09cfacd44520bfc47ecaf02be57fa16d..1db6c140132dac106410b85714b099c826d6a79f 100644 (file)
--- a/templates/postgres.12.template.yml
+++ b/templates/postgres.12.template.yml
@@ -17,6 +17,8 @@ hooks:
        to: sv start postgres || exit 1
 
 run:
+  - exec: DEBIAN_FRONTEND=noninteractive apt-get purge -y postgresql-13 postgresql-client-13 postgresql-contrib-13
+  - exec: apt-get update && apt-get install -y postgresql-12 postgresql-client-12 postgresql-contrib-12
   - exec: mkdir -p /shared/postgres_run
   - exec: chown postgres:postgres /shared/postgres_run
   - exec: chmod 775 /shared/postgres_run

diff --git a/templates/postgres.13.template.yml b/templates/postgres.13.template.yml
new file mode 100644 (file)
index 0000000..f7e7899
--- /dev/null
+++ b/templates/postgres.13.template.yml
@@ -0,0 +1,238 @@
+params:
+  db_synchronous_commit: "off"
+  db_shared_buffers: "256MB"
+  db_work_mem: "10MB"
+  db_default_text_search_config: "pg_catalog.english"
+  db_name: discourse
+  db_user: discourse
+  db_checkpoint_segments: 6
+  db_logging_collector: off
+  db_log_min_duration_statement: 100
+
+hooks:
+  before_code:
+    - replace:
+       filename: /etc/service/unicorn/run
+       from: "# postgres"
+       to: sv start postgres || exit 1
+
+run:
+  - exec: locale-gen $LANG && update-locale
+  - exec: mkdir -p /shared/postgres_run
+  - exec: chown postgres:postgres /shared/postgres_run
+  - exec: chmod 775 /shared/postgres_run
+  - exec: rm -fr /var/run/postgresql
+  - exec: ln -s /shared/postgres_run /var/run/postgresql
+  - exec: socat /dev/null UNIX-CONNECT:/shared/postgres_run/.s.PGSQL.5432 || exit 0 && echo postgres already running stop container ; exit 1
+  - exec: rm -fr /shared/postgres_run/.s*
+  - exec: rm -fr /shared/postgres_run/*.pid
+  - exec: mkdir -p /shared/postgres_run/13-main.pg_stat_tmp
+  - exec: chown postgres:postgres /shared/postgres_run/13-main.pg_stat_tmp
+  - file:
+     path: /etc/service/postgres/run
+     chmod: "+x"
+     contents: |
+        #!/bin/sh
+        exec 2>&1
+        HOME=/var/lib/postgresql USER=postgres exec thpoff chpst -u postgres:postgres:ssl-cert -U postgres:postgres:ssl-cert /usr/lib/postgresql/13/bin/postmaster -D /etc/postgresql/13/main
+
+  - file:
+     path: /etc/service/postgres/log/run
+     chmod: "+x"
+     contents: |
+       #!/bin/sh
+       mkdir -p /var/log/postgres
+       exec svlogd /var/log/postgres
+
+  - file:
+     path: /etc/runit/3.d/99-postgres
+     chmod: "+x"
+     contents: |
+       #!/bin/bash
+       sv stop postgres
+
+  - file:
+     path: /root/upgrade_postgres
+     chmod: "+x"
+     contents: |
+       #!/bin/bash
+       PG_MAJOR_OLD=`cat /shared/postgres_data/PG_VERSION`
+
+       if [ ! "13" = "$PG_MAJOR_OLD" ]; then
+         echo Upgrading PostgreSQL from version ${PG_MAJOR_OLD} to 13
+         free_disk=$(df -P -B1 /shared | tail -n 1 | awk '{print $4}')
+         required=$(($(du -sb /shared/postgres_data | awk '{print $1}') * 2))
+
+         if [ "$free_disk" -lt "$required" ]; then
+          echo "WARNING: Upgrading PostgresSQL would require an addtional $(numfmt --to=si $(($required - $free_disk))) of disk space"
+          echo "Please free up some space, or expand your disk, before continuing."
+          echo ''
+          echo 'To avoid upgrading change "templates/postgres.template.yml" TO "templates/postgres.12.template.yml" in containers/app.yml'
+          exit 1
+         fi
+
+         if [ -d /shared/postgres_data_old ]; then
+           mv /shared/postgres_data_old /shared/postgres_data_older
+         fi
+
+         rm -fr /shared/postgres_data_new
+         install -d -m 0755 -o postgres -g postgres /shared/postgres_data_new && sudo -u postgres /usr/lib/postgresql/13/bin/initdb -D /shared/postgres_data_new || exit 0
+         apt-get update
+         apt-get install -y postgresql-${PG_MAJOR_OLD}
+         chown -R postgres:postgres /var/lib/postgresql/13
+         /etc/init.d/postgresql stop
+         rm -fr /shared/postgres_data/postmaster.pid
+         cd ~postgres
+         cp -pr /etc/postgresql/${PG_MAJOR_OLD}/main/* /shared/postgres_data
+         echo  >> /shared/postgres_data/postgresql.conf
+         echo "data_directory = '/shared/postgres_data'" >> /shared/postgres_data/postgresql.conf
+         SUCCESS=true
+         sudo -u postgres /usr/lib/postgresql/13/bin/pg_upgrade -d /shared/postgres_data -D /shared/postgres_data_new -b /usr/lib/postgresql/${PG_MAJOR_OLD}/bin -B /usr/lib/postgresql/13/bin || SUCCESS=false
+
+         if [[ "$SUCCESS" == 'false' ]]; then
+           echo -------------------------------------------------------------------------------------
+           echo UPGRADE OF POSTGRES FAILED
+           echo
+           echo Please visit https://meta.discourse.org/t/postgresql-13-update/172563 for support.
+           echo
+           echo You can run "./launcher start app" to restart your app in the meanwhile
+           echo
+           exit 1
+         fi
+
+         mv /shared/postgres_data /shared/postgres_data_old
+         mv /shared/postgres_data_new /shared/postgres_data
+
+         echo -------------------------------------------------------------------------------------
+         echo UPGRADE OF POSTGRES COMPLETE
+         echo
+         echo Old ${PG_MAJOR_OLD} database is stored at /shared/postgres_data_old
+         echo
+         echo To complete the upgrade, rebuild again using:
+         echo
+         echo     ./launcher rebuild app
+         echo -------------------------------------------------------------------------------------
+         # Magic exit status to denote no failure
+         exit 77
+       fi
+
+  - exec:
+      cmd:
+        - chown -R root /var/lib/postgresql/13/main
+        - "[ ! -e /shared/postgres_data ] && install -d -m 0755 -o postgres -g postgres /shared/postgres_data && sudo -E -u postgres /usr/lib/postgresql/13/bin/initdb -D /shared/postgres_data || exit 0"
+        - chown -R postgres:postgres /shared/postgres_data
+        - chown -R postgres:postgres /var/run/postgresql
+
+  - exec: /root/upgrade_postgres
+  - exec: rm /root/upgrade_postgres
+
+  - replace:
+      filename: "/etc/postgresql/13/main/postgresql.conf"
+      from: "data_directory = '/var/lib/postgresql/13/main'"
+      to: "data_directory = '/shared/postgres_data'"
+
+  # listen on all interfaces
+  - replace:
+      filename: "/etc/postgresql/13/main/postgresql.conf"
+      from: /#?listen_addresses *=.*/
+      to: "listen_addresses = '*'"
+
+  # sync commit off is faster and less spiky, also marginally less safe
+  - replace:
+      filename: "/etc/postgresql/13/main/postgresql.conf"
+      from: /#?synchronous_commit *=.*/
+      to: "synchronous_commit = $db_synchronous_commit"
+
+  # default is 128MB which is way too small
+  - replace:
+      filename: "/etc/postgresql/13/main/postgresql.conf"
+      from: /#?shared_buffers *=.*/
+      to: "shared_buffers = $db_shared_buffers"
+
+  # default is 1MB which is too small
+  - replace:
+      filename: "/etc/postgresql/13/main/postgresql.conf"
+      from: /#?work_mem *=.*/
+      to: "work_mem = $db_work_mem"
+
+  # allow for other
+  - replace:
+      filename: "/etc/postgresql/13/main/postgresql.conf"
+      from: /#?default_text_search_config *=.*/
+      to: "default_text_search_config = '$db_default_text_search_config'"
+
+  # Necessary to enable backups
+  - exec:
+      cmd:
+        - install -d -m 0755 -o postgres -g postgres /shared/postgres_backup
+
+  - replace:
+      filename: "/etc/postgresql/13/main/postgresql.conf"
+      from: /#?checkpoint_segments *=.*/
+      to: "checkpoint_segments = $db_checkpoint_segments"
+
+  - replace:
+      filename: "/etc/postgresql/13/main/postgresql.conf"
+      from: /#?logging_collector *=.*/
+      to: "logging_collector = $db_logging_collector"
+
+  - replace:
+      filename: "/etc/postgresql/13/main/postgresql.conf"
+      from: /#?log_min_duration_statement *=.*/
+      to: "log_min_duration_statement = $db_log_min_duration_statement"
+
+  - replace:
+      filename: "/etc/postgresql/13/main/pg_hba.conf"
+      from: /^#local +replication +postgres +peer$/
+      to: "local replication postgres  peer"
+
+  # allow all to connect in with md5 auth
+  - replace:
+      filename: "/etc/postgresql/13/main/pg_hba.conf"
+      from: /^host.*all.*all.*137.*$/
+      to: "host all all 0.0.0.0/0 md5"
+
+  - exec:
+      background: true
+      # use fast shutdown for pg
+      stop_signal: INT
+      cmd: HOME=/var/lib/postgresql USER=postgres exec chpst -u postgres:postgres:ssl-cert -U postgres:postgres:ssl-cert /usr/lib/postgresql/13/bin/postmaster -D /etc/postgresql/13/main
+
+  # give db a few secs to start up
+  - exec: "sleep 5"
+
+  - exec: su postgres -c 'createdb $db_name' || true
+  - exec: su postgres -c 'psql $db_name -c "create user $db_user;"' || true
+  - exec: su postgres -c 'psql $db_name -c "grant all privileges on database $db_name to $db_user;"' || true
+  - exec: su postgres -c 'psql $db_name -c "alter schema public owner to $db_user;"'
+  - exec: su postgres -c 'psql template1 -c "create extension if not exists hstore;"'
+  - exec: su postgres -c 'psql template1 -c "create extension if not exists pg_trgm;"'
+  - exec: su postgres -c 'psql $db_name -c "create extension if not exists hstore;"'
+  - exec: su postgres -c 'psql $db_name -c "create extension if not exists pg_trgm;"'
+  - exec:
+      stdin: |
+        update pg_database set encoding = pg_char_to_encoding('UTF8') where datname = '$db_name' AND encoding = pg_char_to_encoding('SQL_ASCII');
+      cmd: sudo -u postgres psql $db_name
+      raise_on_fail: false
+
+  - file:
+     path: /var/lib/postgresql/take-database-backup
+     chown: postgres:postgres
+     chmod: "+x"
+     contents: |
+        #!/bin/bash
+        ID=db-$(date +%F_%T)
+        FILENAME=/shared/postgres_backup/$ID.tar.gz
+        pg_basebackup --format=tar --pgdata=- --xlog --gzip --label=$ID > $FILENAME
+        echo $FILENAME
+
+  - file:
+     path: /var/spool/cron/crontabs/postgres
+     contents: |
+        # m h  dom mon dow   command
+        #MAILTO=?
+        #0 */4 * * * /var/lib/postgresql/take-database-backup
+
+  - exec:
+      hook: postgres
+      cmd: "echo postgres installed!"

diff --git a/templates/postgres.template.yml b/templates/postgres.template.yml
index 2ccecde6df7fd057b51cdbd7e77714d370faf09b..814fa154fee3092c235db5cdd34012d9aeef3a35 100644 (file)
--- a/templates/postgres.template.yml
+++ b/templates/postgres.template.yml
@@ -26,15 +26,15 @@ run:
   - exec: socat /dev/null UNIX-CONNECT:/shared/postgres_run/.s.PGSQL.5432 || exit 0 && echo postgres already running stop container ; exit 1
   - exec: rm -fr /shared/postgres_run/.s*
   - exec: rm -fr /shared/postgres_run/*.pid
-  - exec: mkdir -p /shared/postgres_run/12-main.pg_stat_tmp
-  - exec: chown postgres:postgres /shared/postgres_run/12-main.pg_stat_tmp
+  - exec: mkdir -p /shared/postgres_run/13-main.pg_stat_tmp
+  - exec: chown postgres:postgres /shared/postgres_run/13-main.pg_stat_tmp
   - file:
      path: /etc/service/postgres/run
      chmod: "+x"
      contents: |
         #!/bin/sh
         exec 2>&1
-        HOME=/var/lib/postgresql USER=postgres exec thpoff chpst -u postgres:postgres:ssl-cert -U postgres:postgres:ssl-cert /usr/lib/postgresql/12/bin/postmaster -D /etc/postgresql/12/main
+        HOME=/var/lib/postgresql USER=postgres exec thpoff chpst -u postgres:postgres:ssl-cert -U postgres:postgres:ssl-cert /usr/lib/postgresql/13/bin/postmaster -D /etc/postgresql/13/main
 
   - file:
      path: /etc/service/postgres/log/run
@@ -58,8 +58,8 @@ run:
        #!/bin/bash
        PG_MAJOR_OLD=`cat /shared/postgres_data/PG_VERSION`
 
-       if [ ! "12" = "$PG_MAJOR_OLD" ]; then
-         echo Upgrading PostgreSQL from version ${PG_MAJOR_OLD} to 12
+       if [ ! "13" = "$PG_MAJOR_OLD" ]; then
+         echo Upgrading PostgreSQL from version ${PG_MAJOR_OLD} to 13
          free_disk=$(df -P -B1 /shared | tail -n 1 | awk '{print $4}')
          required=$(($(du -sb /shared/postgres_data | awk '{print $1}') * 2))
 
@@ -67,7 +67,7 @@ run:
           echo "WARNING: Upgrading PostgresSQL would require an addtional $(numfmt --to=si $(($required - $free_disk))) of disk space"
           echo "Please free up some space, or expand your disk, before continuing."
           echo ''
-          echo 'To avoid upgrading change "templates/postgres.template.yml" TO "templates/postgres.10.template.yml" in containers/app.yml'
+          echo 'To avoid upgrading change "templates/postgres.template.yml" TO "templates/postgres.12.template.yml" in containers/app.yml'
           exit 1
          fi
 
@@ -76,10 +76,10 @@ run:
          fi
 
          rm -fr /shared/postgres_data_new
-         install -d -m 0755 -o postgres -g postgres /shared/postgres_data_new && sudo -u postgres /usr/lib/postgresql/12/bin/initdb -D /shared/postgres_data_new || exit 0
+         install -d -m 0755 -o postgres -g postgres /shared/postgres_data_new && sudo -u postgres /usr/lib/postgresql/13/bin/initdb -D /shared/postgres_data_new || exit 0
          apt-get update
          apt-get install -y postgresql-${PG_MAJOR_OLD}
-         chown -R postgres:postgres /var/lib/postgresql/12
+         chown -R postgres:postgres /var/lib/postgresql/13
          /etc/init.d/postgresql stop
          rm -fr /shared/postgres_data/postmaster.pid
          cd ~postgres
@@ -87,13 +87,13 @@ run:
          echo  >> /shared/postgres_data/postgresql.conf
          echo "data_directory = '/shared/postgres_data'" >> /shared/postgres_data/postgresql.conf
          SUCCESS=true
-         sudo -u postgres /usr/lib/postgresql/12/bin/pg_upgrade -d /shared/postgres_data -D /shared/postgres_data_new -b /usr/lib/postgresql/${PG_MAJOR_OLD}/bin -B /usr/lib/postgresql/12/bin || SUCCESS=false
+         sudo -u postgres /usr/lib/postgresql/13/bin/pg_upgrade -d /shared/postgres_data -D /shared/postgres_data_new -b /usr/lib/postgresql/${PG_MAJOR_OLD}/bin -B /usr/lib/postgresql/13/bin || SUCCESS=false
 
          if [[ "$SUCCESS" == 'false' ]]; then
            echo -------------------------------------------------------------------------------------
            echo UPGRADE OF POSTGRES FAILED
            echo
-           echo Please visit https://meta.discourse.org/t/postgresql-12-update/151236 for support
+           echo Please visit https://meta.discourse.org/t/postgresql-13-update/172563 for support.
            echo
            echo You can run "./launcher start app" to restart your app in the meanwhile
            echo
@@ -118,8 +118,8 @@ run:
 
   - exec:
       cmd:
-        - chown -R root /var/lib/postgresql/12/main
-        - "[ ! -e /shared/postgres_data ] && install -d -m 0755 -o postgres -g postgres /shared/postgres_data && sudo -E -u postgres /usr/lib/postgresql/12/bin/initdb -D /shared/postgres_data || exit 0"
+        - chown -R root /var/lib/postgresql/13/main
+        - "[ ! -e /shared/postgres_data ] && install -d -m 0755 -o postgres -g postgres /shared/postgres_data && sudo -E -u postgres /usr/lib/postgresql/13/bin/initdb -D /shared/postgres_data || exit 0"
         - chown -R postgres:postgres /shared/postgres_data
         - chown -R postgres:postgres /var/run/postgresql
 
@@ -127,37 +127,37 @@ run:
   - exec: rm /root/upgrade_postgres
 
   - replace:
-      filename: "/etc/postgresql/12/main/postgresql.conf"
-      from: "data_directory = '/var/lib/postgresql/12/main'"
+      filename: "/etc/postgresql/13/main/postgresql.conf"
+      from: "data_directory = '/var/lib/postgresql/13/main'"
       to: "data_directory = '/shared/postgres_data'"
 
   # listen on all interfaces
   - replace:
-      filename: "/etc/postgresql/12/main/postgresql.conf"
+      filename: "/etc/postgresql/13/main/postgresql.conf"
       from: /#?listen_addresses *=.*/
       to: "listen_addresses = '*'"
 
   # sync commit off is faster and less spiky, also marginally less safe
   - replace:
-      filename: "/etc/postgresql/12/main/postgresql.conf"
+      filename: "/etc/postgresql/13/main/postgresql.conf"
       from: /#?synchronous_commit *=.*/
       to: "synchronous_commit = $db_synchronous_commit"
 
   # default is 128MB which is way too small
   - replace:
-      filename: "/etc/postgresql/12/main/postgresql.conf"
+      filename: "/etc/postgresql/13/main/postgresql.conf"
       from: /#?shared_buffers *=.*/
       to: "shared_buffers = $db_shared_buffers"
 
   # default is 1MB which is too small
   - replace:
-      filename: "/etc/postgresql/12/main/postgresql.conf"
+      filename: "/etc/postgresql/13/main/postgresql.conf"
       from: /#?work_mem *=.*/
       to: "work_mem = $db_work_mem"
 
   # allow for other
   - replace:
-      filename: "/etc/postgresql/12/main/postgresql.conf"
+      filename: "/etc/postgresql/13/main/postgresql.conf"
       from: /#?default_text_search_config *=.*/
       to: "default_text_search_config = '$db_default_text_search_config'"
 
@@ -167,28 +167,28 @@ run:
         - install -d -m 0755 -o postgres -g postgres /shared/postgres_backup
 
   - replace:
-      filename: "/etc/postgresql/12/main/postgresql.conf"
+      filename: "/etc/postgresql/13/main/postgresql.conf"
       from: /#?checkpoint_segments *=.*/
       to: "checkpoint_segments = $db_checkpoint_segments"
 
   - replace:
-      filename: "/etc/postgresql/12/main/postgresql.conf"
+      filename: "/etc/postgresql/13/main/postgresql.conf"
       from: /#?logging_collector *=.*/
       to: "logging_collector = $db_logging_collector"
 
   - replace:
-      filename: "/etc/postgresql/12/main/postgresql.conf"
+      filename: "/etc/postgresql/13/main/postgresql.conf"
       from: /#?log_min_duration_statement *=.*/
       to: "log_min_duration_statement = $db_log_min_duration_statement"
 
   - replace:
-      filename: "/etc/postgresql/12/main/pg_hba.conf"
+      filename: "/etc/postgresql/13/main/pg_hba.conf"
       from: /^#local +replication +postgres +peer$/
       to: "local replication postgres  peer"
 
   # allow all to connect in with md5 auth
   - replace:
-      filename: "/etc/postgresql/12/main/pg_hba.conf"
+      filename: "/etc/postgresql/13/main/pg_hba.conf"
       from: /^host.*all.*all.*127.*$/
       to: "host all all 0.0.0.0/0 md5"
 
@@ -196,7 +196,7 @@ run:
       background: true
       # use fast shutdown for pg
       stop_signal: INT
-      cmd: HOME=/var/lib/postgresql USER=postgres exec chpst -u postgres:postgres:ssl-cert -U postgres:postgres:ssl-cert /usr/lib/postgresql/12/bin/postmaster -D /etc/postgresql/12/main
+      cmd: HOME=/var/lib/postgresql USER=postgres exec chpst -u postgres:postgres:ssl-cert -U postgres:postgres:ssl-cert /usr/lib/postgresql/13/bin/postmaster -D /etc/postgresql/13/main
 
   # give db a few secs to start up
   - exec: "sleep 5"