From 4bece5f47ef5c5ba3a3bdf4c7cb9232b3f20ce37 Mon Sep 17 00:00:00 2001 From: Alan Guo Xiang Tan Date: Wed, 20 Mar 2024 06:26:36 +0800 Subject: [PATCH] Refactor Github action build workflow to build for both arm64/amd64 (#781) Why this change? Now that we can efficiently build Docker images targeted at `linux/arm64`, we will start to release images for `linux/arm64` in the same way we do for `linux/amd64` images. Images released for `linux/amd64` are tagged as follows: 1. discourse/base:2.0.\-slim 2. discourse/base:slim 3. discourse/base:2.0.\ 4. discourse/base:release For `linux/arm64`, the images are tagged as follows: 1. discourse/base:2.0.\-slim-arm64 2. discourse/base:slim-arm64 3. discourse/base:2.0.\-arm64 4. discourse/base:release-arm64 5. discourse/base:aarch64 (For backwards compatibility) For `linux/arm64`, we unfortunately cannot install chrome because chrome does not currently release binaries for the arch. Therefore, we install chromium which chrome is based off and also install the chromedriver binary for `linux/arm64` released by the electron project. --- .github/workflows/build.yml | 124 ++++++++++++++-------------- image/auto_build.rb | 5 ++ image/discourse_test/Dockerfile | 10 +-- image/discourse_test/install-chrome | 20 +++++ 4 files changed, 90 insertions(+), 69 deletions(-) create mode 100755 image/discourse_test/install-chrome diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7e45fdb..c7f5c6a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -15,36 +15,76 @@ env: jobs: base: - runs-on: ubuntu-20.04${{ ((github.event_name != 'schedule') && '-8core') || '' }} - timeout-minutes: 90 + # `unbuntu-20.04-8core` for arch amd64 non-scheduled builds + # `unbuntu-20.04` for arch amd64 scheduled builds + # `unbuntu-20.04-8core-arm` for arch arm64 non-scheduled builds + # `unbuntu-20.04-2core-arm` for arch arm64 scheduled builds + runs-on: ubuntu-20.04${{ ((github.event_name != 'schedule') && '-8core') || (( matrix.arch == 'arm64' && '-2core' ) || '') }}${{ (matrix.arch == 'arm64') && '-arm' || '' }} + strategy: + matrix: + arch: [amd64, arm64] + timeout-minutes: 30 steps: + - name: Install Docker + if: ${{ matrix.arch == 'arm64' }} + run: | + # Add Docker's official GPG key: + sudo apt-get update + sudo apt-get install ca-certificates curl + sudo install -m 0755 -d /etc/apt/keyrings + sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc + sudo chmod a+r /etc/apt/keyrings/docker.asc + + # Add the repository to Apt sources: + echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \ + $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \ + sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + sudo apt-get update + + # Install Docker + sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin + + # Give the current user permission to run docker without sudo + sudo usermod -aG docker $USER + sudo apt-get install -y acl + sudo setfacl --modify user:$USER:rw /var/run/docker.sock + - name: Install Ruby + if: ${{ matrix.arch == 'arm64' }} + run: | + sudo apt-get install -y ruby - uses: actions/checkout@v3 with: fetch-depth: 1 + - name: Set arch helper output + id: arch-helper + run: | + echo "arch_postfix_dash=${{ (matrix.arch == 'arm64' && '-arm64') || '' }}" >> $GITHUB_OUTPUT + echo "arch_postfix_underscore=${{ (matrix.arch == 'arm64' && '_arm64') || '' }}" >> $GITHUB_OUTPUT - name: build slim image run: | - cd image && ruby auto_build.rb base_slim + cd image && ruby auto_build.rb base_slim${{ steps.arch-helper.outputs.arch_postfix_underscore }} - name: tag slim images id: tag-images run: | TAG=`date +%Y%m%d-%H%M` echo "tag=$(echo $TAG)" >> $GITHUB_OUTPUT - docker tag discourse/base:build_slim discourse/base:2.0.$TAG-slim - docker tag discourse/base:build_slim discourse/base:slim + docker tag discourse/base:build_slim${{ steps.arch-helper.outputs.arch_postfix_underscore }} discourse/base:2.0.$TAG-slim${{ steps.arch-helper.outputs.arch_postfix_dash }} + docker tag discourse/base:build_slim${{ steps.arch-helper.outputs.arch_postfix_underscore }} discourse/base:slim${{ steps.arch-helper.outputs.arch_postfix_dash }} - name: build release image run: | - cd image && ruby auto_build.rb base - - name: tag release images + cd image && ruby auto_build.rb base${{ steps.arch-helper.outputs.arch_postfix_underscore }} + - name: tag amd64 release images run: | TAG=${{ steps.tag-images.outputs.tag }} - docker tag discourse/base:build discourse/base:2.0.$TAG - docker tag discourse/base:build discourse/base:release + docker tag discourse/base:build${{ steps.arch-helper.outputs.arch_postfix_underscore }} discourse/base:2.0.$TAG${{ steps.arch-helper.outputs.arch_postfix_dash }} + docker tag discourse/base:build${{ steps.arch-helper.outputs.arch_postfix_underscore }} discourse/base:release${{ steps.arch-helper.outputs.arch_postfix_dash }} - name: build test_build image run: | - cd image && ruby auto_build.rb discourse_test_build + cd image && ruby auto_build.rb discourse_test_build${{ steps.arch-helper.outputs.arch_postfix_underscore }} - name: run specs run: | - docker run --rm -e RUBY_ONLY=1 -e USE_TURBO=1 -e SKIP_PLUGINS=1 -e SKIP_LINT=1 discourse/discourse_test:build + docker run --rm -e RUBY_ONLY=1 -e USE_TURBO=1 -e SKIP_PLUGINS=1 -e SKIP_LINT=1 discourse/discourse_test:build${{ steps.arch-helper.outputs.arch_postfix_underscore }} - name: Print summary run: | docker images discourse/base @@ -55,10 +95,15 @@ jobs: run: | TAG=${{ steps.tag-images.outputs.tag }} docker login --username discoursebuild --password $DOCKERHUB_PASSWORD - docker push discourse/base:2.0.$TAG-slim - docker push discourse/base:slim - docker push discourse/base:2.0.$TAG - docker push discourse/base:release + docker push discourse/base:2.0.$TAG-slim${{ steps.arch-helper.outputs.arch_postfix_dash }} + docker push discourse/base:slim${{ steps.arch-helper.outputs.arch_postfix_dash }} + docker push discourse/base:2.0.$TAG${{ steps.arch-helper.outputs.arch_postfix_dash }} + docker push discourse/base:release${{ steps.arch-helper.outputs.arch_postfix_dash }} + - name: Push discourse/base:aarch64 image for backwards compatibility + if: success() && (github.ref == 'refs/heads/main') && (matrix.arch == 'arm64') + run: | + docker tag discourse/base:release${{ steps.arch-helper.outputs.arch_postfix_dash }} discourse/base:aarch64 + docker push discourse/base:aarch64 test: runs-on: ubuntu-20.04${{ ((github.event_name != 'schedule') && '-8core') || '' }} timeout-minutes: 30 @@ -119,52 +164,3 @@ jobs: docker tag discourse/discourse_dev:build discourse/discourse_dev:release docker login --username discoursebuild --password $DOCKERHUB_PASSWORD docker push discourse/discourse_dev:release - aarch64: - runs-on: ubuntu-20.04${{ ((github.event_name != 'schedule') && '-8core') || '-2core' }}-arm - needs: base - steps: - - name: Install Docker - run: | - # Add Docker's official GPG key: - sudo apt-get update - sudo apt-get install ca-certificates curl - sudo install -m 0755 -d /etc/apt/keyrings - sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc - sudo chmod a+r /etc/apt/keyrings/docker.asc - # Add the repository to Apt sources: - echo \ - "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \ - $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \ - sudo tee /etc/apt/sources.list.d/docker.list > /dev/null - sudo apt-get update - # Install Docker - sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin - # Give the current user permission to run docker without sudo - sudo usermod -aG docker $USER - sudo apt-get install -y acl - sudo setfacl --modify user:$USER:rw /var/run/docker.sock - - name: Install Ruby - run: | - sudo apt-get install -y ruby - - uses: actions/checkout@v3 - with: - fetch-depth: 1 - - name: build slim image - run: | - cd image && ruby auto_build.rb base_slim_arm64 - - name: build release image - run: | - cd image && ruby auto_build.rb base_arm64 - - name: tag release image as release - working-directory: image/base - run: | - docker tag discourse/base:build_arm64 discourse/base:aarch64 - - name: Print summary - run: docker images discourse/base - - name: push to dockerhub - if: success() && (github.ref == 'refs/heads/main') - env: - DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} - run: | - docker login --username discoursebuild --password $DOCKERHUB_PASSWORD - docker push discourse/base:aarch64 diff --git a/image/auto_build.rb b/image/auto_build.rb index 7c7d215..b6d9370 100644 --- a/image/auto_build.rb +++ b/image/auto_build.rb @@ -28,6 +28,11 @@ images = { name: "discourse_test", tag: "discourse/discourse_test:build", }, + discourse_test_build_arm64: { + name: "discourse_test", + tag: "discourse/discourse_test:build_arm64", + extra_args: "--platform linux/arm64 --build-arg=\"from_tag=build_arm64\"", + }, discourse_dev: { name: "discourse_dev", tag: "discourse/discourse_dev:build", diff --git a/image/discourse_test/Dockerfile b/image/discourse_test/Dockerfile index 391b293..3558705 100644 --- a/image/discourse_test/Dockerfile +++ b/image/discourse_test/Dockerfile @@ -17,10 +17,10 @@ RUN chown -R discourse . &&\ FROM base AS with_browsers -RUN wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | sudo apt-key add - &&\ - echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list &&\ +ADD install-chrome /tmp/install-chrome +RUN /tmp/install-chrome &&\ apt update &&\ - apt install -y libgconf-2-4 libxss1 google-chrome-stable firefox-esr &&\ + apt install -y libgconf-2-4 libxss1 firefox-esr &&\ cd /tmp && wget -q "https://download.mozilla.org/?product=firefox-latest-ssl&os=linux64&lang=en-US" -O firefox.tar.bz2 &&\ tar xjvf firefox.tar.bz2 && mv /tmp/firefox /opt/firefox-evergreen &&\ apt clean @@ -28,7 +28,7 @@ RUN wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | sudo ap FROM with_browsers AS release RUN cd /var/www/discourse &&\ - sudo -u discourse bundle install --jobs=4 &&\ + sudo -u discourse bundle install &&\ sudo -E -u discourse -H yarn install &&\ sudo -u discourse yarn cache clean @@ -36,4 +36,4 @@ RUN cd /var/www/discourse && sudo -E -u discourse -H bundle exec rake plugin:ins LOAD_PLUGINS=1 sudo -E -u discourse -H bundle exec rake plugin:install_all_gems &&\ sudo -E -u discourse -H bundle exec ruby script/install_minio_binaries.rb -ENTRYPOINT ["sudo", "-E", "-u", "discourse", "-H", "ruby", "script/docker_test.rb"] +ENTRYPOINT ["sudo", "-E", "-u", "discourse", "-H", "ruby", "script/docker_test.rb"] \ No newline at end of file diff --git a/image/discourse_test/install-chrome b/image/discourse_test/install-chrome new file mode 100755 index 0000000..b6e559e --- /dev/null +++ b/image/discourse_test/install-chrome @@ -0,0 +1,20 @@ +#!/bin/bash +set -e + +# https://googlechromelabs.github.io/chrome-for-testing/ doesn't provide linux/arm64 binaries for chrome or chromedriver +# yet. Therefore on arm64, we install chromium instead of chrome and install a chromedriver for linux/arm64 from +# https://github.com/electron/electron/releases/. +# +# On the current debian Chromium 120.0.6099.224 is installed so we have to install a chromedriver that is of the same version. +if [ "$(dpkg --print-architecture)" = "arm64" ]; then + apt update && apt install -y chromium unzip &&\ + wget -q -O /tmp/chromedriver.zip https://github.com/electron/electron/releases/download/v28.2.2/chromedriver-v28.2.2-linux-arm64.zip &&\ + unzip /tmp/chromedriver.zip -d /tmp/chromedriver &&\ + mv /tmp/chromedriver/chromedriver /usr/bin &&\ + rm -rf /tmp/chromedriver /tmp/chromedriver.zip +else + wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | sudo apt-key add - &&\ + echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list &&\ + apt update &&\ + apt install -y google-chrome-stable +fi -- 2.25.1