From fc61b8a850e0a7f8c1ef0852a13d735121b4d862 Mon Sep 17 00:00:00 2001 From: David Taylor Date: Mon, 26 Aug 2024 06:10:28 +0100 Subject: [PATCH] DEV: Introduce arm64 dev image and multi-arch manifests (#829) - Combines dev image build into the `base` job - Calculates a single timestamp for all builds - Pushes timestamped per-arch images to Dockerhub for base, base-slim and dev images - If both arch build jobs are successful, multiarch manifests are generated & pushed --- .github/workflows/build.yml | 127 +++++++++++++++++++++++------------- image/auto_build.rb | 27 +++++--- 2 files changed, 99 insertions(+), 55 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4f14d9a..c1d4f76 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -14,7 +14,19 @@ env: BUILDKIT_PROGRESS: plain DISCOURSE_TURBO_RSPEC_RETRY_AND_LOG_FLAKY_TESTS: true + jobs: + timestamp: + runs-on: ubuntu-latest + outputs: + timestamp: ${{ steps.timestamp.outputs.timestamp }} + steps: + - id: timestamp + run: | + timestamp=`date +%Y%m%d-%H%M` + echo "timestamp=$timestamp" + echo "timestamp=$timestamp" >> $GITHUB_OUTPUT + base: # `unbuntu-22.04-8core` for arch amd64 non-scheduled builds # `unbuntu-22.04` for arch amd64 scheduled builds @@ -25,58 +37,102 @@ jobs: matrix: arch: [amd64, arm64] timeout-minutes: ${{ (github.event_name != 'schedule' && 30) || ((matrix.arch == 'arm64' && 60) || 30) }} + needs: timestamp + env: + ARCH: ${{matrix.arch}} + TIMESTAMP: ${{ needs.timestamp.outputs.timestamp }} steps: - uses: actions/checkout@v3 with: fetch-depth: 1 - - name: Set arch helper output - id: arch-helper - run: | - echo "arch_postfix_dash=${{ (matrix.arch == 'arm64' && '-arm64') || '' }}" >> $GITHUB_OUTPUT - echo "arch_postfix_underscore=${{ (matrix.arch == 'arm64' && '_arm64') || '' }}" >> $GITHUB_OUTPUT - name: build slim image run: | - cd image && ruby auto_build.rb base_slim${{ steps.arch-helper.outputs.arch_postfix_underscore }} + cd image && ruby auto_build.rb base_slim_$ARCH - name: tag slim images - id: tag-images run: | - TAG=`date +%Y%m%d-%H%M` - echo "tag=$(echo $TAG)" >> $GITHUB_OUTPUT - docker tag discourse/base:build_slim${{ steps.arch-helper.outputs.arch_postfix_underscore }} discourse/base:2.0.$TAG-slim${{ steps.arch-helper.outputs.arch_postfix_dash }} - docker tag discourse/base:build_slim${{ steps.arch-helper.outputs.arch_postfix_underscore }} discourse/base:slim${{ steps.arch-helper.outputs.arch_postfix_dash }} + docker tag discourse/base:build_slim_$ARCH discourse/base:2.0.$TIMESTAMP-slim-$ARCH + docker tag discourse/base:build_slim_$ARCH discourse/base:slim-$ARCH - name: build release image run: | - cd image && ruby auto_build.rb base${{ steps.arch-helper.outputs.arch_postfix_underscore }} - - name: tag amd64 release images + cd image && ruby auto_build.rb base_$ARCH + - name: tag release images run: | - TAG=${{ steps.tag-images.outputs.tag }} - docker tag discourse/base:build${{ steps.arch-helper.outputs.arch_postfix_underscore }} discourse/base:2.0.$TAG${{ steps.arch-helper.outputs.arch_postfix_dash }} - docker tag discourse/base:build${{ steps.arch-helper.outputs.arch_postfix_underscore }} discourse/base:release${{ steps.arch-helper.outputs.arch_postfix_dash }} + docker tag discourse/base:build_$ARCH discourse/base:2.0.$TIMESTAMP-$ARCH + docker tag discourse/base:build_$ARCH discourse/base:release-$ARCH - name: build test_build image run: | - cd image && ruby auto_build.rb discourse_test_build${{ steps.arch-helper.outputs.arch_postfix_underscore }} + cd image && ruby auto_build.rb discourse_test_build_$ARCH - name: run specs run: | - docker run --rm -e RUBY_ONLY=1 -e USE_TURBO=1 -e SKIP_PLUGINS=1 -e SKIP_LINT=1 discourse/discourse_test:build${{ steps.arch-helper.outputs.arch_postfix_underscore }} + docker run --rm -e RUBY_ONLY=1 -e USE_TURBO=1 -e SKIP_PLUGINS=1 -e SKIP_LINT=1 discourse/discourse_test:build_$ARCH + - name: build & tag dev image + run: | + cd image && ruby auto_build.rb discourse_dev_$ARCH + docker tag discourse/discourse_dev:build_$ARCH discourse/discourse_dev:$TIMESTAMP-$ARCH - name: Print summary run: | docker images discourse/base - name: push to dockerhub - if: success() && (github.ref == 'refs/heads/main') + if: github.ref == 'refs/heads/main' env: DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} run: | - TAG=${{ steps.tag-images.outputs.tag }} docker login --username discoursebuild --password $DOCKERHUB_PASSWORD - docker push discourse/base:2.0.$TAG-slim${{ steps.arch-helper.outputs.arch_postfix_dash }} - docker push discourse/base:slim${{ steps.arch-helper.outputs.arch_postfix_dash }} - docker push discourse/base:2.0.$TAG${{ steps.arch-helper.outputs.arch_postfix_dash }} - docker push discourse/base:release${{ steps.arch-helper.outputs.arch_postfix_dash }} + docker push discourse/base:2.0.$TIMESTAMP-slim-$ARCH + docker push discourse/base:2.0.$TIMESTAMP-$ARCH + docker push discourse/discourse_dev:$TIMESTAMP-$ARCH - name: Push discourse/base:aarch64 image for backwards compatibility - if: success() && (github.ref == 'refs/heads/main') && (matrix.arch == 'arm64') + if: (github.ref == 'refs/heads/main') && (matrix.arch == 'arm64') run: | - docker tag discourse/base:release${{ steps.arch-helper.outputs.arch_postfix_dash }} discourse/base:aarch64 + docker tag discourse/base:2.0.$TIMESTAMP-$ARCH discourse/base:aarch64 docker push discourse/base:aarch64 + push_multiarch_manifests: + runs-on: ubuntu-latest + needs: [base, timestamp] + env: + TIMESTAMP: ${{ needs.timestamp.outputs.timestamp }} + if: github.ref == 'refs/heads/main' + steps: + - name: create and push multiarch manifests + run: | + docker login --username discoursebuild --password ${{ secrets.DOCKERHUB_PASSWORD }} + + # Slim timestamped + docker manifest create discourse/base:2.0.$TIMESTAMP-slim \ + -a discourse/base:2.0.$TIMESTAMP-slim-amd64 \ + -a discourse/base:2.0.$TIMESTAMP-slim-arm64 + + # Slim release + docker manifest create discourse/base:release-slim \ + -a discourse/base:2.0.$TIMESTAMP-slim-amd64 \ + -a discourse/base:2.0.$TIMESTAMP-slim-arm64 + + # Full timestamped + docker manifest create discourse/base:2.0.$TIMESTAMP \ + -a discourse/base:2.0.$TIMESTAMP-amd64 \ + -a discourse/base:2.0.$TIMESTAMP-arm64 + + # Full release + docker manifest create discourse/base:release \ + -a discourse/base:2.0.$TIMESTAMP-amd64 \ + -a discourse/base:2.0.$TIMESTAMP-arm64 + + # Dev timestamped + docker manifest create discourse/discourse_dev:$TIMESTAMP \ + -a discourse/discourse_dev:$TIMESTAMP-amd64 \ + -a discourse/discourse_dev:$TIMESTAMP-arm64 + + # Dev release + docker manifest create discourse/discourse_dev:release \ + -a discourse/discourse_dev:$TIMESTAMP-amd64 \ + -a discourse/discourse_dev:$TIMESTAMP-arm64 + + docker manifest push discourse/base:2.0.$TIMESTAMP-slim + docker manifest push discourse/base:release-slim + docker manifest push discourse/base:2.0.$TIMESTAMP + docker manifest push discourse/base:release + docker manifest push discourse/discourse_dev:$TIMESTAMP + docker manifest push discourse/discourse_dev:release test: runs-on: ubuntu-22.04${{ ((github.event_name != 'schedule') && '-8core') || '' }} timeout-minutes: 30 @@ -118,22 +174,3 @@ jobs: docker push discourse/discourse_test:slim docker push discourse/discourse_test:slim-browsers docker push discourse/discourse_test:release - dev: - runs-on: ubuntu-22.04${{ ((github.event_name != 'schedule') && '-8core') || '' }} - timeout-minutes: 30 - needs: base - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 1 - - name: build discourse_dev image - run: | - cd image && ruby auto_build.rb discourse_dev - - name: push to dockerhub - if: success() && (github.ref == 'refs/heads/main') - env: - DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} - run: | - docker tag discourse/discourse_dev:build discourse/discourse_dev:release - docker login --username discoursebuild --password $DOCKERHUB_PASSWORD - docker push discourse/discourse_dev:release diff --git a/image/auto_build.rb b/image/auto_build.rb index 9c57f6d..7e0a827 100644 --- a/image/auto_build.rb +++ b/image/auto_build.rb @@ -4,9 +4,9 @@ require "pty" require "optparse" images = { - base_slim: { + base_slim_amd64: { name: "base", - tag: "discourse/base:build_slim", + tag: "discourse/base:build_slim_amd64", extra_args: "-f slim.Dockerfile", }, base_slim_arm64: { @@ -14,29 +14,36 @@ images = { tag: "discourse/base:build_slim_arm64", extra_args: "-f slim.Dockerfile --platform linux/arm64", }, - base: { + base_amd64: { name: "base", - tag: "discourse/base:build", - extra_args: "-f release.Dockerfile", + tag: "discourse/base:build_amd64", + extra_args: "-f release.Dockerfile --build-arg=\"tag=build_slim_amd64\"", }, base_arm64: { name: "base", tag: "discourse/base:build_arm64", extra_args: "-f release.Dockerfile --platform linux/arm64 --build-arg=\"tag=build_slim_arm64\"", }, - discourse_test_build: { + discourse_test_build_amd64: { name: "discourse_test", - tag: "discourse/discourse_test:build", + tag: "discourse/discourse_test:build_amd64", + extra_args: "--build-arg=\"from_tag=build_amd64\"", }, discourse_test_build_arm64: { name: "discourse_test", tag: "discourse/discourse_test:build_arm64", extra_args: "--platform linux/arm64 --build-arg=\"from_tag=build_arm64\"", }, - discourse_dev: { + discourse_dev_amd64: { name: "discourse_dev", - tag: "discourse/discourse_dev:build", + tag: "discourse/discourse_dev:build_amd64", + extra_args: "--build-arg=\"from_tag=build_slim_amd64\"" }, + discourse_dev_arm64: { + name: "discourse_dev", + tag: "discourse/discourse_dev:build_arm64", + extra_args: "--platform linux/arm64 --build-arg=\"from_tag=build_slim_arm64\"" + } } def run(command) @@ -94,7 +101,7 @@ else end puts "Building #{images[image]}" - dev_deps() if image == :discourse_dev + dev_deps() if image == :discourse_dev_amd64 || image == :discourse_dev_arm64 build(images[image], ARGV[1..-1].join(" ")) end -- 2.25.1