From 1d8026e33391265da31a7062263bfad2738677a1 Mon Sep 17 00:00:00 2001 From: Alex Pyrgiotis Date: Tue, 25 Feb 2025 17:31:32 +0200 Subject: [PATCH] WIP: Longer builds, more reproducible --- .../workflows/release-container-image.yaml | 142 +++++++++--------- dev_scripts/reproduce-image.py | 33 ++-- 2 files changed, 91 insertions(+), 84 deletions(-) diff --git a/.github/workflows/release-container-image.yaml b/.github/workflows/release-container-image.yaml index a6a600e..653ead0 100644 --- a/.github/workflows/release-container-image.yaml +++ b/.github/workflows/release-container-image.yaml @@ -54,19 +54,19 @@ jobs: - prepare strategy: fail-fast: false - matrix: - platform: - - suffix: "" - name: "linux/amd64" - - suffix: "-arm" - name: "linux/arm64" + #matrix: + # platform: + # - suffix: "" + # name: "linux/amd64" + # - suffix: "-arm" + # name: "linux/arm64" steps: - uses: actions/checkout@v4 - - name: Prepare - run: | - platform=${{ matrix.platform.name }} - echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV + #- name: Prepare + # run: | + # platform=${{ matrix.platform.name }} + # echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV - name: Login to GHCR uses: docker/login-action@v3 @@ -80,7 +80,7 @@ jobs: with: driver-opts: image=${{ env.BUILDKIT_IMAGE }} - - name: Build and push by digest + - name: Build and push id: build uses: docker/build-push-action@v6 with: @@ -92,71 +92,71 @@ jobs: # Remove potentially incorrect Docker provenance that cannot be # reproduced. provenance: false - platforms: ${{ matrix.platform.name }} - outputs: type=image,"name=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}",rewrite-timestamp=true,push-by-digest=true,name-canonical=true,push=true + platforms: linux/amd64,linux/arm64 + outputs: type=registry,"name=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}",rewrite-timestamp=true,name-canonical=true cache-from: type=gha cache-to: type=gha,mode=max - - name: Export digest - run: | - mkdir -p ${{ runner.temp }}/digests - digest="${{ steps.build.outputs.digest }}" - touch "${{ runner.temp }}/digests/${digest#sha256:}" - echo "Image digest is: ${digest}" + #- name: Export digest + # run: | + # mkdir -p ${{ runner.temp }}/digests + # digest="${{ steps.build.outputs.digest }}" + # touch "${{ runner.temp }}/digests/${digest#sha256:}" + # echo "Image digest is: ${digest}" - - name: Upload digest - uses: actions/upload-artifact@v4 - with: - name: digests-${{ env.PLATFORM_PAIR }} - path: ${{ runner.temp }}/digests/* - if-no-files-found: error - retention-days: 1 + #- name: Upload digest + # uses: actions/upload-artifact@v4 + # with: + # name: digests-${{ env.PLATFORM_PAIR }} + # path: ${{ runner.temp }}/digests/* + # if-no-files-found: error + # retention-days: 1 - merge: - runs-on: ubuntu-latest - needs: - - prepare - - build - outputs: - digest: ${{ steps.image.outputs.digest }} - image: ${{ steps.image.outputs.image }} - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 + #merge: + # runs-on: ubuntu-latest + # needs: + # - prepare + # - build + # outputs: + # digest: ${{ steps.image.outputs.digest }} + # image: ${{ steps.image.outputs.image }} + # steps: + # - uses: actions/checkout@v4 + # with: + # fetch-depth: 0 - - name: Compute image tag - id: tag - run: | - DEBIAN_ARCHIVE_DATE=${{ needs.prepare.outputs.debian_archive_date }} - TAG=$(git describe --long --first-parent | tail -c +2) - echo "tag=${DEBIAN_ARCHIVE_DATE}-${TAG}" >> $GITHUB_OUTPUT + # - name: Compute image tag + # id: tag + # run: | + # DEBIAN_ARCHIVE_DATE=${{ needs.prepare.outputs.debian_archive_date }} + # TAG=$(git describe --long --first-parent | tail -c +2) + # echo "tag=${DEBIAN_ARCHIVE_DATE}-${TAG}" >> $GITHUB_OUTPUT - - name: Download digests - uses: actions/download-artifact@v4 - with: - path: ${{ runner.temp }}/digests - pattern: digests-* - merge-multiple: true + # - name: Download digests + # uses: actions/download-artifact@v4 + # with: + # path: ${{ runner.temp }}/digests + # pattern: digests-* + # merge-multiple: true - - name: Login to GHCR - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} + # - name: Login to GHCR + # uses: docker/login-action@v3 + # with: + # registry: ghcr.io + # username: ${{ github.repository_owner }} + # password: ${{ secrets.GITHUB_TOKEN }} - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - with: - driver-opts: image=${{ env.BUILDKIT_IMAGE }} + # - name: Set up Docker Buildx + # uses: docker/setup-buildx-action@v3 + # with: + # driver-opts: image=${{ env.BUILDKIT_IMAGE }} - - name: Create manifest list and push - working-directory: ${{ runner.temp }}/digests - run: | - IMAGE=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.tag }} - DIGESTS=$(printf '${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@sha256:%s ' *) - docker buildx imagetools create -t ${IMAGE} ${DIGESTS} + # - name: Create manifest list and push + # working-directory: ${{ runner.temp }}/digests + # run: | + # IMAGE=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.tag }} + # DIGESTS=$(printf '${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@sha256:%s ' *) + # docker buildx imagetools create -t ${IMAGE} ${DIGESTS} - name: Inspect image id: image @@ -173,15 +173,15 @@ jobs: # the container registry. provenance: needs: - - merge + - build permissions: actions: read # for detecting the Github Actions environment. id-token: write # for creating OIDC tokens for signing. packages: write # for uploading attestations. uses: slsa-framework/slsa-github-generator/.github/workflows/generator_container_slsa3.yml@v2.0.0 with: - digest: ${{ needs.merge.outputs.digest }} - image: ${{ needs.merge.outputs.image }} + digest: ${{ needs.build.outputs.digest }} + image: ${{ needs.build.outputs.image }} registry-username: ${{ github.actor }} secrets: registry-password: ${{ secrets.GITHUB_TOKEN }} @@ -190,7 +190,7 @@ jobs: check-reproducibility: needs: - prepare - - merge + - build runs-on: ubuntu-24.04${{ matrix.platform.suffix }} strategy: fail-fast: false @@ -209,5 +209,5 @@ jobs: run: | ./dev_scripts/reproduce-image.py \ --debian-archive-date ${{ needs.build.prepare.debian_archive_date }} \ - --source ${{ needs.merge.outputs.image }}@${{ needs.merge.outputs.digest }} \ + --source ${{ needs.build.outputs.image }}@${{ needs.build.outputs.digest }} \ --platform ${{ matrix.platform.name }} diff --git a/dev_scripts/reproduce-image.py b/dev_scripts/reproduce-image.py index 26783d8..ac6733b 100755 --- a/dev_scripts/reproduce-image.py +++ b/dev_scripts/reproduce-image.py @@ -4,6 +4,7 @@ import argparse import hashlib import logging import pathlib +import platform import stat import subprocess import sys @@ -85,16 +86,20 @@ def diffoci_download(): DIFFOCI_PATH.chmod(DIFFOCI_PATH.stat().st_mode | stat.S_IEXEC) -def diffoci_diff(runtime, source, local_target, platform=None): +def diffoci_diff(runtime, source, target, platform=None): """Diff the source image against the recently built target image using diffoci.""" - target = f"{runtime}://{local_target}" platform_args = [] if not platform else ["--platform", platform] + logger.info(f"Loading image tarball in diffoci") + with open("share/container.tar") as f: + subprocess.run([str(DIFFOCI_PATH), "load"], stdin=f, check=True) + logger.info(f"Performing a strict diff between the two images") try: return run( str(DIFFOCI_PATH), "diff", source, target, + # "--semantic", "--verbose", *platform_args, ) @@ -116,17 +121,19 @@ def build_image( platform_args = [] if not platform else ["--platform", platform] runtime_args = [] if not runtime else ["--runtime", runtime] date_args = [] if not date else ["--debian-archive-date", date] - run( - "python3", - "./install/common/build-image.py", - "--no-save", - "--use-cache", - str(use_cache), - *date_args, - *platform_args, - *runtime_args, - "--tag", - tag, + subprocess.run( + [ + "python3", + "./install/common/build-image.py", + "--use-cache", + str(use_cache), + *date_args, + *platform_args, + *runtime_args, + "--tag", + tag, + ], + check=True, )