From df3a60edc629a47574a24bf97dee6066cb7c4909 Mon Sep 17 00:00:00 2001 From: Alex Pyrgiotis Date: Mon, 13 Jan 2025 15:41:52 +0200 Subject: [PATCH 01/21] Whitespace fixes --- BUILD.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/BUILD.md b/BUILD.md index 167d969..bd1b377 100644 --- a/BUILD.md +++ b/BUILD.md @@ -487,9 +487,9 @@ Install the WiX UI extension. You may need to open a new terminal in order to us wix extension add --global WixToolset.UI.wixext/5.x.y ``` -> [!IMPORTANT] +> [!IMPORTANT] > To avoid compatibility issues, ensure the WiX UI extension version matches the version of the WiX Toolset. -> +> > Run `wix --version` to check the version of WiX Toolset you have installed and replace `5.x.y` with the full version number without the Git revision. ### If you want to sign binaries with Authenticode From d2f483e970df43c8daa2a1932f6bb6097a3aa36a Mon Sep 17 00:00:00 2001 From: Alex Pyrgiotis Date: Mon, 13 Jan 2025 16:09:07 +0200 Subject: [PATCH 02/21] Move container-only build context to `dangerzone/container` Move container-only build context - currently just the entrypoint script - from `dangerzone/gvisor_wrapper` to `dangerzone/container`. Update the rest of the scripts to use this location as well. --- .github/workflows/build.yml | 2 +- .github/workflows/ci.yml | 8 ++++---- Dockerfile | 2 +- dangerzone/{gvisor_wrapper => container}/entrypoint.py | 0 dodo.py | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) rename dangerzone/{gvisor_wrapper => container}/entrypoint.py (100%) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 35f9597..d5c1f09 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -85,7 +85,7 @@ jobs: id: cache-container-image uses: actions/cache@v4 with: - key: v3-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/common.py', 'dangerzone/conversion/doc_to_pixels.py', 'dangerzone/conversion/pixels_to_pdf.py', 'poetry.lock', 'gvisor_wrapper/entrypoint.py') }} + key: v3-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/*.py', 'dangerzone/container/*', 'install/common/build-image.py', 'poetry.lock') }} path: | share/container.tar.gz share/image-id.txt diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dc2bb27..34fb410 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,7 +59,7 @@ jobs: id: cache-container-image uses: actions/cache@v4 with: - key: v3-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/common.py', 'dangerzone/conversion/doc_to_pixels.py', 'dangerzone/conversion/pixels_to_pdf.py', 'poetry.lock', 'gvisor_wrapper/entrypoint.py') }} + key: v3-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/*.py', 'dangerzone/container/*', 'install/common/build-image.py', 'poetry.lock') }} path: |- share/container.tar.gz share/image-id.txt @@ -227,7 +227,7 @@ jobs: - name: Restore container cache uses: actions/cache/restore@v4 with: - key: v3-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/common.py', 'dangerzone/conversion/doc_to_pixels.py', 'dangerzone/conversion/pixels_to_pdf.py', 'poetry.lock', 'gvisor_wrapper/entrypoint.py') }} + key: v3-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/*.py', 'dangerzone/container/*', 'install/common/build-image.py', 'poetry.lock') }} path: |- share/container.tar.gz share/image-id.txt @@ -334,7 +334,7 @@ jobs: - name: Restore container image uses: actions/cache/restore@v4 with: - key: v3-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/common.py', 'dangerzone/conversion/doc_to_pixels.py', 'dangerzone/conversion/pixels_to_pdf.py', 'poetry.lock', 'gvisor_wrapper/entrypoint.py') }} + key: v3-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/*.py', 'dangerzone/container/*', 'install/common/build-image.py', 'poetry.lock') }} path: |- share/container.tar.gz share/image-id.txt @@ -429,7 +429,7 @@ jobs: - name: Restore container image uses: actions/cache/restore@v4 with: - key: v3-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/common.py', 'dangerzone/conversion/doc_to_pixels.py', 'dangerzone/conversion/pixels_to_pdf.py', 'poetry.lock', 'gvisor_wrapper/entrypoint.py') }} + key: v3-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/*.py', 'dangerzone/container/*', 'install/common/build-image.py', 'poetry.lock') }} path: |- share/container.tar.gz share/image-id.txt diff --git a/Dockerfile b/Dockerfile index c89c5a4..5799b09 100644 --- a/Dockerfile +++ b/Dockerfile @@ -97,6 +97,6 @@ COPY --from=dangerzone-image / /home/dangerzone/dangerzone-image/rootfs # store the state of its containers. RUN mkdir /home/dangerzone/.containers -COPY gvisor_wrapper/entrypoint.py / +COPY container/entrypoint.py / ENTRYPOINT ["/entrypoint.py"] diff --git a/dangerzone/gvisor_wrapper/entrypoint.py b/dangerzone/container/entrypoint.py similarity index 100% rename from dangerzone/gvisor_wrapper/entrypoint.py rename to dangerzone/container/entrypoint.py diff --git a/dodo.py b/dodo.py index 2dd3d21..8fca92a 100644 --- a/dodo.py +++ b/dodo.py @@ -65,7 +65,7 @@ IMAGE_DEPS = [ "Dockerfile", "poetry.lock", *list_files("dangerzone/conversion"), - "dangerzone/gvisor_wrapper/entrypoint.py", + *list_files("dangerzone/container"), "install/common/build-image.py", ] IMAGE_TARGETS = ["share/container.tar.gz", "share/image-id.txt"] From e554a573e5d53063691d0be411ba0d1fd1c4a4c6 Mon Sep 17 00:00:00 2001 From: Alex Pyrgiotis Date: Mon, 13 Jan 2025 16:29:33 +0200 Subject: [PATCH 03/21] Reuse the same rootfs for the inner and outer container Remove the need to copy the Dangerzone container image (used by the inner container) within a wrapper gVisor image (used by the outer container). Instead, use the root of the container filesystem for both containers. We can do this safely because we don't mount any secrets to the container, and because gVisor offers a read-only view of the underlying filesystem Fixes #1048 --- Dockerfile | 20 +++----------------- dangerzone/container/entrypoint.py | 6 +++--- docs/developer/gvisor.md | 6 ++++++ 3 files changed, 12 insertions(+), 20 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5799b09..8c9d83d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -36,7 +36,7 @@ RUN mkdir /libreoffice_ext && cd libreoffice_ext \ ########################################### # Dangerzone image -FROM alpine:latest AS dangerzone-image +FROM alpine:latest # Install dependencies RUN apk --no-cache -U upgrade && \ @@ -66,14 +66,6 @@ COPY conversion /opt/dangerzone/dangerzone/conversion RUN addgroup -g 1000 dangerzone && \ adduser -u 1000 -s /bin/true -G dangerzone -h /home/dangerzone -D dangerzone -########################################### -# gVisor wrapper image - -FROM alpine:latest - -RUN apk --no-cache -U upgrade && \ - apk --no-cache add python3 - RUN GVISOR_URL="https://storage.googleapis.com/gvisor/releases/release/latest/$(uname -m)"; \ wget "${GVISOR_URL}/runsc" "${GVISOR_URL}/runsc.sha512" && \ sha512sum -c runsc.sha512 && \ @@ -81,18 +73,12 @@ RUN GVISOR_URL="https://storage.googleapis.com/gvisor/releases/release/latest/$( chmod 555 runsc && \ mv runsc /usr/bin/ -# Add the unprivileged `dangerzone` user. -RUN addgroup dangerzone && \ - adduser -s /bin/true -G dangerzone -h /home/dangerzone -D dangerzone +RUN touch /config.json +RUN chown dangerzone:dangerzone /config.json # Switch to the dangerzone user for the rest of the script. USER dangerzone -# Copy the Dangerzone image, as created by the previous steps, into the home -# directory of the `dangerzone` user. -RUN mkdir /home/dangerzone/dangerzone-image -COPY --from=dangerzone-image / /home/dangerzone/dangerzone-image/rootfs - # Create a directory that will be used by gVisor as the place where it will # store the state of its containers. RUN mkdir /home/dangerzone/.containers diff --git a/dangerzone/container/entrypoint.py b/dangerzone/container/entrypoint.py index 8d09eb2..80a6455 100755 --- a/dangerzone/container/entrypoint.py +++ b/dangerzone/container/entrypoint.py @@ -56,7 +56,7 @@ oci_config: dict[str, typing.Any] = { {"type": "RLIMIT_NOFILE", "hard": 4096, "soft": 4096}, ], }, - "root": {"path": "rootfs", "readonly": True}, + "root": {"path": "/", "readonly": True}, "hostname": "dangerzone", "mounts": [ { @@ -133,7 +133,7 @@ if os.environ.get("RUNSC_DEBUG"): json.dump(oci_config, sys.stderr, indent=2, sort_keys=True) # json.dump doesn't print a trailing newline, so print one here: log("") -with open("/home/dangerzone/dangerzone-image/config.json", "w") as oci_config_out: +with open("/config.json", "w") as oci_config_out: json.dump(oci_config, oci_config_out, indent=2, sort_keys=True) # Run gVisor. @@ -150,7 +150,7 @@ if os.environ.get("RUNSC_DEBUG"): runsc_argv += ["--debug=true", "--alsologtostderr=true"] if os.environ.get("RUNSC_FLAGS"): runsc_argv += [x for x in shlex.split(os.environ.get("RUNSC_FLAGS", "")) if x] -runsc_argv += ["run", "--bundle=/home/dangerzone/dangerzone-image", "dangerzone"] +runsc_argv += ["run", "--bundle=/", "dangerzone"] log( "Running gVisor with command line: {}", " ".join(shlex.quote(s) for s in runsc_argv) ) diff --git a/docs/developer/gvisor.md b/docs/developer/gvisor.md index e85f84c..6898fcd 100644 --- a/docs/developer/gvisor.md +++ b/docs/developer/gvisor.md @@ -1,5 +1,11 @@ # gVisor integration +> [!NOTE] +> **Update on 2025-01-13:** There is no longer a copied container image under +> `/home/dangerzone/dangerzone-image/rootfs`. We now reuse the same container +> image both for the inner and outer container. See +> [#1048](https://github.com/freedomofpress/dangerzone/issues/1048). + Dangerzone has relied on the container runtime available in each supported operating system (Docker Desktop on Windows / macOS, Podman on Linux) to isolate the host from the sanitization process. The problem with this type of isolation From 9c0c880cd33a60f9d48159cc4e8eda53c4be054c Mon Sep 17 00:00:00 2001 From: Alex Pyrgiotis Date: Mon, 13 Jan 2025 16:43:17 +0200 Subject: [PATCH 04/21] docs: Add design document for artifact reproducibility Refs #1047 --- docs/developer/reproducibility.md | 90 +++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 docs/developer/reproducibility.md diff --git a/docs/developer/reproducibility.md b/docs/developer/reproducibility.md new file mode 100644 index 0000000..4fcc0f4 --- /dev/null +++ b/docs/developer/reproducibility.md @@ -0,0 +1,90 @@ +# Reproducible builds + +We want to improve the transparency and auditability of our build artifacts, and +a way to achieve this is via reproducible builds. For a broader understanding of +what reproducible builds entail, check out https://reproducible-builds.org/. + +Our build artifacts consist of: +* Container images (`amd64` and `arm64` architectures) +* macOS installers (for Intel and Apple Silicon CPUs) +* Windows installer +* Fedora packages (for regular Fedora distros and Qubes) +* Debian packages (for Debian and Ubuntu) + +As of writing this, none of the above artifacts are reproducible. For this +reason, we purposefully build them in machines owned by FPF, since we can't +trust third-party servers. A security hole in GitHub, or +in our CI pipeline (check out the +[Ultralytics cryptominer saga](https://github.com/ultralytics/ultralytics/issues/18027)), +may allow attackers to plant a malicious artifact with no detection. + +Still, building our artifacts in private is not ideal. Third parties cannot +easily audit if our artifacts have been built correctly or if they have been +tampered with. For instance, our Apple Silicon container image builds PyMuPDF +from source, and while the PyPI source package is hashed, the produced output +does not have a known hash. So, it's not easy to verify it's been built +correctly (read also the seminal +["Reflections on Trusting Trust"](https://www.cs.cmu.edu/~rdriley/487/papers/Thompson_1984_ReflectionsonTrustingTrust.pdf) +lecture by Ken Thompson on that subject). + +In order to make our builds auditable and allow building artifacts in +third-party servers safely, we want to make each artifact build reproducible. In +the following sections, we'll lay down the plan to do so for each artifact type. + +## Container image + +### Current limitations + +Our container image is currently not reproducible for the following main +reasons: + +* We build PyMuPDF from source, since it's not available in Alpine Linux. The + result of this build is not reproducible. Note that PyMuPDF wheels are + available from PyPI, but there are no ARM wheels for the musl libc platforms. +* Alpine Linux does not have a way to pin packages and their dependencies, and + does not retain old packages. There's a + [workaround](https://github.com/reproducible-containers/repro-pkg-cache) + to download the required packages and store them elsewhere, but then the + cached package downloads cannot be easily audited. + +## Proposed implementation + +We can take advantage of the +[Debian snapshot archives](https://snapshot.debian.org/) +and pin our packages by specifying a date. There's already +[prior art](https://github.com/reproducible-containers/repro-sources-list.sh/) +for that, thanks to the incredible work of @AkihiroSuda on +[reproducible containers](https://github.com/reproducible-containers). +As for PyMuPDF, it is available from the Debian repos, so we won't have to build +it from source. + +Here are a few other obstacles that we need to overcome: +* We currently download the + [latest gVisor version](https://gvisor.dev/docs/user_guide/install/#latest-release) + from a GCS bucket. Now that we have switched to Debian, we can take advantage + of their + [timestamped APT repos](https://gvisor.dev/docs/user_guide/install/#specific-release) + and download specific releases from those. An extra benefit is that such + releases are signed with their APT key. +* We can no longer update the packages in the container image by rebuilding it. + We have to bump the dates in the Dockerfile first, which is a minor hassle, + but much more declarative. +* The `repro-source-list-.sh` script uses the release date of the container + image. However, the Debian image is not updated daily (see + [newest tags](https://hub.docker.com/_/debian/tags) + in DockerHub). So, if we want to ship an emergency release, we have to + circumvent this limitation. A simple way is to trick the script by bumping the + date of the `/etc/apt/sources.list.d/debian.sources` and + `/etc/apt/sources.list` files. +* While we talk about image reproducibility, we can't actually achieve the exact + same SHA-256 hash for two different image builds. That's because the file + timestamps in the image layers will differ, depending on when the build took + place. The rest of the image though (file contents, permissions, manifest) + should be byte-for-byte the same. A simple way to check this is with the + [`diffoci`](https://github.com/reproducible-containers/diffoci) tool, and + specifically this invocation: + + ``` + ./diffoci diff podman:// podman:// \ + --ignore-timestamps --ignore-image-name --verbose + ``` From 5ff1d302785d035384a8b6e0d8eb337f2d5315f0 Mon Sep 17 00:00:00 2001 From: Alex Pyrgiotis Date: Mon, 13 Jan 2025 16:45:02 +0200 Subject: [PATCH 05/21] container: Copy gVisor public key and a helper script Download and copy the following artifacts that will be used for building a Debian-based Dangerzone container image in the subsequent commits: * The APT key for the gVisor repo [1] * A helper script for building reproducible Debian images [2] [1] https://gvisor.dev/archive.key [2] https://github.com/reproducible-containers/repro-sources-list.sh/blob/d15cf12b26395b857b24fba223b108aff1c91b26/repro-sources-list.sh --- dangerzone/container/gvisor.key | 29 ++++++ dangerzone/container/repro-sources-list.sh | 103 +++++++++++++++++++++ 2 files changed, 132 insertions(+) create mode 100644 dangerzone/container/gvisor.key create mode 100755 dangerzone/container/repro-sources-list.sh diff --git a/dangerzone/container/gvisor.key b/dangerzone/container/gvisor.key new file mode 100644 index 0000000..8946884 --- /dev/null +++ b/dangerzone/container/gvisor.key @@ -0,0 +1,29 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQINBF0meAYBEACcBYPOSBiKtid+qTQlbgKGPxUYt0cNZiQqWXylhYUT4PuNlNx5 +s+sBLFvNTpdTrXMmZ8NkekyjD1HardWvebvJT4u+Ho/9jUr4rP71cNwNtocz/w8G +DsUXSLgH8SDkq6xw0L+5eGc78BBg9cOeBeFBm3UPgxTBXS9Zevoi2w1lzSxkXvjx +cGzltzMZfPXERljgLzp9AAfhg/2ouqVQm37fY+P/NDzFMJ1XHPIIp9KJl/prBVud +jJJteFZ5sgL6MwjBQq2kw+q2Jb8Zfjl0BeXDgGMN5M5lGhX2wTfiMbfo7KWyzRnB +RpSP3BxlLqYeQUuLG5Yx8z3oA3uBkuKaFOKvXtiScxmGM/+Ri2YM3m66imwDhtmP +AKwTPI3Re4gWWOffglMVSv2sUAY32XZ74yXjY1VhK3bN3WFUPGrgQx4X7GP0A1Te +lzqkT3VSMXieImTASosK5L5Q8rryvgCeI9tQLn9EpYFCtU3LXvVgTreGNEEjMOnL +dR7yOU+Fs775stn6ucqmdYarx7CvKUrNAhgEeHMonLe1cjYScF7NfLO1GIrQKJR2 +DE0f+uJZ52inOkO8ufh3WVQJSYszuS3HCY7w5oj1aP38k/y9zZdZvVvwAWZaiqBQ +iwjVs6Kub76VVZZhRDf4iYs8k1Zh64nXdfQt250d8U5yMPF3wIJ+c1yhxwARAQAB +tCpUaGUgZ1Zpc29yIEF1dGhvcnMgPGd2aXNvci1ib3RAZ29vZ2xlLmNvbT6JAk4E +EwEKADgCGwMFCwkIBwIGFQoJCAsCBBYCAwECHgECF4AWIQRvHfheOnHCSRjnJ9Vv +xtVU4yvZQwUCYO4TxQAKCRBvxtVU4yvZQ9UoEACLPV7CnEA2bjCPi0NCWB/Mo1WL +evqv7Wv7vmXzI1K9DrqOhxuamQW75SVXg1df0hTJWbKFmDAip6NEC2Rg5P+A8hHj +nW/VG+q4ZFT662jDhnXQiO9L7EZzjyqNF4yWYzzgnqEu/SmGkDLDYiUCcGBqS2oE +EQfk7RHJSLMJXAnNDH7OUDgrirSssg/dlQ5uAHA9Au80VvC5fsTKza8b3Aydw3SV +iB8/Yuikbl8wKbpSGiXtR4viElXjNips0+mBqaUk2xpqSBrsfN+FezcInVXaXFeq +xtpq2/3M3DYbqCRjqeyd9wNi92FHdOusNrK4MYe0pAYbGjc65BwH+F0T4oJ8ZSJV +lIt+FZ0MqM1T97XadybYFsJh8qvajQpZEPL+zzNncc4f1d80e7+lwIZV/al0FZWW +Zlp7TpbeO/uW+lHs5W14YKwaQVh1whapKXTrATipNOOSCw2hnfrT8V7Hy55QWaGZ +f4/kfy929EeCP16d/LqOClv0j0RBr6NhRBQ0l/BE/mXjJwIk6nKwi+Yi4ek1ARi6 +AlCMLn9AZF7aTGpvCiftzIrlyDfVZT5IX03TayxRHZ4b1Rj8eyJaHcjI49u83gkr +4LGX08lEawn9nxFSx4RCg2swGiYw5F436wwwAIozqJuDASeTa3QND3au5v0oYWnl +umDySUl5wPaAaALgzA== +=5/8T +-----END PGP PUBLIC KEY BLOCK----- diff --git a/dangerzone/container/repro-sources-list.sh b/dangerzone/container/repro-sources-list.sh new file mode 100755 index 0000000..ea97e47 --- /dev/null +++ b/dangerzone/container/repro-sources-list.sh @@ -0,0 +1,103 @@ +#!/bin/bash +# +# Copyright The repro-sources-list.sh Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ----------------------------------------------------------------------------- +# repro-sources-list.sh: +# configures /etc/apt/sources.list and similar files for installing packages from a snapshot. +# +# This script is expected to be executed inside Dockerfile. +# +# The following distributions are supported: +# - debian:11 (/etc/apt/sources.list) +# - debian:12 (/etc/apt/sources.list.d/debian.sources) +# - ubuntu:22.04 (/etc/apt/sources.list) +# - ubuntu:24.04 (/etc/apt/sources.listd/ubuntu.sources) +# - archlinux (/etc/pacman.d/mirrorlist) +# +# For the further information, see https://github.com/reproducible-containers/repro-sources-list.sh +# ----------------------------------------------------------------------------- + +set -eux -o pipefail + +. /etc/os-release + +: "${KEEP_CACHE:=1}" + +keep_apt_cache() { + rm -f /etc/apt/apt.conf.d/docker-clean + echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' >/etc/apt/apt.conf.d/keep-cache +} + +case "${ID}" in +"debian") + : "${SNAPSHOT_ARCHIVE_BASE:=http://snapshot.debian.org/archive/}" + : "${BACKPORTS:=}" + if [ -e /etc/apt/sources.list.d/debian.sources ]; then + : "${SOURCE_DATE_EPOCH:=$(stat --format=%Y /etc/apt/sources.list.d/debian.sources)}" + rm -f /etc/apt/sources.list.d/debian.sources + else + : "${SOURCE_DATE_EPOCH:=$(stat --format=%Y /etc/apt/sources.list)}" + fi + snapshot="$(printf "%(%Y%m%dT%H%M%SZ)T\n" "${SOURCE_DATE_EPOCH}")" + # TODO: use the new format for Debian >= 12 + echo "deb [check-valid-until=no] ${SNAPSHOT_ARCHIVE_BASE}debian/${snapshot} ${VERSION_CODENAME} main" >/etc/apt/sources.list + echo "deb [check-valid-until=no] ${SNAPSHOT_ARCHIVE_BASE}debian-security/${snapshot} ${VERSION_CODENAME}-security main" >>/etc/apt/sources.list + echo "deb [check-valid-until=no] ${SNAPSHOT_ARCHIVE_BASE}debian/${snapshot} ${VERSION_CODENAME}-updates main" >>/etc/apt/sources.list + if [ "${BACKPORTS}" = 1 ]; then echo "deb [check-valid-until=no] ${SNAPSHOT_ARCHIVE_BASE}debian/${snapshot} ${VERSION_CODENAME}-backports main" >>/etc/apt/sources.list; fi + if [ "${KEEP_CACHE}" = 1 ]; then keep_apt_cache; fi + ;; +"ubuntu") + : "${SNAPSHOT_ARCHIVE_BASE:=http://snapshot.ubuntu.com/}" + if [ -e /etc/apt/sources.list.d/ubuntu.sources ]; then + : "${SOURCE_DATE_EPOCH:=$(stat --format=%Y /etc/apt/sources.list.d/ubuntu.sources)}" + rm -f /etc/apt/sources.list.d/ubuntu.sources + else + : "${SOURCE_DATE_EPOCH:=$(stat --format=%Y /etc/apt/sources.list)}" + fi + snapshot="$(printf "%(%Y%m%dT%H%M%SZ)T\n" "${SOURCE_DATE_EPOCH}")" + # TODO: use the new format for Ubuntu >= 24.04 + echo "deb [check-valid-until=no] ${SNAPSHOT_ARCHIVE_BASE}ubuntu/${snapshot} ${VERSION_CODENAME} main restricted" >/etc/apt/sources.list + echo "deb [check-valid-until=no] ${SNAPSHOT_ARCHIVE_BASE}ubuntu/${snapshot} ${VERSION_CODENAME}-updates main restricted" >>/etc/apt/sources.list + echo "deb [check-valid-until=no] ${SNAPSHOT_ARCHIVE_BASE}ubuntu/${snapshot} ${VERSION_CODENAME} universe" >>/etc/apt/sources.list + echo "deb [check-valid-until=no] ${SNAPSHOT_ARCHIVE_BASE}ubuntu/${snapshot} ${VERSION_CODENAME}-updates universe" >>/etc/apt/sources.list + echo "deb [check-valid-until=no] ${SNAPSHOT_ARCHIVE_BASE}ubuntu/${snapshot} ${VERSION_CODENAME} multiverse" >>/etc/apt/sources.list + echo "deb [check-valid-until=no] ${SNAPSHOT_ARCHIVE_BASE}ubuntu/${snapshot} ${VERSION_CODENAME}-updates multiverse" >>/etc/apt/sources.list + echo "deb [check-valid-until=no] ${SNAPSHOT_ARCHIVE_BASE}ubuntu/${snapshot} ${VERSION_CODENAME}-backports main restricted universe multiverse" >>/etc/apt/sources.list + echo "deb [check-valid-until=no] ${SNAPSHOT_ARCHIVE_BASE}ubuntu/${snapshot} ${VERSION_CODENAME}-security main restricted" >>/etc/apt/sources.list + echo "deb [check-valid-until=no] ${SNAPSHOT_ARCHIVE_BASE}ubuntu/${snapshot} ${VERSION_CODENAME}-security universe" >>/etc/apt/sources.list + echo "deb [check-valid-until=no] ${SNAPSHOT_ARCHIVE_BASE}ubuntu/${snapshot} ${VERSION_CODENAME}-security multiverse" >>/etc/apt/sources.list + if [ "${KEEP_CACHE}" = 1 ]; then keep_apt_cache; fi + # http://snapshot.ubuntu.com is redirected to https, so we have to install ca-certificates + export DEBIAN_FRONTEND=noninteractive + apt-get -o Acquire::https::Verify-Peer=false update >&2 + apt-get -o Acquire::https::Verify-Peer=false install -y ca-certificates >&2 + ;; +"arch") + : "${SNAPSHOT_ARCHIVE_BASE:=http://archive.archlinux.org/}" + : "${SOURCE_DATE_EPOCH:=$(stat --format=%Y /var/log/pacman.log)}" + export SOURCE_DATE_EPOCH + # shellcheck disable=SC2016 + date -d "@${SOURCE_DATE_EPOCH}" "+Server = ${SNAPSHOT_ARCHIVE_BASE}repos/%Y/%m/%d/\$repo/os/\$arch" >/etc/pacman.d/mirrorlist + ;; +*) + echo >&2 "Unsupported distribution: ${ID}" + exit 1 + ;; +esac + +: "${WRITE_SOURCE_DATE_EPOCH:=/dev/null}" +echo "${SOURCE_DATE_EPOCH}" >"${WRITE_SOURCE_DATE_EPOCH}" +echo "SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH}" From 42646877d76a95c5d7950c79991ae8b3b660cea5 Mon Sep 17 00:00:00 2001 From: Alex Pyrgiotis Date: Mon, 13 Jan 2025 17:26:58 +0200 Subject: [PATCH 06/21] Switch base image to Debian Stable Switch base image from Alpine Linux to Debian Stable, in order to reduce our image footprint, improve our security posture, and build our container image reproducibly. Fixes #1046 Refs #1047 --- Dockerfile | 119 ++++++++++++------------- dangerzone/conversion/doc_to_pixels.py | 2 + 2 files changed, 58 insertions(+), 63 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8c9d83d..d1f2c10 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,78 +1,71 @@ -########################################### -# Build PyMuPDF +# NOTE: Updating the packages to their latest versions requires bumping the +# Dockerfile args below. For more info about this file, read +# docs/developer/reproducibility.md. -FROM alpine:latest as pymupdf-build -ARG ARCH -ARG REQUIREMENTS_TXT +ARG DEBIAN_IMAGE_DATE=20250113 -# Install PyMuPDF via hash-checked requirements file -COPY ${REQUIREMENTS_TXT} /tmp/requirements.txt +FROM debian:bookworm-${DEBIAN_IMAGE_DATE}-slim -# PyMuPDF provides non-arm musl wheels only. -# Only install build-dependencies if we are actually building the wheel -RUN case "$ARCH" in \ - "arm64") \ - # This is required for copying later, but is created only in the pre-built wheels - mkdir -p /usr/lib/python3.12/site-packages/PyMuPDF.libs/ \ - && apk --no-cache add linux-headers g++ linux-headers gcc make python3-dev py3-pip clang-dev ;; \ - *) \ - apk --no-cache add py3-pip ;; \ - esac -RUN pip install -vv --break-system-packages --require-hashes -r /tmp/requirements.txt +ARG GVISOR_ARCHIVE_DATE=20250106 +ARG DEBIAN_ARCHIVE_DATE=20250114 +ARG H2ORESTART_CHECKSUM=8a5be77359695c14faaf33891d3eca6c9d73c1224599aab50a9d2ccc04640580 +ARG H2ORESTART_VERSION=v0.6.8 +ENV DEBIAN_FRONTEND=noninteractive -########################################### -# Download H2ORestart -FROM alpine:latest as h2orestart-dl -ARG H2ORESTART_CHECKSUM=d09bc5c93fe2483a7e4a57985d2a8d0e4efae2efb04375fe4b59a68afd7241e2 +# The following way of installing packages is taken from +# https://github.com/reproducible-containers/repro-sources-list.sh/blob/master/Dockerfile.debian-12, +# and adapted to allow installing gVisor from each own repo as well. +RUN \ + --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + --mount=type=bind,source=./container/repro-sources-list.sh,target=/usr/local/bin/repro-sources-list.sh \ + --mount=type=bind,source=./container/gvisor.key,target=/tmp/gvisor.key \ + : "Hacky way to set a date for the Debian snapshot repos" && \ + touch -d ${DEBIAN_ARCHIVE_DATE} /etc/apt/sources.list.d/debian.sources && \ + touch -d ${DEBIAN_ARCHIVE_DATE} /etc/apt/sources.list && \ + repro-sources-list.sh && \ + : "Setup APT to install gVisor from its separate APT repo" && \ + apt-get update && \ + apt-get upgrade -y && \ + apt-get install -y --no-install-recommends apt-transport-https ca-certificates gnupg && \ + gpg -o /usr/share/keyrings/gvisor-archive-keyring.gpg --dearmor /tmp/gvisor.key && \ + echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/gvisor-archive-keyring.gpg] https://storage.googleapis.com/gvisor/releases ${GVISOR_ARCHIVE_DATE} main" > /etc/apt/sources.list.d/gvisor.list && \ + : "Install the necessary gVisor and Dangerzone dependencies" && \ + apt-get update && \ + apt-get install -y --no-install-recommends \ + python3 python3-fitz libreoffice-nogui libreoffice-java-common \ + python3 python3-magic default-jre-headless fonts-noto-cjk fonts-dejavu \ + runsc unzip wget && \ + : "Clean up for improving reproducibility (optional)" && \ + rm -rf /var/cache/fontconfig/ && \ + rm -rf /etc/ssl/certs/java/cacerts && \ + rm -rf /var/log/* /var/cache/ldconfig/aux-cache + +# Download H2ORestart from GitHub using a pinned version and hash. Note that +# it's available in Debian repos, but not in Bookworm yet. RUN mkdir /libreoffice_ext && cd libreoffice_ext \ && H2ORESTART_FILENAME=h2orestart.oxt \ - && H2ORESTART_VERSION="v0.6.6" \ && wget https://github.com/ebandal/H2Orestart/releases/download/$H2ORESTART_VERSION/$H2ORESTART_FILENAME \ && echo "$H2ORESTART_CHECKSUM $H2ORESTART_FILENAME" | sha256sum -c \ - && install -dm777 "/usr/lib/libreoffice/share/extensions/" + && install -dm777 "/usr/lib/libreoffice/share/extensions/" \ + && rm /root/.wget-hsts +# Create an unprivileged user both for gVisor and for running Dangerzone. +RUN mkdir -p /opt/dangerzone/dangerzone && \ + touch /opt/dangerzone/dangerzone/__init__.py && \ + addgroup --gid 1000 dangerzone && \ + adduser --uid 1000 --ingroup dangerzone --shell /bin/true \ + --disabled-password --home /home/dangerzone dangerzone -########################################### -# Dangerzone image - -FROM alpine:latest - -# Install dependencies -RUN apk --no-cache -U upgrade && \ - apk --no-cache add \ - libreoffice \ - openjdk8 \ - python3 \ - py3-magic \ - font-noto-cjk - -COPY --from=pymupdf-build /usr/lib/python3.12/site-packages/fitz/ /usr/lib/python3.12/site-packages/fitz -COPY --from=pymupdf-build /usr/lib/python3.12/site-packages/pymupdf/ /usr/lib/python3.12/site-packages/pymupdf -COPY --from=pymupdf-build /usr/lib/python3.12/site-packages/PyMuPDF.libs/ /usr/lib/python3.12/site-packages/PyMuPDF.libs -COPY --from=h2orestart-dl /libreoffice_ext/ /libreoffice_ext - -RUN install -dm777 "/usr/lib/libreoffice/share/extensions/" - -RUN mkdir -p /opt/dangerzone/dangerzone -RUN touch /opt/dangerzone/dangerzone/__init__.py -COPY conversion /opt/dangerzone/dangerzone/conversion - -# Add the unprivileged user. Set the UID/GID of the dangerzone user/group to -# 1000, since we will point to it from the OCI config. -# -# NOTE: A tmpfs will be mounted over /home/dangerzone directory, -# so nothing within it from the image will be persisted. -RUN addgroup -g 1000 dangerzone && \ - adduser -u 1000 -s /bin/true -G dangerzone -h /home/dangerzone -D dangerzone - -RUN GVISOR_URL="https://storage.googleapis.com/gvisor/releases/release/latest/$(uname -m)"; \ - wget "${GVISOR_URL}/runsc" "${GVISOR_URL}/runsc.sha512" && \ - sha512sum -c runsc.sha512 && \ - rm -f runsc.sha512 && \ - chmod 555 runsc && \ - mv runsc /usr/bin/ +COPY conversion/doc_to_pixels.py \ + conversion/common.py \ + conversion/errors.py \ + conversion/__init__.py \ + /opt/dangerzone/dangerzone/conversion +# Let the entrypoint script write the OCI config for the inner container under +# /config.json. RUN touch /config.json RUN chown dangerzone:dangerzone /config.json diff --git a/dangerzone/conversion/doc_to_pixels.py b/dangerzone/conversion/doc_to_pixels.py index 6737607..a7a7776 100644 --- a/dangerzone/conversion/doc_to_pixels.py +++ b/dangerzone/conversion/doc_to_pixels.py @@ -129,6 +129,8 @@ class DocumentToPixels(DangerzoneConverter): # At least .odt, .docx, .odg, .odp, .ods, and .pptx "application/zip": { "type": "libreoffice", + # NOTE: Older `file` command cannot detect hwpx files properly. + "libreoffice_ext": "h2orestart.oxt", }, # At least .doc, .docx, .odg, .odp, .odt, .pdf, .ppt, .pptx, .xls, and .xlsx "application/octet-stream": { From 460b7a178beb20e9efaac22655fc11e8314dc95c Mon Sep 17 00:00:00 2001 From: Alex Pyrgiotis Date: Mon, 13 Jan 2025 17:37:19 +0200 Subject: [PATCH 07/21] Do not use poetry.lock when building the container image Remove all the scaffolding in our `build-image.py` script for using the `poetry.lock` file, now that we install PyMuPDF from the Debian repos. --- .github/workflows/build.yml | 2 +- .github/workflows/ci.yml | 9 ++- dodo.py | 1 - install/common/build-image.py | 117 ++++++++++---------------------- install/linux/vendor-pymupdf.py | 2 +- poetry.lock | 2 +- pyproject.toml | 2 +- 7 files changed, 44 insertions(+), 91 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d5c1f09..8e53232 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -85,7 +85,7 @@ jobs: id: cache-container-image uses: actions/cache@v4 with: - key: v3-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/*.py', 'dangerzone/container/*', 'install/common/build-image.py', 'poetry.lock') }} + key: v3-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/*.py', 'dangerzone/container/*', 'install/common/build-image.py') }} path: | share/container.tar.gz share/image-id.txt diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 34fb410..b6bb54d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,7 +59,7 @@ jobs: id: cache-container-image uses: actions/cache@v4 with: - key: v3-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/*.py', 'dangerzone/container/*', 'install/common/build-image.py', 'poetry.lock') }} + key: v3-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/*.py', 'dangerzone/container/*', 'install/common/build-image.py') }} path: |- share/container.tar.gz share/image-id.txt @@ -67,7 +67,6 @@ jobs: - name: Build Dangerzone container image if: ${{ steps.cache-container-image.outputs.cache-hit != 'true' }} run: | - sudo apt-get install -y python3-poetry python3 ./install/common/build-image.py - name: Upload container image @@ -227,7 +226,7 @@ jobs: - name: Restore container cache uses: actions/cache/restore@v4 with: - key: v3-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/*.py', 'dangerzone/container/*', 'install/common/build-image.py', 'poetry.lock') }} + key: v3-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/*.py', 'dangerzone/container/*', 'install/common/build-image.py') }} path: |- share/container.tar.gz share/image-id.txt @@ -334,7 +333,7 @@ jobs: - name: Restore container image uses: actions/cache/restore@v4 with: - key: v3-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/*.py', 'dangerzone/container/*', 'install/common/build-image.py', 'poetry.lock') }} + key: v3-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/*.py', 'dangerzone/container/*', 'install/common/build-image.py') }} path: |- share/container.tar.gz share/image-id.txt @@ -429,7 +428,7 @@ jobs: - name: Restore container image uses: actions/cache/restore@v4 with: - key: v3-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/*.py', 'dangerzone/container/*', 'install/common/build-image.py', 'poetry.lock') }} + key: v3-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/*.py', 'dangerzone/container/*', 'install/common/build-image.py') }} path: |- share/container.tar.gz share/image-id.txt diff --git a/dodo.py b/dodo.py index 8fca92a..9172009 100644 --- a/dodo.py +++ b/dodo.py @@ -63,7 +63,6 @@ TESSDATA_TARGETS = list_language_data() IMAGE_DEPS = [ "Dockerfile", - "poetry.lock", *list_files("dangerzone/conversion"), *list_files("dangerzone/container"), "install/common/build-image.py", diff --git a/install/common/build-image.py b/install/common/build-image.py index 6d99877..e2e3b01 100644 --- a/install/common/build-image.py +++ b/install/common/build-image.py @@ -1,6 +1,5 @@ import argparse import gzip -import os import platform import secrets import subprocess @@ -9,7 +8,6 @@ from pathlib import Path BUILD_CONTEXT = "dangerzone/" IMAGE_NAME = "dangerzone.rocks/dangerzone" -REQUIREMENTS_TXT = "container-pip-requirements.txt" if platform.system() in ["Darwin", "Windows"]: CONTAINER_RUNTIME = "docker" elif platform.system() == "Linux": @@ -84,91 +82,48 @@ def main(): with open(image_id_path, "w") as f: f.write(tag) - print("Exporting container pip dependencies") - with ContainerPipDependencies(): - if not args.use_cache: - print("Pulling base image") - subprocess.run( - [ - args.runtime, - "pull", - "alpine:latest", - ], - check=True, - ) + # Build the container image, and tag it with the calculated tag + print("Building container image") + cache_args = [] if args.use_cache else ["--no-cache"] + subprocess.run( + [ + args.runtime, + "build", + BUILD_CONTEXT, + *cache_args, + "-f", + "Dockerfile", + "--tag", + image_name_tagged, + ], + check=True, + ) - # Build the container image, and tag it with the calculated tag - print("Building container image") - cache_args = [] if args.use_cache else ["--no-cache"] - subprocess.run( + if not args.no_save: + print("Saving container image") + cmd = subprocess.Popen( [ - args.runtime, - "build", - BUILD_CONTEXT, - *cache_args, - "--build-arg", - f"REQUIREMENTS_TXT={REQUIREMENTS_TXT}", - "--build-arg", - f"ARCH={ARCH}", - "-f", - "Dockerfile", - "--tag", + CONTAINER_RUNTIME, + "save", image_name_tagged, ], - check=True, + stdout=subprocess.PIPE, ) - if not args.no_save: - print("Saving container image") - cmd = subprocess.Popen( - [ - CONTAINER_RUNTIME, - "save", - image_name_tagged, - ], - stdout=subprocess.PIPE, - ) - - print("Compressing container image") - chunk_size = 4 << 20 - with gzip.open( - tarball_path, - "wb", - compresslevel=args.compress_level, - ) as gzip_f: - while True: - chunk = cmd.stdout.read(chunk_size) - if len(chunk) > 0: - gzip_f.write(chunk) - else: - break - cmd.wait(5) - - -class ContainerPipDependencies: - """Generates PIP dependencies within container""" - - def __enter__(self): - try: - container_requirements_txt = subprocess.check_output( - ["poetry", "export", "--only", "container"], universal_newlines=True - ) - except subprocess.CalledProcessError as e: - print("FAILURE", e.returncode, e.output) - print(f"REQUIREMENTS: {container_requirements_txt}") - # XXX Export container dependencies and exclude pymupdfb since it is not needed in container - req_txt_pymupdfb_stripped = container_requirements_txt.split("pymupdfb")[0] - with open(Path(BUILD_CONTEXT) / REQUIREMENTS_TXT, "w") as f: - if ARCH == "arm64": - # PyMuPDF needs to be built on ARM64 machines - # But is already provided as a prebuilt-wheel on other architectures - f.write(req_txt_pymupdfb_stripped) - else: - f.write(container_requirements_txt) - - def __exit__(self, exc_type, exc_value, exc_tb): - print("Leaving the context...") - os.remove(Path(BUILD_CONTEXT) / REQUIREMENTS_TXT) + print("Compressing container image") + chunk_size = 4 << 20 + with gzip.open( + tarball_path, + "wb", + compresslevel=args.compress_level, + ) as gzip_f: + while True: + chunk = cmd.stdout.read(chunk_size) + if len(chunk) > 0: + gzip_f.write(chunk) + else: + break + cmd.wait(5) if __name__ == "__main__": diff --git a/install/linux/vendor-pymupdf.py b/install/linux/vendor-pymupdf.py index 0c49720..9cb5ccc 100755 --- a/install/linux/vendor-pymupdf.py +++ b/install/linux/vendor-pymupdf.py @@ -28,7 +28,7 @@ def main(): ) logger.info("Getting PyMuPDF deps as requirements.txt") - cmd = ["poetry", "export", "--only", "container"] + cmd = ["poetry", "export", "--only", "debian"] container_requirements_txt = subprocess.check_output(cmd) # XXX: Hack for Ubuntu Focal. diff --git a/poetry.lock b/poetry.lock index 247d955..6c638f7 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1244,4 +1244,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = ">=3.9,<3.13" -content-hash = "68663ce40ba8a7c7f7cc7868e5f771472555773ff2ef04dad7e0150218ca3eb0" +content-hash = "2d7753fa7ee1056d871fe67d718cfa2ea9acdfada1c6c3b1e41f98d5220d3879" diff --git a/pyproject.toml b/pyproject.toml index f69488d..81e0671 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,7 +57,7 @@ strip-ansi = "*" pytest-subprocess = "^1.5.2" pytest-rerunfailures = "^14.0" -[tool.poetry.group.container.dependencies] +[tool.poetry.group.debian.dependencies] pymupdf = "1.24.11" # Last version to support python 3.8 (needed for Ubuntu Focal support) [tool.poetry.group.dev.dependencies] From 1ca3ef9796fe888214e8b2e160036a37d6458d0d Mon Sep 17 00:00:00 2001 From: Alex Pyrgiotis Date: Mon, 13 Jan 2025 17:43:15 +0200 Subject: [PATCH 08/21] Allow using the container engine cache when building our image Remove our suggestions for not using the container cache, which stemmed from the fact that our Dangerzone image was not reproducible. Now that we have switched to Debian Stable and the Dockerfile is all we need to reproducibly build the exact same container image, we can just use the cache to speed up builds. --- docs/developer/doit.md | 14 -------------- dodo.py | 13 +------------ install/common/build-image.py | 4 ++-- pyproject.toml | 7 +------ 4 files changed, 4 insertions(+), 34 deletions(-) diff --git a/docs/developer/doit.md b/docs/developer/doit.md index a461d96..bf8fb16 100644 --- a/docs/developer/doit.md +++ b/docs/developer/doit.md @@ -44,20 +44,6 @@ doit * You can run `doit list --all -s` to see the full list of tasks, their dependencies, and whether they are up to date. * You can run `doit info ` to see which dependencies are missing. -* You can change this line in `pyproject.toml` to `true`, to allow using the - Docker/Podman build cache: - - ``` - use_cache = true - ``` - - > [!WARNING] - > Using caching may speed up image builds, but is not suitable for release - > artifacts. The ID of our base container image (Alpine Linux) does not change - > that often, but its APK package index does. So, if we use caching, we risk - > skipping the `apk upgrade` layer and end up with packages that are days - > behind. - * You can pass the following environment variables to the script, in order to affect some global parameters: - `CONTAINER_RUNTIME`: The container runtime to use. Either `podman` (default) diff --git a/dodo.py b/dodo.py index 9172009..9fa48a8 100644 --- a/dodo.py +++ b/dodo.py @@ -27,16 +27,6 @@ PARAM_APPLE_ID = { "help": "The Apple developer ID that will be used to sign the .dmg", } -PARAM_USE_CACHE = { - "name": "use_cache", - "long": "use-cache", - "help": ( - "Whether to use cached results or not. For reproducibility reasons," - " it's best to leave it to false" - ), - "default": False, -} - ### File dependencies # # Define all the file dependencies for our tasks in a single place, since some file @@ -205,11 +195,10 @@ def task_build_image(): return { "actions": [ - f"python install/common/build-image.py --use-cache=%(use_cache)s --runtime={CONTAINER_RUNTIME}", + f"python install/common/build-image.py --runtime={CONTAINER_RUNTIME}", ["cp", img_src, img_dst], ["cp", img_id_src, img_id_dst], ], - "params": [PARAM_USE_CACHE], "file_dep": IMAGE_DEPS, "targets": [img_src, img_dst, img_id_src, img_id_dst], "task_dep": ["init_release_dir", "check_container_runtime"], diff --git a/install/common/build-image.py b/install/common/build-image.py index e2e3b01..fb1b55f 100644 --- a/install/common/build-image.py +++ b/install/common/build-image.py @@ -51,9 +51,9 @@ def main(): "--use-cache", type=str2bool, nargs="?", - default=False, + default=True, const=True, - help="Use the builder's cache to speed up the builds (not suitable for release builds)", + help="Use the builder's cache to speed up the builds", ) args = parser.parse_args() diff --git a/pyproject.toml b/pyproject.toml index 81e0671..2c9ddf4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,4 +1,4 @@ - [tool.poetry] +[tool.poetry] name = "dangerzone" version = "0.8.1" description = "Take potentially dangerous PDFs, office documents, or images and convert them to safe PDFs" @@ -66,11 +66,6 @@ httpx = "^0.27.2" [tool.doit] verbosity = 3 -[tool.doit.tasks.build_image] -# DO NOT change this to 'true' for release artifacts, else we risk building -# images that are a few days behind. See also: docs/developer/doit.md -use_cache = false - [tool.ruff.lint] select = [ # isort From fccfd510b727f285f3b82e47f76f410dc9c38884 Mon Sep 17 00:00:00 2001 From: Alex Pyrgiotis Date: Mon, 13 Jan 2025 18:10:44 +0200 Subject: [PATCH 09/21] Add jinja2-cli package dependency Add jinja2-cli as a package dependency, since it will be used to create the Dockerfile from some user parameters and a template. --- poetry.lock | 115 ++++++++++++++++++++++++++++++++++++++++++++++--- pyproject.toml | 1 + 2 files changed, 110 insertions(+), 6 deletions(-) diff --git a/poetry.lock b/poetry.lock index 6c638f7..329e202 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.4 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.5 and should not be changed by hand. [[package]] name = "altgraph" @@ -299,8 +299,6 @@ files = [ {file = "cx_Freeze-7.2.7-cp310-cp310-win32.whl", hash = "sha256:c34a6e85897f5cb1be84a204feef564adb6f1c753626bf0cf7713a8c4809ed27"}, {file = "cx_Freeze-7.2.7-cp310-cp310-win_amd64.whl", hash = "sha256:5ea3f05d31a7432b0516a58e4b7301277c0a5ac693597d531d9ca913d79169ce"}, {file = "cx_Freeze-7.2.7-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:65e900cec673dc9bf6b8e21a760b3964b5b20b2586f274184c4c474c78e343f3"}, - {file = "cx_Freeze-7.2.7-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8c9119b8dbe93b362d4658ed817e28d9d2fd9c29d5ade565ca23cd0e049440a8"}, - {file = "cx_Freeze-7.2.7-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3a26b65bd4e2414bbab84307a906ea207de3c1f3198bc9f6286a81597ddebe38"}, {file = "cx_Freeze-7.2.7-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aa7e75e8dd4ffce44465141c2857fcf5fb5a31dfbcabbbee9929b55e79f051e2"}, {file = "cx_Freeze-7.2.7-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b7b1fe00f689a0c06197bef07ba3dce985d768c405e0042dfa796bbafa53d6b4"}, {file = "cx_Freeze-7.2.7-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:34d0ec5a41d59c55878711cf3812f46444d71f911c8ed1de467591c36aed9d6d"}, @@ -310,8 +308,6 @@ files = [ {file = "cx_Freeze-7.2.7-cp311-cp311-win32.whl", hash = "sha256:909784281dad31c92c0f402894d99a6188bb81f22b593d9d55fc437d54ab35e0"}, {file = "cx_Freeze-7.2.7-cp311-cp311-win_amd64.whl", hash = "sha256:8fff35321128f680a825d28074edd14b81bdd68b98128d4fa655c6c8901de3ce"}, {file = "cx_Freeze-7.2.7-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:6e22b449cf401a7d152e97a5feec71b492083067ca0336b74eb988fbfebfb3a6"}, - {file = "cx_Freeze-7.2.7-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:8611df6e740a7980145f2d6a3d9f9161fe4d6a12c0055da52ccccffe7d77d470"}, - {file = "cx_Freeze-7.2.7-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:66fa38df2423a3ded9c59a6a089293db3b7a6d3493a701b31d24f97cd29f2143"}, {file = "cx_Freeze-7.2.7-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:41c6fd1e77a733fc1a0f4e3678e136726cc45a11b37ff5722ac18afdad703934"}, {file = "cx_Freeze-7.2.7-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d00dc0bf7ed9a30491ec34bfb8f4347446afae506febc2afc2531ecbf9db94bf"}, {file = "cx_Freeze-7.2.7-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a5a27b2a1925138e122435ef3d52c281d185a57d1f67e08232ea61be83174cd0"}, @@ -522,6 +518,43 @@ files = [ {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, ] +[[package]] +name = "jinja2" +version = "3.1.5" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ + {file = "jinja2-3.1.5-py3-none-any.whl", hash = "sha256:aba0f4dc9ed8013c424088f68a5c226f7d6097ed89b246d7749c2ec4175c6adb"}, + {file = "jinja2-3.1.5.tar.gz", hash = "sha256:8fefff8dc3034e27bb80d67c671eb8a9bc424c0ef4c0826edbff304cceff43bb"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "jinja2-cli" +version = "0.8.2" +description = "A CLI interface to Jinja2" +optional = false +python-versions = "*" +files = [ + {file = "jinja2-cli-0.8.2.tar.gz", hash = "sha256:a16bb1454111128e206f568c95938cdef5b5a139929378f72bb8cf6179e18e50"}, + {file = "jinja2_cli-0.8.2-py2.py3-none-any.whl", hash = "sha256:b91715c79496beaddad790171e7258a87db21c1a0b6d2b15bca3ba44b74aac5d"}, +] + +[package.dependencies] +jinja2 = "*" + +[package.extras] +tests = ["flake8", "jinja2", "pytest"] +toml = ["jinja2", "toml"] +xml = ["jinja2", "xmltodict"] +yaml = ["jinja2", "pyyaml"] + [[package]] name = "lief" version = "0.15.1" @@ -611,6 +644,76 @@ importlib-metadata = {version = ">=4.4", markers = "python_version < \"3.10\""} docs = ["mdx-gh-links (>=0.2)", "mkdocs (>=1.5)", "mkdocs-gen-files", "mkdocs-literate-nav", "mkdocs-nature (>=0.6)", "mkdocs-section-index", "mkdocstrings[python]"] testing = ["coverage", "pyyaml"] +[[package]] +name = "markupsafe" +version = "3.0.2" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.9" +files = [ + {file = "MarkupSafe-3.0.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:7e94c425039cde14257288fd61dcfb01963e658efbc0ff54f5306b06054700f8"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9e2d922824181480953426608b81967de705c3cef4d1af983af849d7bd619158"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:38a9ef736c01fccdd6600705b09dc574584b89bea478200c5fbf112a6b0d5579"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bbcb445fa71794da8f178f0f6d66789a28d7319071af7a496d4d507ed566270d"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:57cb5a3cf367aeb1d316576250f65edec5bb3be939e9247ae594b4bcbc317dfb"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:3809ede931876f5b2ec92eef964286840ed3540dadf803dd570c3b7e13141a3b"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:e07c3764494e3776c602c1e78e298937c3315ccc9043ead7e685b7f2b8d47b3c"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:b424c77b206d63d500bcb69fa55ed8d0e6a3774056bdc4839fc9298a7edca171"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-win32.whl", hash = "sha256:fcabf5ff6eea076f859677f5f0b6b5c1a51e70a376b0579e0eadef8db48c6b50"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:6af100e168aa82a50e186c82875a5893c5597a0c1ccdb0d8b40240b1f28b969a"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9025b4018f3a1314059769c7bf15441064b2207cb3f065e6ea1e7359cb46db9d"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:93335ca3812df2f366e80509ae119189886b0f3c2b81325d39efdb84a1e2ae93"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2cb8438c3cbb25e220c2ab33bb226559e7afb3baec11c4f218ffa7308603c832"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a123e330ef0853c6e822384873bef7507557d8e4a082961e1defa947aa59ba84"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e084f686b92e5b83186b07e8a17fc09e38fff551f3602b249881fec658d3eca"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d8213e09c917a951de9d09ecee036d5c7d36cb6cb7dbaece4c71a60d79fb9798"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:5b02fb34468b6aaa40dfc198d813a641e3a63b98c2b05a16b9f80b7ec314185e"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0bff5e0ae4ef2e1ae4fdf2dfd5b76c75e5c2fa4132d05fc1b0dabcd20c7e28c4"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-win32.whl", hash = "sha256:6c89876f41da747c8d3677a2b540fb32ef5715f97b66eeb0c6b66f5e3ef6f59d"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:70a87b411535ccad5ef2f1df5136506a10775d267e197e4cf531ced10537bd6b"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:9778bd8ab0a994ebf6f84c2b949e65736d5575320a17ae8984a77fab08db94cf"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:846ade7b71e3536c4e56b386c2a47adf5741d2d8b94ec9dc3e92e5e1ee1e2225"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1c99d261bd2d5f6b59325c92c73df481e05e57f19837bdca8413b9eac4bd8028"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e17c96c14e19278594aa4841ec148115f9c7615a47382ecb6b82bd8fea3ab0c8"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:88416bd1e65dcea10bc7569faacb2c20ce071dd1f87539ca2ab364bf6231393c"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2181e67807fc2fa785d0592dc2d6206c019b9502410671cc905d132a92866557"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:52305740fe773d09cffb16f8ed0427942901f00adedac82ec8b67752f58a1b22"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ad10d3ded218f1039f11a75f8091880239651b52e9bb592ca27de44eed242a48"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-win32.whl", hash = "sha256:0f4ca02bea9a23221c0182836703cbf8930c5e9454bacce27e767509fa286a30"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:8e06879fc22a25ca47312fbe7c8264eb0b662f6db27cb2d3bbbc74b1df4b9b87"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ba9527cdd4c926ed0760bc301f6728ef34d841f405abf9d4f959c478421e4efd"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f8b3d067f2e40fe93e1ccdd6b2e1d16c43140e76f02fb1319a05cf2b79d99430"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:569511d3b58c8791ab4c2e1285575265991e6d8f8700c7be0e88f86cb0672094"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:15ab75ef81add55874e7ab7055e9c397312385bd9ced94920f2802310c930396"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f3818cb119498c0678015754eba762e0d61e5b52d34c8b13d770f0719f7b1d79"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:cdb82a876c47801bb54a690c5ae105a46b392ac6099881cdfb9f6e95e4014c6a"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:cabc348d87e913db6ab4aa100f01b08f481097838bdddf7c7a84b7575b7309ca"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:444dcda765c8a838eaae23112db52f1efaf750daddb2d9ca300bcae1039adc5c"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-win32.whl", hash = "sha256:bcf3e58998965654fdaff38e58584d8937aa3096ab5354d493c77d1fdd66d7a1"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:e6a2a455bd412959b57a172ce6328d2dd1f01cb2135efda2e4576e8a23fa3b0f"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:b5a6b3ada725cea8a5e634536b1b01c30bcdcd7f9c6fff4151548d5bf6b3a36c"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:a904af0a6162c73e3edcb969eeeb53a63ceeb5d8cf642fade7d39e7963a22ddb"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4aa4e5faecf353ed117801a068ebab7b7e09ffb6e1d5e412dc852e0da018126c"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0ef13eaeee5b615fb07c9a7dadb38eac06a0608b41570d8ade51c56539e509d"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d16a81a06776313e817c951135cf7340a3e91e8c1ff2fac444cfd75fffa04afe"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:6381026f158fdb7c72a168278597a5e3a5222e83ea18f543112b2662a9b699c5"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:3d79d162e7be8f996986c064d1c7c817f6df3a77fe3d6859f6f9e7be4b8c213a"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:131a3c7689c85f5ad20f9f6fb1b866f402c445b220c19fe4308c0b147ccd2ad9"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-win32.whl", hash = "sha256:ba8062ed2cf21c07a9e295d5b8a2a5ce678b913b45fdf68c32d95d6c1291e0b6"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-win_amd64.whl", hash = "sha256:e444a31f8db13eb18ada366ab3cf45fd4b31e4db1236a4448f68778c1d1a5a2f"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:eaa0a10b7f72326f1372a713e73c3f739b524b3af41feb43e4921cb529f5929a"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:48032821bbdf20f5799ff537c7ac3d1fba0ba032cfc06194faffa8cda8b560ff"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1a9d3f5f0901fdec14d8d2f66ef7d035f2157240a433441719ac9a3fba440b13"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:88b49a3b9ff31e19998750c38e030fc7bb937398b1f78cfa599aaef92d693144"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cfad01eed2c2e0c01fd0ecd2ef42c492f7f93902e39a42fc9ee1692961443a29"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:1225beacc926f536dc82e45f8a4d68502949dc67eea90eab715dea3a21c1b5f0"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:3169b1eefae027567d1ce6ee7cae382c57fe26e82775f460f0b2778beaad66c0"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:eb7972a85c54febfb25b5c4b4f3af4dcc731994c7da0d8a0b4a6eb0640e1d178"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-win32.whl", hash = "sha256:8c4e8c3ce11e1f92f6536ff07154f9d49677ebaaafc32db9db4620bc11ed480f"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:6e296a513ca3d94054c2c881cc913116e90fd030ad1c656b3869762b754f5f8a"}, + {file = "markupsafe-3.0.2.tar.gz", hash = "sha256:ee55d3edf80167e48ea11a923c7386f4669df67d7994554387f84e7d8b0a2bf0"}, +] + [[package]] name = "mypy" version = "1.13.0" @@ -1244,4 +1347,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = ">=3.9,<3.13" -content-hash = "2d7753fa7ee1056d871fe67d718cfa2ea9acdfada1c6c3b1e41f98d5220d3879" +content-hash = "7d87c326970a3c85842c97a11df570d694ea89fbcafd3e94978ed66a4138c7da" diff --git a/pyproject.toml b/pyproject.toml index 2c9ddf4..1eb52c7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,6 +35,7 @@ cx_freeze = {version = "^7.2.5", platform = "win32"} pywin32 = {version = "*", platform = "win32"} pyinstaller = {version = "*", platform = "darwin"} doit = "^0.36.0" +jinja2-cli = "^0.8.2" # Dependencies required for linting the code. [tool.poetry.group.lint.dependencies] From a8436bba98be34c1aee702184e249e3486749a0d Mon Sep 17 00:00:00 2001 From: Alex Pyrgiotis Date: Mon, 13 Jan 2025 18:11:57 +0200 Subject: [PATCH 10/21] Render the Dockerfile from a template and some params Allow updating the Dockerfile from a template and some envs, so that it's easier to bump the dates in it. --- BUILD.md | 6 +++ Dockerfile.env | 5 ++ Dockerfile.in | 81 +++++++++++++++++++++++++++++++ RELEASE.md | 1 + docs/developer/reproducibility.md | 21 ++++++++ 5 files changed, 114 insertions(+) create mode 100644 Dockerfile.env create mode 100644 Dockerfile.in diff --git a/BUILD.md b/BUILD.md index bd1b377..7a47da3 100644 --- a/BUILD.md +++ b/BUILD.md @@ -515,3 +515,9 @@ poetry run .\install\windows\build-app.bat ``` When you're done you will have `dist\Dangerzone.msi`. + +## Updating the container image + +The Dangezone container image is reproducible. This means that every time we +build it, the result will be bit-for-bit the same, with some minor exceptions. +Read more on how you can update it in `docs/developer/reproducibility.md`. diff --git a/Dockerfile.env b/Dockerfile.env new file mode 100644 index 0000000..cae3fc9 --- /dev/null +++ b/Dockerfile.env @@ -0,0 +1,5 @@ +DEBIAN_IMAGE_DATE=20250113 +DEBIAN_ARCHIVE_DATE=20250114 +GVISOR_ARCHIVE_DATE=20250106 +H2ORESTART_CHECKSUM=8a5be77359695c14faaf33891d3eca6c9d73c1224599aab50a9d2ccc04640580 +H2ORESTART_VERSION=v0.6.8 diff --git a/Dockerfile.in b/Dockerfile.in new file mode 100644 index 0000000..0bf5ee2 --- /dev/null +++ b/Dockerfile.in @@ -0,0 +1,81 @@ +# NOTE: Updating the packages to their latest versions requires bumping the +# Dockerfile args below. For more info about this file, read +# docs/developer/reproducibility.md. + +ARG DEBIAN_IMAGE_DATE={{DEBIAN_IMAGE_DATE}} + +FROM debian:bookworm-${DEBIAN_IMAGE_DATE}-slim + +ARG GVISOR_ARCHIVE_DATE={{GVISOR_ARCHIVE_DATE}} +ARG DEBIAN_ARCHIVE_DATE={{DEBIAN_ARCHIVE_DATE}} +ARG H2ORESTART_CHECKSUM=8a5be77359695c14faaf33891d3eca6c9d73c1224599aab50a9d2ccc04640580 +ARG H2ORESTART_VERSION=v0.6.8 + +ENV DEBIAN_FRONTEND=noninteractive + +# The following way of installing packages is taken from +# https://github.com/reproducible-containers/repro-sources-list.sh/blob/master/Dockerfile.debian-12, +# and adapted to allow installing gVisor from each own repo as well. +RUN \ + --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + --mount=type=bind,source=./container/repro-sources-list.sh,target=/usr/local/bin/repro-sources-list.sh \ + --mount=type=bind,source=./container/gvisor.key,target=/tmp/gvisor.key \ + : "Hacky way to set a date for the Debian snapshot repos" && \ + touch -d ${DEBIAN_ARCHIVE_DATE} /etc/apt/sources.list.d/debian.sources && \ + touch -d ${DEBIAN_ARCHIVE_DATE} /etc/apt/sources.list && \ + repro-sources-list.sh && \ + : "Setup APT to install gVisor from its separate APT repo" && \ + apt-get update && \ + apt-get upgrade -y && \ + apt-get install -y --no-install-recommends apt-transport-https ca-certificates gnupg && \ + gpg -o /usr/share/keyrings/gvisor-archive-keyring.gpg --dearmor /tmp/gvisor.key && \ + echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/gvisor-archive-keyring.gpg] https://storage.googleapis.com/gvisor/releases ${GVISOR_ARCHIVE_DATE} main" > /etc/apt/sources.list.d/gvisor.list && \ + : "Install the necessary gVisor and Dangerzone dependencies" && \ + apt-get update && \ + apt-get install -y --no-install-recommends \ + python3 python3-fitz libreoffice-nogui libreoffice-java-common \ + python3 python3-magic default-jre-headless fonts-noto-cjk fonts-dejavu \ + runsc unzip wget && \ + : "Clean up for improving reproducibility (optional)" && \ + rm -rf /var/cache/fontconfig/ && \ + rm -rf /etc/ssl/certs/java/cacerts && \ + rm -rf /var/log/* /var/cache/ldconfig/aux-cache + +# Download H2ORestart from GitHub using a pinned version and hash. Note that +# it's available in Debian repos, but not in Bookworm yet. +RUN mkdir /libreoffice_ext && cd libreoffice_ext \ + && H2ORESTART_FILENAME=h2orestart.oxt \ + && wget https://github.com/ebandal/H2Orestart/releases/download/$H2ORESTART_VERSION/$H2ORESTART_FILENAME \ + && echo "$H2ORESTART_CHECKSUM $H2ORESTART_FILENAME" | sha256sum -c \ + && install -dm777 "/usr/lib/libreoffice/share/extensions/" \ + && rm /root/.wget-hsts + +# Create an unprivileged user both for gVisor and for running Dangerzone. +RUN mkdir -p /opt/dangerzone/dangerzone && \ + touch /opt/dangerzone/dangerzone/__init__.py && \ + addgroup --gid 1000 dangerzone && \ + adduser --uid 1000 --ingroup dangerzone --shell /bin/true \ + --disabled-password --home /home/dangerzone dangerzone + +COPY conversion/doc_to_pixels.py \ + conversion/common.py \ + conversion/errors.py \ + conversion/__init__.py \ + /opt/dangerzone/dangerzone/conversion + +# Let the entrypoint script write the OCI config for the inner container under +# /config.json. +RUN touch /config.json +RUN chown dangerzone:dangerzone /config.json + +# Switch to the dangerzone user for the rest of the script. +USER dangerzone + +# Create a directory that will be used by gVisor as the place where it will +# store the state of its containers. +RUN mkdir /home/dangerzone/.containers + +COPY container/entrypoint.py / + +ENTRYPOINT ["/entrypoint.py"] diff --git a/RELEASE.md b/RELEASE.md index 6e58338..b0a7e8c 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -14,6 +14,7 @@ Here is a list of tasks that should be done before issuing the release: - [ ] Update `share/version.txt` - [ ] Update the "Version" field in `install/linux/dangerzone.spec` - [ ] Bump the Debian version by adding a new changelog entry in `debian/changelog` +- [ ] Bump the dates in the `Dockerfile` - [ ] Update screenshot in `README.md`, if necessary - [ ] CHANGELOG.md should be updated to include a list of all major changes since the last release - [ ] A draft release should be created. Copy the release notes text from the template at [`docs/templates/release-notes`](https://github.com/freedomofpress/dangerzone/tree/main/docs/templates/) diff --git a/docs/developer/reproducibility.md b/docs/developer/reproducibility.md index 4fcc0f4..bd099b7 100644 --- a/docs/developer/reproducibility.md +++ b/docs/developer/reproducibility.md @@ -88,3 +88,24 @@ Here are a few other obstacles that we need to overcome: ./diffoci diff podman:// podman:// \ --ignore-timestamps --ignore-image-name --verbose ``` + +### Updating the image + +The fact that our image is reproducible also means that it's frozen in time. +This means that rebuilding the image without updating our Dockerfile will not +receive security updates. + +We list the necessary variables that make up our image in the `Dockerfile.env` +file. These are: +* `DEBIAN_IMAGE_DATE`: The date that the Debian container image was released +* `DEBIAN_ARCHIVE_DATE`: The Debian snapshot repo that we want to use +* `GVISOR_ARCHIVE_DATE`: The gVisor APT repo that we want to use +* `H2ORESTART_CHECKSUM`: The SHA-256 checksum of the H2ORestart plugin +* `H2ORESTART_VERSION`: The version of the H2ORestart plugin + +If you bump these values in `Dockerfile.env`, you can create a new Dockerfile +with: + +``` +poetry run jinja2 Dockerfile.in Dockerfile.env > Dockerfile +``` From 375efe5af4658e70ad966f3753ee6b75ef5d1e20 Mon Sep 17 00:00:00 2001 From: Alex Pyrgiotis Date: Tue, 14 Jan 2025 10:41:09 +0200 Subject: [PATCH 11/21] Allow setting a tag for the container image Allow setting a tag for the container image, when building it with the `build-image.py` script. This should be used for development purposes only, since the proper image name should be dictated by the script. --- install/common/build-image.py | 41 ++++++++++++++++++++++++----------- 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/install/common/build-image.py b/install/common/build-image.py index fb1b55f..cca9b5f 100644 --- a/install/common/build-image.py +++ b/install/common/build-image.py @@ -55,6 +55,11 @@ def main(): const=True, help="Use the builder's cache to speed up the builds", ) + parser.add_argument( + "--tag", + default=None, + help="Provide a custom tag for the image (for development only)", + ) args = parser.parse_args() tarball_path = Path("share") / "container.tar.gz" @@ -62,20 +67,30 @@ def main(): print(f"Building for architecture '{ARCH}'") - # Designate a unique tag for this image, depending on the Git commit it was created - # from: - # 1. If created from a Git tag (e.g., 0.8.0), the image tag will be `0.8.0`. - # 2. If created from a commit, it will be something like `0.8.0-31-g6bdaa7a`. - # 3. If the contents of the Git repo are dirty, we will append a unique identifier - # for this run, something like `0.8.0-31-g6bdaa7a-fdcb` or `0.8.0-fdcb`. - dirty_ident = secrets.token_hex(2) - tag = ( - subprocess.check_output( - ["git", "describe", "--long", "--first-parent", f"--dirty=-{dirty_ident}"], + if args.tag: + tag = args.tag + else: + # Designate a unique tag for this image, depending on the Git commit it was created + # from: + # 1. If created from a Git tag (e.g., 0.8.0), the image tag will be `0.8.0`. + # 2. If created from a commit, it will be something like `0.8.0-31-g6bdaa7a`. + # 3. If the contents of the Git repo are dirty, we will append a unique identifier + # for this run, something like `0.8.0-31-g6bdaa7a-fdcb` or `0.8.0-fdcb`. + dirty_ident = secrets.token_hex(2) + tag = ( + subprocess.check_output( + [ + "git", + "describe", + "--long", + "--first-parent", + f"--dirty=-{dirty_ident}", + ], + ) + .decode() + .strip()[1:] # remove the "v" prefix of the tag. ) - .decode() - .strip()[1:] # remove the "v" prefix of the tag. - ) + image_name_tagged = IMAGE_NAME + ":" + tag print(f"Will tag the container image as '{image_name_tagged}'") From 7a59940493989b9954fb193280373c3b41509ff5 Mon Sep 17 00:00:00 2001 From: Alex Pyrgiotis Date: Tue, 14 Jan 2025 10:49:59 +0200 Subject: [PATCH 12/21] dev_scripts: Add script for enforcing image reproducibility Add a dev script for Linux platforms that verifies that a source image can be reproducibly built from the current Git commit. The reproducibility check is enforced by the `diffoci` tool, which is downloaded as part of running the script. --- dev_scripts/reproduce.py | 151 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 151 insertions(+) create mode 100755 dev_scripts/reproduce.py diff --git a/dev_scripts/reproduce.py b/dev_scripts/reproduce.py new file mode 100755 index 0000000..b37d69f --- /dev/null +++ b/dev_scripts/reproduce.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 + +import argparse +import hashlib +import logging +import pathlib +import stat +import subprocess +import sys +import urllib.request + +logger = logging.getLogger(__name__) + +DIFFOCI_URL = "https://github.com/reproducible-containers/diffoci/releases/download/v0.1.5/diffoci-v0.1.5.linux-amd64" +DIFFOCI_CHECKSUM = "01d25fe690196945a6bd510d30559338aa489c034d3a1b895a0d82a4b860698f" +DIFFOCI_PATH = ( + pathlib.Path.home() / ".local" / "share" / "dangerzone-dev" / "helpers" / "diffoci" +) + + +def run(*args): + """Simple function that runs a command, validates it, and returns the output""" + logger.debug(f"Running command: {" ".join(args)}") + return subprocess.run( + args, + check=True, + stdout=subprocess.PIPE, + # stderr=subprocess.PIPE + ).stdout + + +def git_commit_get(): + return run("git", "rev-parse", "--short", "HEAD").decode().strip() + + +def git_verify(commit, source): + if not commit in source: + raise RuntimeError( + f"Image '{image}' does not seem to be built from commit '{commit}'" + ) + + +def diffoci_download(): + """Download the diffoci tool, based on a URL and its checksum.""" + with urllib.request.urlopen(DIFFOCI_URL) as f: + diffoci_bin = f.read() + + m = hashlib.sha256() + m.update(diffoci_bin) + diffoci_checksum = m.hexdigest() + if not diffoci_checksum == DIFFOCI_CHECKSUM: + raise ValueError( + "Unexpected checksum for downloaded diffoci binary:" + f" {diffoci_checksum} !={DIFFOCI_CHECKSUM}" + ) + + DIFFOCI_PATH.parent.mkdir(parents=True, exist_ok=True) + DIFFOCI_PATH.open("wb+").write(diffoci_bin) + DIFFOCI_PATH.chmod(DIFFOCI_PATH.stat().st_mode | stat.S_IEXEC) + + +def diffoci_diff(source, local_target): + """Diff the source image against the recently built target image using diffoci.""" + target = f"podman://{local_target}" + try: + return run( + str(DIFFOCI_PATH), + "diff", + source, + target, + "--ignore-timestamps", + "--ignore-image-name", + "--verbose", + ) + except subprocess.CalledProcessError as e: + error = e.stdout.decode() + raise RuntimeError( + f"Could not rebuild an identical image to {source}. Diffoci report:\n{error}" + ) + + +def build_image(tag, use_cache=False): + """Build the Dangerzone container image with a special tag.""" + run( + "python3", + "./install/common/build-image.py", + "--no-save", + "--use-cache", + str(use_cache), + "--tag", + tag, + ) + + +def parse_args(): + parser = argparse.ArgumentParser( + prog=sys.argv[0], + description="Dev script for verifying container image reproducibility", + ) + parser.add_argument( + "--source", + required=True, + help="The source image name that you want to reproduce (in diffoci format)", + ) + parser.add_argument( + "--use-cache", + default=False, + action="store_true", + help="Whether to reuse the build cache (off by default for better reproducibility)", + ) + return parser.parse_args() + + +def main(): + logging.basicConfig( + level=logging.DEBUG, + format="%(asctime)s - %(levelname)s - %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + ) + args = parse_args() + + logger.info(f"Ensuring that current Git commit matches image '{args.source}'") + commit = git_commit_get() + git_verify(commit, args.source) + + if not DIFFOCI_PATH.exists(): + logger.info(f"Downloading diffoci helper from {DIFFOCI_URL}") + diffoci_download() + + tag = f"reproduce-{commit}" + target = f"dangerzone.rocks/dangerzone:{tag}" + logger.info(f"Building container image and tagging it as '{target}'") + build_image(tag, args.use_cache) + + logger.info( + f"Ensuring that source image '{args.source}' is semantically identical with" + f" built image '{target}'" + ) + try: + diffoci_diff(args.source, target) + except subprocess.CalledProcessError as e: + raise RuntimeError( + f"Could not reproduce image {args.source} for commit {commit}" + ) + breakpoint() + + logger.info(f"Successfully reproduced image '{args.source}' from commit '{commit}'") + + +if __name__ == "__main__": + sys.exit(main()) From 279322bf438a9344bc6c5e52881870dbb0cea15e Mon Sep 17 00:00:00 2001 From: Alex Pyrgiotis Date: Tue, 14 Jan 2025 10:51:33 +0200 Subject: [PATCH 13/21] ci: Add a CI job that enforces image reproducibility Add a CI job that uses the `reproduce.py` dev script to enforce image reproducibility, for every PR that we send to the repo. Fixes #1047 --- .github/workflows/ci.yml | 38 +++++++++++++++++++++++++++++++ docs/developer/reproducibility.md | 14 ++++++++++++ 2 files changed, 52 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b6bb54d..fc92043 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -471,3 +471,41 @@ jobs: # file successfully. xvfb-run -s '-ac' ./dev_scripts/env.py --distro ${{ matrix.distro }} --version ${{ matrix.version }} run --dev \ bash -c 'cd dangerzone; poetry run make test' + + reproduce-image: + needs: + - build-container-image + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Install dev. dependencies + run: |- + sudo apt-get update + sudo apt-get install -y git python3-poetry --no-install-recommends + poetry install --only package + + - name: Verify that the Dockerfile matches the commited template and params + run: |- + poetry run jinja2 Dockerfile.in Dockerfile.env > out + diff Dockerfile out + + - name: Get current date + id: date + run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT + + - name: Restore container cache + uses: actions/cache/restore@v4 + with: + key: v3-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/*.py', 'dangerzone/container/*', 'install/common/build-image.py') }} + path: |- + share/container.tar.gz + share/image-id.txt + fail-on-cache-miss: true + + - name: Reproduce the same container image + run: | + podman load -i share/container.tar.gz + ./dev_scripts/reproduce.py --source podman://dangerzone.rocks/dangerzone:$(cat share/image-id.txt) diff --git a/docs/developer/reproducibility.md b/docs/developer/reproducibility.md index bd099b7..4146526 100644 --- a/docs/developer/reproducibility.md +++ b/docs/developer/reproducibility.md @@ -109,3 +109,17 @@ with: ``` poetry run jinja2 Dockerfile.in Dockerfile.env > Dockerfile ``` + +### Reproducing the image + +For a simple way to reproduce a Dangerzone container image, either local or +pushed to a container registry, you can run the following command in a Linux +environment: + +``` +./dev_scripts/reproduce.py +``` + +This command will download the `diffoci` helper, build a container image from +the current Git commit, and ensure that the built image matches the source one, +with the exception of image names and file timestamps. From d53c4d06b5bf2bea56e77b86bb7a8211353be67b Mon Sep 17 00:00:00 2001 From: Alex Pyrgiotis Date: Tue, 14 Jan 2025 12:07:45 +0200 Subject: [PATCH 14/21] fixup! Render the Dockerfile from a template and some params --- poetry.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index 329e202..4b3cec7 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1347,4 +1347,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = ">=3.9,<3.13" -content-hash = "7d87c326970a3c85842c97a11df570d694ea89fbcafd3e94978ed66a4138c7da" +content-hash = "2d7753fa7ee1056d871fe67d718cfa2ea9acdfada1c6c3b1e41f98d5220d3879" From e02dbfdc7932fef6331557d942f1374fb385e8f3 Mon Sep 17 00:00:00 2001 From: Alex Pyrgiotis Date: Tue, 14 Jan 2025 12:25:01 +0200 Subject: [PATCH 15/21] WIP: Reproduce --- .github/workflows/ci.yml | 15 +++------------ .github/workflows/scan.yml | 10 +++++++--- dev_scripts/reproduce.py | 2 +- 3 files changed, 11 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fc92043..ec06d0b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -492,18 +492,9 @@ jobs: poetry run jinja2 Dockerfile.in Dockerfile.env > out diff Dockerfile out - - name: Get current date - id: date - run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT - - - name: Restore container cache - uses: actions/cache/restore@v4 - with: - key: v3-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/*.py', 'dangerzone/container/*', 'install/common/build-image.py') }} - path: |- - share/container.tar.gz - share/image-id.txt - fail-on-cache-miss: true + - name: Build Dangerzone container image + run: | + python3 ./install/common/build-image.py --no-save - name: Reproduce the same container image run: | diff --git a/.github/workflows/scan.yml b/.github/workflows/scan.yml index c4bd6a3..f7c187f 100644 --- a/.github/workflows/scan.yml +++ b/.github/workflows/scan.yml @@ -21,13 +21,17 @@ jobs: sudo apt install pipx pipx install poetry pipx inject poetry poetry-plugin-export + poetry install --only package + - name: Bump date of Debian snapshot archive + run: | + date=$(date "+%Y%m%d") + sed -i "s/DEBIAN_ARCHIVE_DATE=[0-9]\+/DEBIAN_ARCHIVE_DATE=${date}/" Dockerfile.env + poetry run jinja2 Dockerfile.in Dockerfile.env > Dockerfile - name: Build container image run: python3 ./install/common/build-image.py --runtime docker --no-save - name: Get image tag id: tag - run: | - tag=$(docker images dangerzone.rocks/dangerzone --format '{{ .Tag }}') - echo "tag=$tag" >> $GITHUB_OUTPUT + run: echo "tag=$(cat share/image-id.txt)" >> $GITHUB_OUTPUT # NOTE: Scan first without failing, else we won't be able to read the scan # report. - name: Scan container image (no fail) diff --git a/dev_scripts/reproduce.py b/dev_scripts/reproduce.py index b37d69f..0c05c4c 100755 --- a/dev_scripts/reproduce.py +++ b/dev_scripts/reproduce.py @@ -36,7 +36,7 @@ def git_commit_get(): def git_verify(commit, source): if not commit in source: raise RuntimeError( - f"Image '{image}' does not seem to be built from commit '{commit}'" + f"Image '{source}' does not seem to be built from commit '{commit}'" ) From 45f43964a597ddef8482715c5603fbefcb2f7d9c Mon Sep 17 00:00:00 2001 From: Alex Pyrgiotis Date: Tue, 14 Jan 2025 14:19:12 +0200 Subject: [PATCH 16/21] fixup! Do not use poetry.lock when building the container image --- .github/workflows/build.yml | 2 +- .github/workflows/ci.yml | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8e53232..c4c3b13 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -85,7 +85,7 @@ jobs: id: cache-container-image uses: actions/cache@v4 with: - key: v3-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/*.py', 'dangerzone/container/*', 'install/common/build-image.py') }} + key: v4-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/*.py', 'dangerzone/container/*', 'install/common/build-image.py') }} path: | share/container.tar.gz share/image-id.txt diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ec06d0b..8f3416f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,7 +59,7 @@ jobs: id: cache-container-image uses: actions/cache@v4 with: - key: v3-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/*.py', 'dangerzone/container/*', 'install/common/build-image.py') }} + key: v4-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/*.py', 'dangerzone/container/*', 'install/common/build-image.py') }} path: |- share/container.tar.gz share/image-id.txt @@ -226,7 +226,7 @@ jobs: - name: Restore container cache uses: actions/cache/restore@v4 with: - key: v3-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/*.py', 'dangerzone/container/*', 'install/common/build-image.py') }} + key: v4-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/*.py', 'dangerzone/container/*', 'install/common/build-image.py') }} path: |- share/container.tar.gz share/image-id.txt @@ -333,7 +333,7 @@ jobs: - name: Restore container image uses: actions/cache/restore@v4 with: - key: v3-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/*.py', 'dangerzone/container/*', 'install/common/build-image.py') }} + key: v4-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/*.py', 'dangerzone/container/*', 'install/common/build-image.py') }} path: |- share/container.tar.gz share/image-id.txt @@ -428,7 +428,7 @@ jobs: - name: Restore container image uses: actions/cache/restore@v4 with: - key: v3-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/*.py', 'dangerzone/container/*', 'install/common/build-image.py') }} + key: v4-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/*.py', 'dangerzone/container/*', 'install/common/build-image.py') }} path: |- share/container.tar.gz share/image-id.txt From b42bd67f6c5e7c58ba30c4bee56b007faea3ad7b Mon Sep 17 00:00:00 2001 From: Alex Pyrgiotis Date: Tue, 14 Jan 2025 14:20:36 +0200 Subject: [PATCH 17/21] FIXUP: Remove stray podmna load command --- .github/workflows/ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8f3416f..2394f86 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -498,5 +498,4 @@ jobs: - name: Reproduce the same container image run: | - podman load -i share/container.tar.gz ./dev_scripts/reproduce.py --source podman://dangerzone.rocks/dangerzone:$(cat share/image-id.txt) From 10b8cd48afb5fc4215b90f96791bc91aafbea43f Mon Sep 17 00:00:00 2001 From: Alex Pyrgiotis Date: Tue, 14 Jan 2025 14:21:39 +0200 Subject: [PATCH 18/21] FIXUP: Change name of reproduce CI job --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2394f86..a78f2f4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -472,7 +472,7 @@ jobs: xvfb-run -s '-ac' ./dev_scripts/env.py --distro ${{ matrix.distro }} --version ${{ matrix.version }} run --dev \ bash -c 'cd dangerzone; poetry run make test' - reproduce-image: + check-reproducibility: needs: - build-container-image runs-on: ubuntu-latest From 96ab442873f32b650da642f487bb0702c30cb304 Mon Sep 17 00:00:00 2001 From: Alex Pyrgiotis Date: Tue, 14 Jan 2025 14:46:32 +0200 Subject: [PATCH 19/21] FIXUP: Add links for container params --- Dockerfile | 4 ++-- Dockerfile.env | 8 ++++++-- Dockerfile.in | 4 ++-- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index d1f2c10..c334784 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,8 +8,8 @@ FROM debian:bookworm-${DEBIAN_IMAGE_DATE}-slim ARG GVISOR_ARCHIVE_DATE=20250106 ARG DEBIAN_ARCHIVE_DATE=20250114 -ARG H2ORESTART_CHECKSUM=8a5be77359695c14faaf33891d3eca6c9d73c1224599aab50a9d2ccc04640580 -ARG H2ORESTART_VERSION=v0.6.8 +ARG H2ORESTART_CHECKSUM=7760dc2963332c50d15eee285933ec4b48d6a1de9e0c0f6082946f93090bd132 +ARG H2ORESTART_VERSION=v0.7.0 ENV DEBIAN_FRONTEND=noninteractive diff --git a/Dockerfile.env b/Dockerfile.env index cae3fc9..11523f2 100644 --- a/Dockerfile.env +++ b/Dockerfile.env @@ -1,5 +1,9 @@ +# Can be bumped to the latest date in https://hub.docker.com/_/debian/tags?name=bookworm- DEBIAN_IMAGE_DATE=20250113 +# Can be bumped to today's date DEBIAN_ARCHIVE_DATE=20250114 +# Can be bumped to the latest date in https://github.com/google/gvisor/tags GVISOR_ARCHIVE_DATE=20250106 -H2ORESTART_CHECKSUM=8a5be77359695c14faaf33891d3eca6c9d73c1224599aab50a9d2ccc04640580 -H2ORESTART_VERSION=v0.6.8 +# Can be bumped to the latest version and checksum from https://github.com/ebandal/H2Orestart/releases +H2ORESTART_CHECKSUM=7760dc2963332c50d15eee285933ec4b48d6a1de9e0c0f6082946f93090bd132 +H2ORESTART_VERSION=v0.7.0 diff --git a/Dockerfile.in b/Dockerfile.in index 0bf5ee2..758d0b4 100644 --- a/Dockerfile.in +++ b/Dockerfile.in @@ -8,8 +8,8 @@ FROM debian:bookworm-${DEBIAN_IMAGE_DATE}-slim ARG GVISOR_ARCHIVE_DATE={{GVISOR_ARCHIVE_DATE}} ARG DEBIAN_ARCHIVE_DATE={{DEBIAN_ARCHIVE_DATE}} -ARG H2ORESTART_CHECKSUM=8a5be77359695c14faaf33891d3eca6c9d73c1224599aab50a9d2ccc04640580 -ARG H2ORESTART_VERSION=v0.6.8 +ARG H2ORESTART_CHECKSUM={{H2ORESTART_CHECKSUM}} +ARG H2ORESTART_VERSION={{H2ORESTART_VERSION}} ENV DEBIAN_FRONTEND=noninteractive From cbeb103067d831d3bad8854265210d2a653ea8a0 Mon Sep 17 00:00:00 2001 From: Alex Pyrgiotis Date: Tue, 14 Jan 2025 14:53:31 +0200 Subject: [PATCH 20/21] FIXUP: Separate some Dockerfile commands --- Dockerfile | 13 ++++++++----- Dockerfile.in | 13 ++++++++----- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/Dockerfile b/Dockerfile index c334784..1d1fc98 100644 --- a/Dockerfile +++ b/Dockerfile @@ -52,11 +52,14 @@ RUN mkdir /libreoffice_ext && cd libreoffice_ext \ && rm /root/.wget-hsts # Create an unprivileged user both for gVisor and for running Dangerzone. -RUN mkdir -p /opt/dangerzone/dangerzone && \ - touch /opt/dangerzone/dangerzone/__init__.py && \ - addgroup --gid 1000 dangerzone && \ - adduser --uid 1000 --ingroup dangerzone --shell /bin/true \ - --disabled-password --home /home/dangerzone dangerzone +RUN addgroup --gid 1000 dangerzone +RUN adduser --uid 1000 --ingroup dangerzone --shell /bin/true \ + --disabled-password --home /home/dangerzone dangerzone + +# Copy Dangerzone's conversion logic under /opt/dangerzone, and allow Python to +# import it. +RUN mkdir -p /opt/dangerzone/dangerzone +RUN touch /opt/dangerzone/dangerzone/__init__.py COPY conversion/doc_to_pixels.py \ conversion/common.py \ diff --git a/Dockerfile.in b/Dockerfile.in index 758d0b4..58dfe4e 100644 --- a/Dockerfile.in +++ b/Dockerfile.in @@ -52,11 +52,14 @@ RUN mkdir /libreoffice_ext && cd libreoffice_ext \ && rm /root/.wget-hsts # Create an unprivileged user both for gVisor and for running Dangerzone. -RUN mkdir -p /opt/dangerzone/dangerzone && \ - touch /opt/dangerzone/dangerzone/__init__.py && \ - addgroup --gid 1000 dangerzone && \ - adduser --uid 1000 --ingroup dangerzone --shell /bin/true \ - --disabled-password --home /home/dangerzone dangerzone +RUN addgroup --gid 1000 dangerzone +RUN adduser --uid 1000 --ingroup dangerzone --shell /bin/true \ + --disabled-password --home /home/dangerzone dangerzone + +# Copy Dangerzone's conversion logic under /opt/dangerzone, and allow Python to +# import it. +RUN mkdir -p /opt/dangerzone/dangerzone +RUN touch /opt/dangerzone/dangerzone/__init__.py COPY conversion/doc_to_pixels.py \ conversion/common.py \ From f019ce05d6f107102a88dcc5f9a4855169499577 Mon Sep 17 00:00:00 2001 From: Alex Pyrgiotis Date: Tue, 14 Jan 2025 15:04:31 +0200 Subject: [PATCH 21/21] Update RELEASE.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Alexis Métaireau --- RELEASE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE.md b/RELEASE.md index b0a7e8c..c651c4c 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -14,7 +14,7 @@ Here is a list of tasks that should be done before issuing the release: - [ ] Update `share/version.txt` - [ ] Update the "Version" field in `install/linux/dangerzone.spec` - [ ] Bump the Debian version by adding a new changelog entry in `debian/changelog` -- [ ] Bump the dates in the `Dockerfile` +- [ ] Bump the dates and versions in the `Dockerfile` - [ ] Update screenshot in `README.md`, if necessary - [ ] CHANGELOG.md should be updated to include a list of all major changes since the last release - [ ] A draft release should be created. Copy the release notes text from the template at [`docs/templates/release-notes`](https://github.com/freedomofpress/dangerzone/tree/main/docs/templates/)