almet/dangerzone
1
0
Fork 0
mirror of https://github.com/freedomofpress/dangerzone.git synced 2025-04-30 10:42:37 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 7

Add platform

Browse source
This commit is contained in:
Alex Pyrgiotis 2025-02-06 13:37:12 +02:00
parent a6aa66f925
commit 8aaebfb108
No known key found for this signature in database
GPG key ID: B6C15EBA0357C9AA
7 changed files with 874 additions and 96 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.gitignore vendored
View file

@ -146,7 +146,7 @@ tests/test_docs/**/*-safe.pdf
tests/test_docs_large/
install/windows/Dangerzone.wxs
share/container.tar
share/container.tar.gz
share/container.tar.*
share/image-id.txt
container/container-pip-requirements.txt
.doit.db.db

59
Dockerfile
View file

@ -4,7 +4,7 @@
ARG DEBIAN_IMAGE_DATE=20250113
FROM debian:bookworm-${DEBIAN_IMAGE_DATE}-slim as dangerzone-image
FROM debian:bookworm-${DEBIAN_IMAGE_DATE}-slim AS dangerzone-image
ARG GVISOR_ARCHIVE_DATE=20250120
ARG DEBIAN_ARCHIVE_DATE=20250127
@ -22,8 +22,8 @@ RUN \
--mount=type=bind,source=./container_helpers/repro-sources-list.sh,target=/usr/local/bin/repro-sources-list.sh \
--mount=type=bind,source=./container_helpers/gvisor.key,target=/tmp/gvisor.key \
: "Hacky way to set a date for the Debian snapshot repos" && \
touch -d ${DEBIAN_ARCHIVE_DATE} /etc/apt/sources.list.d/debian.sources && \
touch -d ${DEBIAN_ARCHIVE_DATE} /etc/apt/sources.list && \
touch -d ${DEBIAN_ARCHIVE_DATE}Z /etc/apt/sources.list.d/debian.sources && \
touch -d ${DEBIAN_ARCHIVE_DATE}Z /etc/apt/sources.list && \
repro-sources-list.sh && \
: "Setup APT to install gVisor from its separate APT repo" && \
apt-get update && \
@ -52,9 +52,13 @@ RUN mkdir /opt/libreoffice_ext && cd /opt/libreoffice_ext \
&& rm /root/.wget-hsts
# Create an unprivileged user both for gVisor and for running Dangerzone.
# XXX: Make the shadow filed "date of last password change" a constant
# number.
RUN addgroup --gid 1000 dangerzone
RUN adduser --uid 1000 --ingroup dangerzone --shell /bin/true \
--disabled-password --home /home/dangerzone dangerzone
--disabled-password --home /home/dangerzone dangerzone \
&& chage -d 99999 dangerzone \
&& rm /etc/shadow-
# Copy Dangerzone's conversion logic under /opt/dangerzone, and allow Python to
# import it.
@ -165,20 +169,42 @@ RUN mkdir /home/dangerzone/.containers
# The `ln` binary, even if you specify it by its full path, cannot run
# (probably because `ld-linux.so` can't be found). For this reason, we have
# to create the symlinks beforehand, in a previous build stage. Then, in an
# empty contianer image (scratch images), we can copy these symlinks and the
# /usr, and stich everything together.
# empty container image (scratch images), we can copy these symlinks and the
# /usr, and stitch everything together.
###############################################################################
# Create the filesystem hierarchy that will be used to symlink /usr.
RUN mkdir /new_root
RUN mkdir /new_root/root /new_root/run /new_root/tmp
RUN chmod 777 /new_root/tmp
RUN mkdir -p \
/new_root \
/new_root/root \
/new_root/run \
/new_root/tmp \
/new_root/home/dangerzone/dangerzone-image/rootfs
# XXX: Remove /etc/resolv.conf, so that the network configuration of the host
# does not leak.
RUN cp -r /etc /var /new_root/ \
&& rm /new_root/etc/resolv.conf
RUN cp -r /etc /opt /usr /new_root/home/dangerzone/dangerzone-image/rootfs \
&& rm /new_root/home/dangerzone/dangerzone-image/rootfs/etc/resolv.conf
RUN ln -s /home/dangerzone/dangerzone-image/rootfs/usr /new_root/usr
RUN ln -s usr/bin /new_root/bin
RUN ln -s usr/lib /new_root/lib
RUN ln -s usr/lib64 /new_root/lib64
RUN ln -s usr/sbin /new_root/sbin
RUN ln -s usr/bin /new_root/home/dangerzone/dangerzone-image/rootfs/bin
RUN ln -s usr/lib /new_root/home/dangerzone/dangerzone-image/rootfs/lib
RUN ln -s usr/lib64 /new_root/home/dangerzone/dangerzone-image/rootfs/lib64
# Fix permissions in /home/dangerzone, so that our entrypoint script can make
# changes in the following folders.
RUN chown dangerzone:dangerzone \
/new_root/home/dangerzone \
/new_root/home/dangerzone/dangerzone-image/
# Fix permissions in /tmp, so that it can be used by unprivileged users.
RUN chmod 777 /new_root/tmp
## Final image
@ -188,21 +214,6 @@ FROM scratch
# /usr can be a symlink.
COPY --from=dangerzone-image /new_root/ /
# Copy the bare minimum to run Dangerzone in the inner container image.
COPY --from=dangerzone-image /etc/ /home/dangerzone/dangerzone-image/rootfs/etc/
COPY --from=dangerzone-image /opt/ /home/dangerzone/dangerzone-image/rootfs/opt/
COPY --from=dangerzone-image /usr/ /home/dangerzone/dangerzone-image/rootfs/usr/
RUN ln -s usr/bin /home/dangerzone/dangerzone-image/rootfs/bin
RUN ln -s usr/lib /home/dangerzone/dangerzone-image/rootfs/lib
RUN ln -s usr/lib64 /home/dangerzone/dangerzone-image/rootfs/lib64
# Copy the bare minimum to let the security scanner find vulnerabilities.
COPY --from=dangerzone-image /etc/ /etc/
COPY --from=dangerzone-image /var/ /var/
# Allow our entrypoint script to make changes in the following folders.
RUN chown dangerzone:dangerzone /home/dangerzone /home/dangerzone/dangerzone-image/
# Switch to the dangerzone user for the rest of the script.
USER dangerzone

52
Dockerfile.in
View file

@ -4,7 +4,7 @@
ARG DEBIAN_IMAGE_DATE={{DEBIAN_IMAGE_DATE}}
FROM debian:bookworm-${DEBIAN_IMAGE_DATE}-slim as dangerzone-image
FROM debian:bookworm-${DEBIAN_IMAGE_DATE}-slim AS dangerzone-image
ARG GVISOR_ARCHIVE_DATE={{GVISOR_ARCHIVE_DATE}}
ARG DEBIAN_ARCHIVE_DATE={{DEBIAN_ARCHIVE_DATE}}
@ -22,8 +22,8 @@ RUN \
--mount=type=bind,source=./container_helpers/repro-sources-list.sh,target=/usr/local/bin/repro-sources-list.sh \
--mount=type=bind,source=./container_helpers/gvisor.key,target=/tmp/gvisor.key \
: "Hacky way to set a date for the Debian snapshot repos" && \
touch -d ${DEBIAN_ARCHIVE_DATE} /etc/apt/sources.list.d/debian.sources && \
touch -d ${DEBIAN_ARCHIVE_DATE} /etc/apt/sources.list && \
touch -d ${DEBIAN_ARCHIVE_DATE}Z /etc/apt/sources.list.d/debian.sources && \
touch -d ${DEBIAN_ARCHIVE_DATE}Z /etc/apt/sources.list && \
repro-sources-list.sh && \
: "Setup APT to install gVisor from its separate APT repo" && \
apt-get update && \
@ -171,18 +171,36 @@ RUN mkdir /home/dangerzone/.containers
# Create the filesystem hierarchy that will be used to symlink /usr.
RUN mkdir /new_root
RUN mkdir /new_root/root /new_root/run /new_root/tmp
RUN chmod 777 /new_root/tmp
RUN mkdir -p \
/new_root \
/new_root/etc \
/new_root/root \
/new_root/run \
/new_root/tmp \
/new_root/var \
/new_root/home/dangerzone/dangerzone-image/rootfs \
/new_root/home/dangerzone/dangerzone-image/rootfs/etc \
/new_root/home/dangerzone/dangerzone-image/rootfs/opt \
/new_root/home/dangerzone/dangerzone-image/rootfs/usr
RUN ln -s /home/dangerzone/dangerzone-image/rootfs/usr /new_root/usr
RUN ln -s usr/bin /new_root/bin
RUN ln -s usr/lib /new_root/lib
RUN ln -s usr/lib64 /new_root/lib64
RUN ln -s usr/sbin /new_root/sbin
## Final image
# Fix permissions in /home/dangerzone, so that our entrypoint script can make
# changes in the following folders.
RUN chown dangerzone:dangerzone \
/new_root/home/dangerzone \
/new_root/home/dangerzone/dangerzone-image/
# Fix permissions in /tmp, so that it can be used by unprivileged users.
RUN chmod 777 /new_root/tmp
FROM scratch
## Intermediate image
FROM scratch AS intermediate
# Copy the filesystem hierarchy that we created in the previous stage, so that
# /usr can be a symlink.
@ -200,8 +218,22 @@ RUN ln -s usr/lib64 /home/dangerzone/dangerzone-image/rootfs/lib64
COPY --from=dangerzone-image /etc/ /etc/
COPY --from=dangerzone-image /var/ /var/
# Allow our entrypoint script to make changes in the following folders.
RUN chown dangerzone:dangerzone /home/dangerzone /home/dangerzone/dangerzone-image/
RUN chmod g-s \
/etc/ \
/var/ \
/root/ \
/run/ \
/home/dangerzone/dangerzone-image/rootfs/etc/ \
/home/dangerzone/dangerzone-image/rootfs/opt/ \
/home/dangerzone/dangerzone-image/rootfs/usr/
### Final image
#FROM scratch
## Copy the filesystem hierarchy that we created in the previous stage, so that
## /usr can be a symlink.
#COPY --from=intermediate / /
# Switch to the dangerzone user for the rest of the script.
USER dangerzone

8
dev_scripts/env.py
View file

@ -160,8 +160,8 @@ DOCKERFILE_BUILD_DEV = r"""FROM {distro}:{version}
# Create a non-root user to run Dangerzone
RUN adduser user
# See https://github.com/freedomofpress/dangerzone/issues/286#issuecomment-1347149783
RUN echo user:2000:2000 > /etc/subuid
RUN echo user:2000:2000 > /etc/subgid
RUN echo user:2000:250000 > /etc/subuid
RUN echo user:2000:250000 > /etc/subgid
# XXX: We need the empty source folder, so that we can trick Poetry to create a
# link to the project's path. This way, we should be able to do `import
@ -456,7 +456,7 @@ class Env:
"--uidmap",
"0:1:1000",
"--uidmap",
"1001:1001:64536",
"1001:1001:251999",
]
gidmaps = [
"--gidmap",
@ -464,7 +464,7 @@ class Env:
"--gidmap",
"0:1:1000",
"--gidmap",
"1001:1001:64536",
"1001:1001:251999",
]
run_cmd += uidmaps + gidmaps

665
dev_scripts/repro-build Executable file
View file

@ -0,0 +1,665 @@
#!/usr/bin/env python3
import argparse
import datetime
import hashlib
import json
import logging
import os
import pprint
import shlex
import shutil
import subprocess
import sys
import tarfile
from pathlib import Path
logger = logging.getLogger(__name__)
MEDIA_TYPE_INDEX_V1_JSON = "application/vnd.oci.image.index.v1+json"
MEDIA_TYPE_MANIFEST_V1_JSON = "application/vnd.oci.image.manifest.v1+json"
ENV_RUNTIME = "REPRO_RUNTIME"
ENV_DATETIME = "REPRO_DATETIME"
ENV_SDE = "REPRO_SOURCE_DATE_EPOCH"
ENV_CACHE = "REPRO_CACHE"
ENV_BUILDKIT = "REPRO_BUILDKIT_IMAGE"
ENV_ROOTLESS = "REPRO_ROOTLESS"
DEFAULT_BUILDKIT_IMAGE = "moby/buildkit:v0.19.0@sha256:14aa1b4dd92ea0a4cd03a54d0c6079046ea98cd0c0ae6176bdd7036ba370cbbe"
DEFAULT_BUILDKIT_IMAGE_ROOTLESS = "moby/buildkit:v0.19.0-rootless@sha256:e901cffdad753892a7c3afb8b9972549fca02c73888cf340c91ed801fdd96d71"
MSG_BUILD_CTX = """Build environment:
- Container runtime: {runtime}
- BuildKit image: {buildkit_image}
- Rootless support: {rootless}
- Caching enabled: {use_cache}
- Build context: {context}
- Dockerfile: {dockerfile}
- Output: {output}
Build parameters:
- SOURCE_DATE_EPOCH: {sde}
- Build args: {build_args}
- Tag: {tag}
- Platform: {platform}
Podman-only arguments:
- BuildKit arguments: {buildkit_args}
Docker-only arguments:
- Docker Buildx arguments: {buildx_args}
"""
def pretty_error(obj: dict, msg: str):
raise Exception(f"{msg}\n{pprint.pprint(obj)}")
def get_key(obj: dict, key: str) -> object:
if key not in obj:
pretty_error(f"Could not find key '{key}' in the dictionary:", obj)
return obj[key]
def run(cmd, dry=False, check=True):
action = "Would have run" if dry else "Running"
logger.debug(f"{action}: {shlex.join(cmd)}")
if not dry:
subprocess.run(cmd, check=check)
def snip_contents(contents: str, num: int) -> str:
contents = contents.replace("\n", "")
if len(contents) > num:
return (
contents[:num]
+ f" [... {len(contents) - num} characters omitted."
+ " Pass --show-contents to print them in their entirety]"
)
return contents
def detect_container_runtime() -> str:
"""Auto-detect the installed container runtime in the system."""
if shutil.which("docker"):
return "docker"
elif shutil.which("podman"):
return "podman"
else:
return None
def parse_runtime(args) -> str:
if args.runtime is not None:
return args.runtime
runtime = os.environ.get(ENV_RUNTIME)
if runtime is None:
raise RuntimeError("No container runtime detected in your system")
if runtime not in ("docker", "podman"):
raise RuntimeError(
"Only 'docker' or 'podman' container runtimes"
" are currently supported by this script"
)
def parse_use_cache(args) -> bool:
if args.no_cache:
return False
return bool(int(os.environ.get(ENV_CACHE, "1")))
def parse_rootless(args, runtime: str) -> bool:
rootless = args.rootless or bool(int(os.environ.get(ENV_ROOTLESS, "0")))
if runtime != "podman" and rootless:
raise RuntimeError("Rootless mode is only supported with Podman runtime")
return rootless
def parse_sde(args) -> str:
sde = os.environ.get(ENV_SDE, args.source_date_epoch)
dt = os.environ.get(ENV_DATETIME, args.datetime)
if (sde is not None and dt is not None) or (sde is None and dt is None):
raise RuntimeError("You need to pass either a source date epoch or a datetime")
if sde is not None:
return str(sde)
if dt is not None:
d = datetime.datetime.fromisoformat(dt)
# If the datetime is naive, assume its timezone is UTC. The check is
# taken from:
# https://docs.python.org/3/library/datetime.html#determining-if-an-object-is-aware-or-naive
if d.tzinfo is None or d.tzinfo.utcoffset(d) is None:
d = d.replace(tzinfo=datetime.timezone.utc)
return int(d.timestamp())
def parse_buildkit_image(args, rootless: bool, runtime: str) -> str:
default = DEFAULT_BUILDKIT_IMAGE_ROOTLESS if rootless else DEFAULT_BUILDKIT_IMAGE
img = args.buildkit_image or os.environ.get(ENV_BUILDKIT, default)
if runtime == "podman" and not img.startswith("docker.io/"):
img = "docker.io/" + img
return img
def parse_build_args(args) -> str:
return args.build_arg or []
def parse_buildkit_args(args, runtime: str) -> str:
if not args.buildkit_args:
return []
if runtime != "podman":
raise RuntimeError("Cannot specify BuildKit arguments using the Podman runtime")
return shlex.split(args.buildkit_args)
def parse_buildx_args(args, runtime: str) -> str:
if not args.buildx_args:
return []
if runtime != "docker":
raise RuntimeError(
"Cannot specify Docker Buildx arguments using the Podman runtime"
)
return shlex.split(args.buildx_args)
def parse_image_digest(args) -> str | None:
if not args.expected_image_digest:
return None
parsed = args.expected_image_digest.split(":", 1)
if len(parsed) == 1:
return parsed[0]
else:
return parsed[1]
def parse_path(path: str | None) -> str | None:
return path and str(Path(path).absolute())
##########################
# OCI parsing logic
#
# Compatible with:
# * https://github.com/opencontainers/image-spec/blob/main/image-layout.md
def oci_print_info(parsed: dict, full: bool) -> None:
print(f"The OCI tarball contains an index and {len(parsed) - 1} manifest(s):")
print()
print(f"Image digest: {parsed[1]['digest']}")
for i, info in enumerate(parsed):
print()
if i == 0:
print(f"Index ({info['path']}):")
else:
print(f"Manifest {i} ({info['path']}):")
print(f" Digest: {info['digest']}")
print(f" Media type: {info['media_type']}")
print(f" Platform: {info['platform'] or '-'}")
contents = info["contents"] if full else snip_contents(info["contents"], 600)
print(f" Contents: {contents}")
print()
def oci_normalize_path(path):
if path.startswith("sha256:"):
hash_algo, checksum = path.split(":")
path = f"blobs/{hash_algo}/{checksum}"
return path
def oci_get_file_from_tarball(tar: tarfile.TarFile, path: str) -> dict:
return
def oci_parse_manifest(tar: tarfile.TarFile, path: str, platform: dict | None) -> dict:
"""Parse manifest information in JSON format.
Interestingly, the platform info for a manifest is not included in the
manifest itself, but in the descriptor that points to it. So, we have to
carry it from the previous manifest and include in the info here.
"""
path = oci_normalize_path(path)
contents = tar.extractfile(path).read().decode()
digest = "sha256:" + hashlib.sha256(contents.encode()).hexdigest()
contents_dict = json.loads(contents)
media_type = get_key(contents_dict, "mediaType")
manifests = contents_dict.get("manifests", [])
if platform:
os = get_key(platform, "os")
arch = get_key(platform, "architecture")
platform = f"{os}/{arch}"
return {
"path": path,
"contents": contents,
"digest": digest,
"media_type": media_type,
"platform": platform,
"manifests": manifests,
}
def oci_parse_manifests_dfs(
tar: tarfile.TarFile, path: str, parsed: list, platform: dict | None = None
) -> None:
info = oci_parse_manifest(tar, path, platform)
parsed.append(info)
for m in info["manifests"]:
oci_parse_manifests_dfs(tar, m["digest"], parsed, m.get("platform"))
def oci_parse_tarball(path: Path) -> dict:
parsed = []
with tarfile.TarFile.open(path) as tar:
oci_parse_manifests_dfs(tar, "index.json", parsed)
return parsed
##########################
# Image building logic
def podman_build(
context: str,
dockerfile: str | None,
tag: str | None,
buildkit_image: str,
sde: int,
rootless: bool,
use_cache: bool,
output: Path,
build_args: list,
platform: str,
buildkit_args: list,
dry: bool,
):
rootless_args = []
rootful_args = []
if rootless:
rootless_args = [
"--userns",
"keep-id:uid=1000,gid=1000",
"--security-opt",
"seccomp=unconfined",
"--security-opt",
"apparmor=unconfined",
"-e",
"BUILDKITD_FLAGS=--oci-worker-no-process-sandbox",
]
else:
rootful_args = ["--privileged"]
dockerfile_args_podman = []
dockerfile_args_buildkit = []
if dockerfile:
dockerfile_args_podman = ["-v", f"{dockerfile}:/tmp/Dockerfile"]
dockerfile_args_buildkit = ["--local", "dockerfile=/tmp"]
else:
dockerfile_args_buildkit = ["--local", "dockerfile=/tmp/work"]
tag_args = f",name={tag}" if tag else ""
cache_args = []
if use_cache:
cache_args = [
"--export-cache",
"type=local,mode=max,dest=/tmp/cache",
"--import-cache",
"type=local,src=/tmp/cache",
]
_build_args = []
for arg in build_args:
_build_args.append("--opt")
_build_args.append(f"build-arg:{arg}")
platform_args = ["--opt", f"platform={platform}"] if platform else []
cmd = [
"podman",
"run",
"-it",
"--rm",
"-v",
"buildkit_cache:/tmp/cache",
"-v",
f"{output.parent}:/tmp/image",
"-v",
f"{context}:/tmp/work",
"--entrypoint",
"buildctl-daemonless.sh",
*rootless_args,
*rootful_args,
*dockerfile_args_podman,
buildkit_image,
"build",
"--frontend",
"dockerfile.v0",
"--local",
"context=/tmp/work",
"--opt",
f"build-arg:SOURCE_DATE_EPOCH={sde}",
*_build_args,
"--output",
f"type=docker,dest=/tmp/image/{output.name},rewrite-timestamp=true{tag_args}",
*cache_args,
*dockerfile_args_buildkit,
*platform_args,
*buildkit_args,
]
run(cmd, dry)
def docker_build(
context: str,
dockerfile: str | None,
tag: str | None,
buildkit_image: str,
sde: int,
use_cache: bool,
output: Path,
build_args: list,
platform: str,
buildx_args: list,
dry: bool,
):
builder_id = hashlib.sha256(buildkit_image.encode()).hexdigest()
builder_name = f"repro-build-{builder_id}"
tag_args = ["-t", tag] if tag else []
cache_args = [] if use_cache else ["--no-cache", "--pull"]
cmd = [
"docker",
"buildx",
"create",
"--name",
builder_name,
"--driver-opt",
f"image={buildkit_image}",
]
run(cmd, dry, check=False)
dockerfile_args = ["-f", dockerfile] if dockerfile else []
_build_args = []
for arg in build_args:
_build_args.append("--build-arg")
_build_args.append(arg)
platform_args = ["--platform", platform] if platform else []
cmd = [
"docker",
"buildx",
"--builder",
builder_name,
"build",
"--build-arg",
f"SOURCE_DATE_EPOCH={sde}",
*_build_args,
"--provenance",
"false",
"--output",
f"type=docker,dest={output},rewrite-timestamp=true",
*cache_args,
*tag_args,
*dockerfile_args,
*platform_args,
*buildx_args,
context,
]
run(cmd, dry)
##########################
# Command logic
def build(args):
runtime = parse_runtime(args)
use_cache = parse_use_cache(args)
sde = parse_sde(args)
rootless = parse_rootless(args, runtime)
buildkit_image = parse_buildkit_image(args, rootless, runtime)
build_args = parse_build_args(args)
platform = args.platform
buildkit_args = parse_buildkit_args(args, runtime)
buildx_args = parse_buildx_args(args, runtime)
tag = args.tag
dockerfile = parse_path(args.file)
output = Path(parse_path(args.output))
dry = args.dry
context = parse_path(args.context)
logger.info(
MSG_BUILD_CTX.format(
runtime=runtime,
buildkit_image=buildkit_image,
sde=sde,
rootless=rootless,
use_cache=use_cache,
context=context,
dockerfile=dockerfile or "(not provided)",
tag=tag or "(not provided)",
output=output,
build_args=",".join(build_args) or "(not provided)",
platform=platform or "(default)",
buildkit_args=" ".join(buildkit_args) or "(not provided)",
buildx_args=" ".join(buildx_args) or "(not provided)",
)
)
try:
if runtime == "docker":
docker_build(
context,
dockerfile,
tag,
buildkit_image,
sde,
use_cache,
output,
build_args,
platform,
buildx_args,
dry,
)
else:
podman_build(
context,
dockerfile,
tag,
buildkit_image,
sde,
rootless,
use_cache,
output,
build_args,
platform,
buildkit_args,
dry,
)
except subprocess.CalledProcessError as e:
logger.error(f"Failed with {e.returncode}")
sys.exit(e.returncode)
def analyze(args) -> None:
expected_image_digest = parse_image_digest(args)
tarball_path = Path(args.tarball)
parsed = oci_parse_tarball(tarball_path)
oci_print_info(parsed, args.show_contents)
if expected_image_digest:
cur_digest = parsed[1]["digest"].split(":")[1]
if cur_digest != expected_image_digest:
raise Exception(
f"The image does not have the expected digest: {cur_digest} != {expected_image_digest}"
)
print(f"✅ Image digest matches {expected_image_digest}")
def define_build_cmd_args(parser: argparse.ArgumentParser) -> None:
parser.add_argument(
"--runtime",
choices=["docker", "podman"],
default=detect_container_runtime(),
help="The container runtime for building the image (default: %(default)s)",
)
parser.add_argument(
"--datetime",
metavar="YYYY-MM-DD",
default=None,
help=(
"Provide a date and (optionally) a time in ISO format, which will"
" be used as the timestamp of the image layers"
),
)
parser.add_argument(
"--buildkit-image",
metavar="NAME:TAG@DIGEST",
default=None,
help=(
"The BuildKit container image which will be used for building the"
" reproducible container image. Make sure to pass the '-rootless'"
" variant if you are using rootless Podman"
" (default: docker.io/moby/buildkit:v0.19.0)"
),
)
parser.add_argument(
"--source-date-epoch",
"--sde",
metavar="SECONDS",
type=int,
default=None,
help="Provide a Unix timestamp for the image layers",
)
parser.add_argument(
"--no-cache",
default=False,
action="store_true",
help="Do not use existing cached images for the container build. Build from the start with a new set of cached layers.",
)
parser.add_argument(
"--rootless",
default=False,
action="store_true",
help="Run BuildKit in rootless mode (Podman only)",
)
parser.add_argument(
"-f",
"--file",
metavar="FILE",
default=None,
help="Pathname of a Dockerfile",
)
parser.add_argument(
"-o",
"--output",
metavar="FILE",
default=Path.cwd() / "image.tar",
help="Path to save OCI tarball (default: %(default)s)",
)
parser.add_argument(
"-t",
"--tag",
metavar="TAG",
default=None,
help="Tag the built image with the name %(metavar)s",
)
parser.add_argument(
"--build-arg",
metavar="ARG=VALUE",
action="append",
default=None,
help="Set build-time variables",
)
parser.add_argument(
"--platform",
metavar="PLAT1,PLAT2",
default=None,
help="Set platform for the image",
)
parser.add_argument(
"--buildkit-args",
metavar="'ARG1 ARG2'",
default=None,
help="Extra arguments for BuildKit (Podman only)",
)
parser.add_argument(
"--buildx-args",
metavar="'ARG1 ARG2'",
default=None,
help="Extra arguments for Docker Buildx (Docker only)",
)
parser.add_argument(
"--dry",
default=False,
action="store_true",
help="Do not run any commands, just print what would happen",
)
parser.add_argument(
"context",
metavar="CONTEXT",
help="Path to the build context",
)
def parse_args() -> dict:
parser = argparse.ArgumentParser()
subparsers = parser.add_subparsers(dest="command", help="Available commands")
build_parser = subparsers.add_parser("build", help="Perform a build operation")
build_parser.set_defaults(func=build)
define_build_cmd_args(build_parser)
analyze_parser = subparsers.add_parser("analyze", help="Analyze an OCI tarball")
analyze_parser.set_defaults(func=analyze)
analyze_parser.add_argument(
"tarball",
metavar="FILE",
help="Path to OCI image in .tar format",
)
analyze_parser.add_argument(
"--expected-image-digest",
metavar="DIGEST",
default=None,
help="The expected digest for the provided image",
)
analyze_parser.add_argument(
"--show-contents",
default=False,
action="store_true",
help="Show full file contents",
)
return parser.parse_args()
def main() -> None:
logging.basicConfig(
level=logging.DEBUG,
format="%(asctime)s - %(levelname)s - %(message)s",
datefmt="%Y-%m-%d %H:%M:%S",
)
args = parse_args()
if not hasattr(args, "func"):
args.func = build
args.func(args)
if __name__ == "__main__":
sys.exit(main())

71
dev_scripts/reproduce-image.py
View file

@ -18,6 +18,11 @@ DIFFOCI_PATH = (
)
IMAGE_NAME = "dangerzone.rocks/dangerzone"
if platform.system() in ["Darwin", "Windows"]:
CONTAINER_RUNTIME = "docker"
elif platform.system() == "Linux":
CONTAINER_RUNTIME = "podman"
def run(*args):
"""Simple function that runs a command, validates it, and returns the output"""
@ -80,9 +85,10 @@ def diffoci_download():
DIFFOCI_PATH.chmod(DIFFOCI_PATH.stat().st_mode | stat.S_IEXEC)
def diffoci_diff(source, local_target):
def diffoci_diff(runtime, source, local_target, platform=None):
"""Diff the source image against the recently built target image using diffoci."""
target = f"podman://{local_target}"
target = f"{runtime}://{local_target}"
platform_args = [] if not platform else ["--platform", platform]
try:
return run(
str(DIFFOCI_PATH),
@ -91,6 +97,7 @@ def diffoci_diff(source, local_target):
target,
"--semantic",
"--verbose",
*platform_args,
)
except subprocess.CalledProcessError as e:
error = e.stdout.decode()
@ -99,14 +106,29 @@ def diffoci_diff(source, local_target):
)
def build_image(tag, use_cache=False):
def build_image(
tag,
use_cache=False,
platform=None,
runtime=None,
date=None,
buildx=False
):
"""Build the Dangerzone container image with a special tag."""
platform_args = [] if not platform else ["--platform", platform]
runtime_args = [] if not runtime else ["--runtime", runtime]
date_args = [] if not date else ["--debian-archive-date", date]
buildx_args = [] if not buildx else ["--buildx"]
run(
"python3",
"./install/common/build-image.py",
"--no-save",
"--use-cache",
str(use_cache),
*date_args,
*platform_args,
*runtime_args,
*buildx_args,
"--tag",
tag,
)
@ -116,12 +138,28 @@ def parse_args():
image_tag = git_determine_tag()
# TODO: Remove the local "podman://" prefix once we have started pushing images to a
# remote.
default_image_name = f"podman://{IMAGE_NAME}:{image_tag}"
default_image_name = f"{IMAGE_NAME}:{image_tag}"
parser = argparse.ArgumentParser(
prog=sys.argv[0],
description="Dev script for verifying container image reproducibility",
)
parser.add_argument(
"--buildx",
action="store_true",
help="Use the buildx platform of Docker or Podman",
)
parser.add_argument(
"--platform",
default=None,
help=f"The platform for building the image (default: current platform)",
)
parser.add_argument(
"--runtime",
choices=["docker", "podman"],
default=CONTAINER_RUNTIME,
help=f"The container runtime for building the image (default: {CONTAINER_RUNTIME})",
)
parser.add_argument(
"--source",
default=default_image_name,
@ -137,6 +175,17 @@ def parse_args():
action="store_true",
help="Whether to reuse the build cache (off by default for better reproducibility)",
)
parser.add_argument(
"--skip-check-commit",
default=False,
action="store_true",
help="Skip checking if the source image tag contains the current Git commit",
)
parser.add_argument(
"--debian-archive-date",
default=None,
help="Use a specific Debian snapshot archive, by its date",
)
return parser.parse_args()
@ -148,8 +197,9 @@ def main():
)
args = parse_args()
logger.info(f"Ensuring that current Git commit matches image '{args.source}'")
commit = git_commit_get()
if not args.skip_check_commit:
logger.info(f"Ensuring that current Git commit matches image '{args.source}'")
git_verify(commit, args.source)
if not diffoci_is_installed():
@ -159,14 +209,21 @@ def main():
tag = f"reproduce-{commit}"
target = f"{IMAGE_NAME}:{tag}"
logger.info(f"Building container image and tagging it as '{target}'")
build_image(tag, args.use_cache)
build_image(
tag,
args.use_cache,
args.platform,
args.runtime,
args.debian_archive_date,
args.buildx,
)
logger.info(
f"Ensuring that source image '{args.source}' is semantically identical with"
f" built image '{target}'"
)
try:
diffoci_diff(args.source, target)
diffoci_diff(args.runtime, args.source, target, args.platform)
except subprocess.CalledProcessError as e:
raise RuntimeError(
f"Could not reproduce image {args.source} for commit {commit}"

107
install/common/build-image.py
View file

@ -13,8 +13,6 @@ if platform.system() in ["Darwin", "Windows"]:
elif platform.system() == "Linux":
CONTAINER_RUNTIME = "podman"
ARCH = platform.machine()
def str2bool(v):
if isinstance(v, bool):
@ -50,6 +48,16 @@ def determine_git_tag():
)
def determine_debian_archive_date():
"""Get the date of the Debian archive from Dockerfile.env."""
for env in Path("Dockerfile.env").read_text().split("\n"):
if env.startswith("DEBIAN_ARCHIVE_DATE"):
return env.split("=")[1]
raise Exception(
"Could not find 'DEBIAN_ARCHIVE_DATE' build argument in Dockerfile.env"
)
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
@ -59,17 +67,23 @@ def main():
help=f"The container runtime for building the image (default: {CONTAINER_RUNTIME})",
)
parser.add_argument(
"--no-save",
action="store_true",
help="Do not save the container image as a tarball in share/container.tar.gz",
"--platform",
default=None,
help=f"The platform for building the image (default: current platform)",
)
parser.add_argument(
"--compress-level",
type=int,
choices=range(0, 10),
default=9,
help="The Gzip compression level, from 0 (lowest) to 9 (highest, default)",
"--output",
"-o",
default=str(Path("share") / "container.tar"),
help="Path to store the container image",
)
# parser.add_argument(
# "--compress-level",
# type=int,
# choices=range(0, 10),
# default=9,
# help="The Gzip compression level, from 0 (lowest) to 9 (highest, default)",
# )
parser.add_argument(
"--use-cache",
type=str2bool,
@ -83,63 +97,62 @@ def main():
default=None,
help="Provide a custom tag for the image (for development only)",
)
parser.add_argument(
"--debian-archive-date",
"-d",
default=determine_debian_archive_date(),
help="Use a specific Debian snapshot archive, by its date (default %(default)s)",
)
parser.add_argument(
"--dry",
default=False,
action="store_true",
help="Do not run any commands, just print what would happen",
)
args = parser.parse_args()
tarball_path = Path("share") / "container.tar.gz"
image_id_path = Path("share") / "image-id.txt"
print(f"Building for architecture '{ARCH}'")
tag = args.tag or determine_git_tag()
image_name_tagged = IMAGE_NAME + ":" + tag
tag = args.tag or f"{args.debian_archive_date}-{determine_git_tag()}"
image_name_tagged = f"{IMAGE_NAME}:{tag}"
print(f"Will tag the container image as '{image_name_tagged}'")
image_id_path = Path("share") / "image-id.txt"
if not args.dry:
with open(image_id_path, "w") as f:
f.write(tag)
# Build the container image, and tag it with the calculated tag
print("Building container image")
cache_args = [] if args.use_cache else ["--no-cache"]
platform_args = [] if not args.platform else ["--platform", args.platform]
rootless_args = [] if args.runtime == "docker" else ["--rootless"]
rootless_args = []
dry_args = [] if not args.dry else ["--dry"]
subprocess.run(
[
args.runtime,
"./dev_scripts/repro-build",
"build",
BUILD_CONTEXT,
"--runtime",
args.runtime,
"--build-arg",
f"DEBIAN_ARCHIVE_DATE={args.debian_archive_date}",
"--datetime",
args.debian_archive_date,
*dry_args,
*cache_args,
"-f",
"Dockerfile",
*platform_args,
*rootless_args,
"--tag",
image_name_tagged,
"--output",
args.output,
"-f",
"Dockerfile",
BUILD_CONTEXT,
],
check=True,
)
if not args.no_save:
print("Saving container image")
cmd = subprocess.Popen(
[
CONTAINER_RUNTIME,
"save",
image_name_tagged,
],
stdout=subprocess.PIPE,
)
print("Compressing container image")
chunk_size = 4 << 20
with gzip.open(
tarball_path,
"wb",
compresslevel=args.compress_level,
) as gzip_f:
while True:
chunk = cmd.stdout.read(chunk_size)
if len(chunk) > 0:
gzip_f.write(chunk)
else:
break
cmd.wait(5)
if __name__ == "__main__":
sys.exit(main())