mirror of
https://github.com/freedomofpress/dangerzone.git
synced 2025-04-28 18:02:38 +02:00
WIP: Overhaul build-image and repro-build
Some checks are pending
Build dev environments / Build dev-env (debian-bookworm) (push) Waiting to run
Build dev environments / Build dev-env (debian-bullseye) (push) Waiting to run
Build dev environments / Build dev-env (debian-trixie) (push) Waiting to run
Build dev environments / Build dev-env (fedora-40) (push) Waiting to run
Build dev environments / Build dev-env (fedora-41) (push) Waiting to run
Build dev environments / Build dev-env (ubuntu-20.04) (push) Waiting to run
Build dev environments / Build dev-env (ubuntu-22.04) (push) Waiting to run
Build dev environments / Build dev-env (ubuntu-24.04) (push) Waiting to run
Build dev environments / Build dev-env (ubuntu-24.10) (push) Waiting to run
Build dev environments / build-container-image (push) Waiting to run
Tests / run-lint (push) Waiting to run
Tests / build-container-image (push) Waiting to run
Tests / Download and cache Tesseract data (push) Waiting to run
Tests / windows (push) Blocked by required conditions
Tests / macOS (arch64) (push) Blocked by required conditions
Tests / macOS (x86_64) (push) Blocked by required conditions
Tests / build-deb (debian bookworm) (push) Blocked by required conditions
Tests / build-deb (debian bullseye) (push) Blocked by required conditions
Tests / build-deb (debian trixie) (push) Blocked by required conditions
Tests / build-deb (ubuntu 20.04) (push) Blocked by required conditions
Tests / build-deb (ubuntu 22.04) (push) Blocked by required conditions
Tests / build-deb (ubuntu 24.04) (push) Blocked by required conditions
Tests / build-deb (ubuntu 24.10) (push) Blocked by required conditions
Tests / install-deb (debian bookworm) (push) Blocked by required conditions
Tests / install-deb (debian bullseye) (push) Blocked by required conditions
Tests / install-deb (debian trixie) (push) Blocked by required conditions
Tests / install-deb (ubuntu 20.04) (push) Blocked by required conditions
Tests / install-deb (ubuntu 22.04) (push) Blocked by required conditions
Tests / install-deb (ubuntu 24.04) (push) Blocked by required conditions
Tests / install-deb (ubuntu 24.10) (push) Blocked by required conditions
Tests / build-install-rpm (fedora 40) (push) Blocked by required conditions
Tests / build-install-rpm (fedora 41) (push) Blocked by required conditions
Tests / run tests (debian bookworm) (push) Blocked by required conditions
Tests / run tests (debian bullseye) (push) Blocked by required conditions
Tests / run tests (debian trixie) (push) Blocked by required conditions
Tests / run tests (fedora 40) (push) Blocked by required conditions
Tests / run tests (fedora 41) (push) Blocked by required conditions
Tests / run tests (ubuntu 20.04) (push) Blocked by required conditions
Tests / run tests (ubuntu 22.04) (push) Blocked by required conditions
Tests / run tests (ubuntu 24.04) (push) Blocked by required conditions
Tests / run tests (ubuntu 24.10) (push) Blocked by required conditions
Tests / check-reproducibility (push) Waiting to run
Release multi-arch container image / build (linux/amd64) (push) Waiting to run
Release multi-arch container image / build (linux/arm64) (push) Waiting to run
Release multi-arch container image / merge (push) Blocked by required conditions
Release multi-arch container image / provenance (push) Blocked by required conditions
Some checks are pending
Build dev environments / Build dev-env (debian-bookworm) (push) Waiting to run
Build dev environments / Build dev-env (debian-bullseye) (push) Waiting to run
Build dev environments / Build dev-env (debian-trixie) (push) Waiting to run
Build dev environments / Build dev-env (fedora-40) (push) Waiting to run
Build dev environments / Build dev-env (fedora-41) (push) Waiting to run
Build dev environments / Build dev-env (ubuntu-20.04) (push) Waiting to run
Build dev environments / Build dev-env (ubuntu-22.04) (push) Waiting to run
Build dev environments / Build dev-env (ubuntu-24.04) (push) Waiting to run
Build dev environments / Build dev-env (ubuntu-24.10) (push) Waiting to run
Build dev environments / build-container-image (push) Waiting to run
Tests / run-lint (push) Waiting to run
Tests / build-container-image (push) Waiting to run
Tests / Download and cache Tesseract data (push) Waiting to run
Tests / windows (push) Blocked by required conditions
Tests / macOS (arch64) (push) Blocked by required conditions
Tests / macOS (x86_64) (push) Blocked by required conditions
Tests / build-deb (debian bookworm) (push) Blocked by required conditions
Tests / build-deb (debian bullseye) (push) Blocked by required conditions
Tests / build-deb (debian trixie) (push) Blocked by required conditions
Tests / build-deb (ubuntu 20.04) (push) Blocked by required conditions
Tests / build-deb (ubuntu 22.04) (push) Blocked by required conditions
Tests / build-deb (ubuntu 24.04) (push) Blocked by required conditions
Tests / build-deb (ubuntu 24.10) (push) Blocked by required conditions
Tests / install-deb (debian bookworm) (push) Blocked by required conditions
Tests / install-deb (debian bullseye) (push) Blocked by required conditions
Tests / install-deb (debian trixie) (push) Blocked by required conditions
Tests / install-deb (ubuntu 20.04) (push) Blocked by required conditions
Tests / install-deb (ubuntu 22.04) (push) Blocked by required conditions
Tests / install-deb (ubuntu 24.04) (push) Blocked by required conditions
Tests / install-deb (ubuntu 24.10) (push) Blocked by required conditions
Tests / build-install-rpm (fedora 40) (push) Blocked by required conditions
Tests / build-install-rpm (fedora 41) (push) Blocked by required conditions
Tests / run tests (debian bookworm) (push) Blocked by required conditions
Tests / run tests (debian bullseye) (push) Blocked by required conditions
Tests / run tests (debian trixie) (push) Blocked by required conditions
Tests / run tests (fedora 40) (push) Blocked by required conditions
Tests / run tests (fedora 41) (push) Blocked by required conditions
Tests / run tests (ubuntu 20.04) (push) Blocked by required conditions
Tests / run tests (ubuntu 22.04) (push) Blocked by required conditions
Tests / run tests (ubuntu 24.04) (push) Blocked by required conditions
Tests / run tests (ubuntu 24.10) (push) Blocked by required conditions
Tests / check-reproducibility (push) Waiting to run
Release multi-arch container image / build (linux/amd64) (push) Waiting to run
Release multi-arch container image / build (linux/arm64) (push) Waiting to run
Release multi-arch container image / merge (push) Blocked by required conditions
Release multi-arch container image / provenance (push) Blocked by required conditions
This commit is contained in:
parent
79b80f0599
commit
6c96d98c44
4 changed files with 722 additions and 78 deletions
|
@ -7,7 +7,7 @@
|
|||
|
||||
ARG DEBIAN_IMAGE_DATE=20250113
|
||||
|
||||
FROM debian:bookworm-${DEBIAN_IMAGE_DATE}-slim as dangerzone-image
|
||||
FROM debian:bookworm-${DEBIAN_IMAGE_DATE}-slim AS dangerzone-image
|
||||
|
||||
ARG GVISOR_ARCHIVE_DATE=20250120
|
||||
ARG DEBIAN_ARCHIVE_DATE=20250127
|
||||
|
@ -25,8 +25,8 @@ RUN \
|
|||
--mount=type=bind,source=./container_helpers/repro-sources-list.sh,target=/usr/local/bin/repro-sources-list.sh \
|
||||
--mount=type=bind,source=./container_helpers/gvisor.key,target=/tmp/gvisor.key \
|
||||
: "Hacky way to set a date for the Debian snapshot repos" && \
|
||||
touch -d ${DEBIAN_ARCHIVE_DATE} /etc/apt/sources.list.d/debian.sources && \
|
||||
touch -d ${DEBIAN_ARCHIVE_DATE} /etc/apt/sources.list && \
|
||||
touch -d ${DEBIAN_ARCHIVE_DATE}Z /etc/apt/sources.list.d/debian.sources && \
|
||||
touch -d ${DEBIAN_ARCHIVE_DATE}Z /etc/apt/sources.list && \
|
||||
repro-sources-list.sh && \
|
||||
: "Setup APT to install gVisor from its separate APT repo" && \
|
||||
apt-get update && \
|
||||
|
|
|
@ -4,7 +4,7 @@
|
|||
|
||||
ARG DEBIAN_IMAGE_DATE={{DEBIAN_IMAGE_DATE}}
|
||||
|
||||
FROM debian:bookworm-${DEBIAN_IMAGE_DATE}-slim as dangerzone-image
|
||||
FROM debian:bookworm-${DEBIAN_IMAGE_DATE}-slim AS dangerzone-image
|
||||
|
||||
ARG GVISOR_ARCHIVE_DATE={{GVISOR_ARCHIVE_DATE}}
|
||||
ARG DEBIAN_ARCHIVE_DATE={{DEBIAN_ARCHIVE_DATE}}
|
||||
|
@ -22,8 +22,8 @@ RUN \
|
|||
--mount=type=bind,source=./container_helpers/repro-sources-list.sh,target=/usr/local/bin/repro-sources-list.sh \
|
||||
--mount=type=bind,source=./container_helpers/gvisor.key,target=/tmp/gvisor.key \
|
||||
: "Hacky way to set a date for the Debian snapshot repos" && \
|
||||
touch -d ${DEBIAN_ARCHIVE_DATE} /etc/apt/sources.list.d/debian.sources && \
|
||||
touch -d ${DEBIAN_ARCHIVE_DATE} /etc/apt/sources.list && \
|
||||
touch -d ${DEBIAN_ARCHIVE_DATE}Z /etc/apt/sources.list.d/debian.sources && \
|
||||
touch -d ${DEBIAN_ARCHIVE_DATE}Z /etc/apt/sources.list && \
|
||||
repro-sources-list.sh && \
|
||||
: "Setup APT to install gVisor from its separate APT repo" && \
|
||||
apt-get update && \
|
||||
|
|
660
dev_scripts/repro-build
Executable file
660
dev_scripts/repro-build
Executable file
|
@ -0,0 +1,660 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
###################
|
||||
# Copied from:
|
||||
# https://github.com/freedomofpress/repro-build/blob/8f85db91a5595bf29d9ba22f6021aca77c1037a8/repro-build
|
||||
|
||||
import argparse
|
||||
import datetime
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import pprint
|
||||
import shlex
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tarfile
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
MEDIA_TYPE_INDEX_V1_JSON = "application/vnd.oci.image.index.v1+json"
|
||||
MEDIA_TYPE_MANIFEST_V1_JSON = "application/vnd.oci.image.manifest.v1+json"
|
||||
|
||||
ENV_RUNTIME = "REPRO_RUNTIME"
|
||||
ENV_DATETIME = "REPRO_DATETIME"
|
||||
ENV_SDE = "REPRO_SOURCE_DATE_EPOCH"
|
||||
ENV_CACHE = "REPRO_CACHE"
|
||||
ENV_BUILDKIT = "REPRO_BUILDKIT_IMAGE"
|
||||
ENV_ROOTLESS = "REPRO_ROOTLESS"
|
||||
|
||||
DEFAULT_BUILDKIT_IMAGE = "moby/buildkit:v0.19.0@sha256:14aa1b4dd92ea0a4cd03a54d0c6079046ea98cd0c0ae6176bdd7036ba370cbbe"
|
||||
DEFAULT_BUILDKIT_IMAGE_ROOTLESS = "moby/buildkit:v0.19.0-rootless@sha256:e901cffdad753892a7c3afb8b9972549fca02c73888cf340c91ed801fdd96d71"
|
||||
|
||||
MSG_BUILD_CTX = """Build environment:
|
||||
- Container runtime: {runtime}
|
||||
- Buildkit image: {buildkit_image}
|
||||
- Rootless support: {rootless}
|
||||
- Caching enabled: {use_cache}
|
||||
- Build context: {context}
|
||||
- Dockerfile: {dockerfile}
|
||||
- Output: {output}
|
||||
|
||||
Build parameters:
|
||||
- SOURCE_DATE_EPOCH: {sde}
|
||||
- Build args: {build_args}
|
||||
- Tag: {tag}
|
||||
- Platform: {platform}
|
||||
|
||||
Podman-only arguments:
|
||||
- Buildkit arguments: {buildkit_args}
|
||||
|
||||
Docker-only arguments:
|
||||
- Docker Buildx arguments: {buildx_args}
|
||||
"""
|
||||
|
||||
|
||||
def pretty_error(obj: dict, msg: str):
|
||||
raise Exception(f"{msg}\n{pprint.pprint(obj)}")
|
||||
|
||||
|
||||
def get_key(obj: dict, key: str) -> object:
|
||||
if key not in obj:
|
||||
pretty_error(f"Could not find key '{key}' in the dictionary:", obj)
|
||||
return obj[key]
|
||||
|
||||
|
||||
def run(cmd, dry=False, check=True):
|
||||
action = "Would have run" if dry else "Running"
|
||||
logger.debug(f"{action}: {shlex.join(cmd)}")
|
||||
if not dry:
|
||||
subprocess.run(cmd, check=check)
|
||||
|
||||
|
||||
def snip_contents(contents: str, num: int) -> str:
|
||||
contents = contents.replace("\n", "")
|
||||
if len(contents) > num:
|
||||
return (
|
||||
contents[:num]
|
||||
+ f" [... {len(contents) - num} characters omitted."
|
||||
+ " Pass --show-contents to print them in their entirety]"
|
||||
)
|
||||
return contents
|
||||
|
||||
|
||||
def detect_container_runtime() -> str:
|
||||
"""Auto-detect the installed container runtime in the system."""
|
||||
if shutil.which("docker"):
|
||||
return "docker"
|
||||
elif shutil.which("podman"):
|
||||
return "podman"
|
||||
else:
|
||||
return None
|
||||
|
||||
|
||||
def parse_runtime(args) -> str:
|
||||
if args.runtime is not None:
|
||||
return args.runtime
|
||||
|
||||
runtime = os.environ.get(ENV_RUNTIME)
|
||||
if runtime is None:
|
||||
raise RuntimeError("No container runtime detected in your system")
|
||||
if runtime not in ("docker", "podman"):
|
||||
raise RuntimeError(
|
||||
"Only 'docker' or 'podman' container runtimes"
|
||||
" are currently supported by this script"
|
||||
)
|
||||
|
||||
|
||||
def parse_use_cache(args) -> bool:
|
||||
if args.no_cache:
|
||||
return False
|
||||
return bool(int(os.environ.get(ENV_CACHE, "1")))
|
||||
|
||||
|
||||
def parse_rootless(args, runtime: str) -> bool:
|
||||
rootless = args.rootless or bool(int(os.environ.get(ENV_ROOTLESS, "0")))
|
||||
if runtime != "podman" and rootless:
|
||||
raise RuntimeError("Rootless mode is only supported with Podman runtime")
|
||||
return rootless
|
||||
|
||||
|
||||
def parse_sde(args) -> str:
|
||||
sde = os.environ.get(ENV_SDE, args.source_date_epoch)
|
||||
dt = os.environ.get(ENV_DATETIME, args.datetime)
|
||||
|
||||
if (sde is not None and dt is not None) or (sde is None and dt is None):
|
||||
raise RuntimeError("You need to pass either a source date epoch or a datetime")
|
||||
|
||||
if sde is not None:
|
||||
return str(sde)
|
||||
|
||||
if dt is not None:
|
||||
d = datetime.datetime.fromisoformat(dt)
|
||||
# If the datetime is naive, assume its timezone is UTC. The check is taken from:
|
||||
# https://docs.python.org/3/library/datetime.html#determining-if-an-object-is-aware-or-naive
|
||||
if d.tzinfo is None or d.tzinfo.utcoffset(d) is None:
|
||||
d = d.replace(tzinfo=datetime.timezone.utc)
|
||||
return int(d.timestamp())
|
||||
|
||||
|
||||
def parse_buildkit_image(args, rootless: bool, runtime: str) -> str:
|
||||
default = DEFAULT_BUILDKIT_IMAGE_ROOTLESS if rootless else DEFAULT_BUILDKIT_IMAGE
|
||||
img = args.buildkit_image or os.environ.get(ENV_BUILDKIT, default)
|
||||
|
||||
if runtime == "podman" and not img.startswith("docker.io/"):
|
||||
img = "docker.io/" + img
|
||||
|
||||
return img
|
||||
|
||||
|
||||
def parse_build_args(args) -> str:
|
||||
return args.build_arg or []
|
||||
|
||||
|
||||
def parse_buildkit_args(args, runtime: str) -> str:
|
||||
if not args.buildkit_args:
|
||||
return []
|
||||
|
||||
if runtime != "podman":
|
||||
raise RuntimeError("Cannot specify Buildkit arguments using the Podman runtime")
|
||||
|
||||
return shlex.split(args.buildkit_args)
|
||||
|
||||
|
||||
def parse_buildx_args(args, runtime: str) -> str:
|
||||
if not args.buildx_args:
|
||||
return []
|
||||
|
||||
if runtime != "docker":
|
||||
raise RuntimeError(
|
||||
"Cannot specify Docker Buildx arguments using the Podman runtime"
|
||||
)
|
||||
|
||||
return shlex.split(args.buildx_args)
|
||||
|
||||
|
||||
def parse_image_digest(args) -> str | None:
|
||||
if not args.expected_image_digest:
|
||||
return None
|
||||
parsed = args.expected_image_digest.split(":", 1)
|
||||
if len(parsed) == 1:
|
||||
return parsed[0]
|
||||
else:
|
||||
return parsed[1]
|
||||
|
||||
|
||||
def parse_path(path: str | None) -> str | None:
|
||||
return path and str(Path(path).absolute())
|
||||
|
||||
|
||||
##########################
|
||||
# OCI parsing logic
|
||||
#
|
||||
# Compatible with:
|
||||
# * https://github.com/opencontainers/image-spec/blob/main/image-layout.md
|
||||
|
||||
|
||||
def oci_print_info(parsed: dict, full: bool) -> None:
|
||||
print(f"The OCI tarball contains an index and {len(parsed) - 1} manifest(s):")
|
||||
print()
|
||||
print(f"Image digest: {parsed[1]['digest']}")
|
||||
for i, info in enumerate(parsed):
|
||||
print()
|
||||
if i == 0:
|
||||
print(f"Index ({info['path']}):")
|
||||
else:
|
||||
print(f"Manifest {i} ({info['path']}):")
|
||||
print(f" Digest: {info['digest']}")
|
||||
print(f" Media type: {info['media_type']}")
|
||||
print(f" Platform: {info['platform'] or '-'}")
|
||||
contents = info["contents"] if full else snip_contents(info["contents"], 600)
|
||||
print(f" Contents: {contents}")
|
||||
print()
|
||||
|
||||
|
||||
def oci_normalize_path(path):
|
||||
if path.startswith("sha256:"):
|
||||
hash_algo, checksum = path.split(":")
|
||||
path = f"blobs/{hash_algo}/{checksum}"
|
||||
return path
|
||||
|
||||
|
||||
def oci_get_file_from_tarball(tar: tarfile.TarFile, path: str) -> dict:
|
||||
return
|
||||
|
||||
|
||||
def oci_parse_manifest(tar: tarfile.TarFile, path: str, platform: dict | None) -> dict:
|
||||
path = oci_normalize_path(path)
|
||||
contents = tar.extractfile(path).read().decode()
|
||||
digest = "sha256:" + hashlib.sha256(contents.encode()).hexdigest()
|
||||
contents_dict = json.loads(contents)
|
||||
media_type = get_key(contents_dict, "mediaType")
|
||||
manifests = contents_dict.get("manifests", [])
|
||||
|
||||
if platform:
|
||||
os = get_key(platform, "os")
|
||||
arch = get_key(platform, "architecture")
|
||||
platform = f"{os}/{arch}"
|
||||
|
||||
return {
|
||||
"path": path,
|
||||
"contents": contents,
|
||||
"digest": digest,
|
||||
"media_type": media_type,
|
||||
"platform": platform,
|
||||
"manifests": manifests,
|
||||
}
|
||||
|
||||
|
||||
def oci_parse_manifests_dfs(
|
||||
tar: tarfile.TarFile, path: str, parsed: list, platform: dict | None = None
|
||||
) -> None:
|
||||
info = oci_parse_manifest(tar, path, platform)
|
||||
parsed.append(info)
|
||||
for m in info["manifests"]:
|
||||
oci_parse_manifests_dfs(tar, m["digest"], parsed, m.get("platform"))
|
||||
|
||||
|
||||
def oci_parse_tarball(path: Path) -> dict:
|
||||
parsed = []
|
||||
with tarfile.TarFile.open(path) as tar:
|
||||
oci_parse_manifests_dfs(tar, "index.json", parsed)
|
||||
return parsed
|
||||
|
||||
|
||||
##########################
|
||||
# Image building logic
|
||||
|
||||
|
||||
def podman_build(
|
||||
context: str,
|
||||
dockerfile: str | None,
|
||||
tag: str | None,
|
||||
buildkit_image: str,
|
||||
sde: int,
|
||||
rootless: bool,
|
||||
use_cache: bool,
|
||||
output: Path,
|
||||
build_args: list,
|
||||
platform: str,
|
||||
buildkit_args: list,
|
||||
dry: bool,
|
||||
):
|
||||
rootless_args = []
|
||||
rootful_args = []
|
||||
if rootless:
|
||||
rootless_args = [
|
||||
"--userns",
|
||||
"keep-id:uid=1000,gid=1000",
|
||||
"--security-opt",
|
||||
"seccomp=unconfined",
|
||||
"--security-opt",
|
||||
"apparmor=unconfined",
|
||||
"-e",
|
||||
"BUILDKITD_FLAGS=--oci-worker-no-process-sandbox",
|
||||
]
|
||||
else:
|
||||
rootful_args = ["--privileged"]
|
||||
|
||||
dockerfile_args_podman = []
|
||||
dockerfile_args_buildkit = []
|
||||
if dockerfile:
|
||||
dockerfile_args_podman = ["-v", f"{dockerfile}:/tmp/Dockerfile"]
|
||||
dockerfile_args_buildkit = ["--local", "dockerfile=/tmp"]
|
||||
|
||||
tag_args = f",name={tag}" if tag else ""
|
||||
|
||||
cache_args = []
|
||||
if use_cache:
|
||||
cache_args = [
|
||||
"--export-cache",
|
||||
"type=local,dest=/tmp/cache",
|
||||
"--import-cache",
|
||||
"type=local,src=/tmp/cache",
|
||||
]
|
||||
|
||||
_build_args = []
|
||||
for arg in build_args:
|
||||
_build_args.append("--opt")
|
||||
_build_args.append(f"build-arg:{arg}")
|
||||
platform_args = ["--opt", f"platform={platform}"] if platform else []
|
||||
|
||||
cmd = [
|
||||
"podman",
|
||||
"run",
|
||||
"-it",
|
||||
"--rm",
|
||||
"-v",
|
||||
"buildkit_cache:/tmp/cache",
|
||||
"-v",
|
||||
f"{output.parent}:/tmp/image",
|
||||
"-v",
|
||||
f"{context}:/tmp/work",
|
||||
"--entrypoint",
|
||||
"buildctl-daemonless.sh",
|
||||
*rootless_args,
|
||||
*rootful_args,
|
||||
*dockerfile_args_podman,
|
||||
buildkit_image,
|
||||
"build",
|
||||
"--frontend",
|
||||
"dockerfile.v0",
|
||||
"--local",
|
||||
"context=/tmp/work",
|
||||
"--opt",
|
||||
f"build-arg:SOURCE_DATE_EPOCH={sde}",
|
||||
*_build_args,
|
||||
"--output",
|
||||
f"type=oci,dest=/tmp/image/{output.name},rewrite-timestamp=true{tag_args}",
|
||||
*cache_args,
|
||||
*dockerfile_args_buildkit,
|
||||
*platform_args,
|
||||
*buildkit_args,
|
||||
]
|
||||
|
||||
run(cmd, dry)
|
||||
|
||||
|
||||
def docker_build(
|
||||
context: str,
|
||||
dockerfile: str | None,
|
||||
tag: str | None,
|
||||
buildkit_image: str,
|
||||
sde: int,
|
||||
use_cache: bool,
|
||||
output: Path,
|
||||
build_args: list,
|
||||
platform: str,
|
||||
buildx_args: list,
|
||||
dry: bool,
|
||||
):
|
||||
builder_id = hashlib.sha256(buildkit_image.encode()).hexdigest()
|
||||
builder_name = f"repro-build-{builder_id}"
|
||||
tag_args = ["-t", tag] if tag else []
|
||||
cache_args = [] if use_cache else ["--no-cache", "--pull"]
|
||||
|
||||
cmd = [
|
||||
"docker",
|
||||
"buildx",
|
||||
"create",
|
||||
"--name",
|
||||
builder_name,
|
||||
"--driver-opt",
|
||||
f"image={buildkit_image}",
|
||||
]
|
||||
run(cmd, dry, check=False)
|
||||
|
||||
dockerfile_args = ["-f", dockerfile] if dockerfile else []
|
||||
_build_args = []
|
||||
for arg in build_args:
|
||||
_build_args.append("--build-arg")
|
||||
_build_args.append(arg)
|
||||
platform_args = ["--platform", platform] if platform else []
|
||||
|
||||
cmd = [
|
||||
"docker",
|
||||
"buildx",
|
||||
"--builder",
|
||||
builder_name,
|
||||
"build",
|
||||
"--build-arg",
|
||||
f"SOURCE_DATE_EPOCH={sde}",
|
||||
*_build_args,
|
||||
"--provenance",
|
||||
"false",
|
||||
"--output",
|
||||
f"type=oci,dest={output},rewrite-timestamp=true",
|
||||
*cache_args,
|
||||
*tag_args,
|
||||
*dockerfile_args,
|
||||
*platform_args,
|
||||
*buildx_args,
|
||||
context,
|
||||
]
|
||||
run(cmd, dry)
|
||||
|
||||
|
||||
##########################
|
||||
# Command logic
|
||||
|
||||
|
||||
def build(args):
|
||||
runtime = parse_runtime(args)
|
||||
use_cache = parse_use_cache(args)
|
||||
sde = parse_sde(args)
|
||||
rootless = parse_rootless(args, runtime)
|
||||
buildkit_image = parse_buildkit_image(args, rootless, runtime)
|
||||
build_args = parse_build_args(args)
|
||||
platform = args.platform
|
||||
buildkit_args = parse_buildkit_args(args, runtime)
|
||||
buildx_args = parse_buildx_args(args, runtime)
|
||||
tag = args.tag
|
||||
dockerfile = parse_path(args.file)
|
||||
output = Path(parse_path(args.output))
|
||||
dry = args.dry
|
||||
context = parse_path(args.context)
|
||||
|
||||
logger.info(
|
||||
MSG_BUILD_CTX.format(
|
||||
runtime=runtime,
|
||||
buildkit_image=buildkit_image,
|
||||
sde=sde,
|
||||
rootless=rootless,
|
||||
use_cache=use_cache,
|
||||
context=context,
|
||||
dockerfile=dockerfile or "(not provided)",
|
||||
tag=tag or "(not provided)",
|
||||
output=output,
|
||||
build_args=",".join(build_args) or "(not provided)",
|
||||
platform=platform or "(default)",
|
||||
buildkit_args=" ".join(buildkit_args) or "(not provided)",
|
||||
buildx_args=" ".join(buildx_args) or "(not provided)",
|
||||
)
|
||||
)
|
||||
|
||||
try:
|
||||
if runtime == "docker":
|
||||
docker_build(
|
||||
context,
|
||||
dockerfile,
|
||||
tag,
|
||||
buildkit_image,
|
||||
sde,
|
||||
use_cache,
|
||||
output,
|
||||
build_args,
|
||||
platform,
|
||||
buildx_args,
|
||||
dry,
|
||||
)
|
||||
else:
|
||||
podman_build(
|
||||
context,
|
||||
dockerfile,
|
||||
tag,
|
||||
buildkit_image,
|
||||
sde,
|
||||
rootless,
|
||||
use_cache,
|
||||
output,
|
||||
build_args,
|
||||
platform,
|
||||
buildkit_args,
|
||||
dry,
|
||||
)
|
||||
except subprocess.CalledProcessError as e:
|
||||
logger.error(f"Failed with {e.returncode}")
|
||||
sys.exit(e.returncode)
|
||||
|
||||
|
||||
def analyze(args) -> None:
|
||||
expected_image_digest = parse_image_digest(args)
|
||||
tarball_path = Path(args.tarball)
|
||||
|
||||
parsed = oci_parse_tarball(tarball_path)
|
||||
oci_print_info(parsed, args.show_contents)
|
||||
|
||||
if expected_image_digest:
|
||||
cur_digest = parsed[1]["digest"].split(":")[1]
|
||||
if cur_digest != expected_image_digest:
|
||||
raise Exception(
|
||||
f"The image does not have the expected digest: {cur_digest} != {expected_image_digest}"
|
||||
)
|
||||
print(f"✅ Image digest matches {expected_image_digest}")
|
||||
|
||||
|
||||
def define_build_cmd_args(parser: argparse.ArgumentParser) -> None:
|
||||
parser.add_argument(
|
||||
"--runtime",
|
||||
choices=["docker", "podman"],
|
||||
default=detect_container_runtime(),
|
||||
help="The container runtime for building the image (default: %(default)s)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--datetime",
|
||||
metavar="YYYY-MM-DD",
|
||||
default=None,
|
||||
help=(
|
||||
"Provide a date and (optionally) a time in ISO format, which will"
|
||||
" be used as the timestamp of the image layers"
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--buildkit-image",
|
||||
metavar="NAME:TAG@DIGEST",
|
||||
default=None,
|
||||
help=(
|
||||
"The Buildkit container image which will be used for building the"
|
||||
" reproducible container image. Make sure to pass the '-rootless'"
|
||||
" variant if you are using rootless Podman"
|
||||
" (default: docker.io/moby/buildkit:v0.19.0)"
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--source-date-epoch",
|
||||
"--sde",
|
||||
metavar="SECONDS",
|
||||
type=int,
|
||||
default=None,
|
||||
help="Provide a Unix timestamp for the image layers",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--no-cache",
|
||||
default=False,
|
||||
action="store_true",
|
||||
help="Do not use existing cached images for the container build. Build from the start with a new set of cached layers.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--rootless",
|
||||
default=False,
|
||||
action="store_true",
|
||||
help="Run Buildkit in rootless mode (Podman only)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-f",
|
||||
"--file",
|
||||
metavar="FILE",
|
||||
default=None,
|
||||
help="Pathname of a Dockerfile",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-o",
|
||||
"--output",
|
||||
metavar="FILE",
|
||||
default=Path.cwd() / "image.tar",
|
||||
help="Path to save OCI tarball (default: %(default)s)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-t",
|
||||
"--tag",
|
||||
metavar="TAG",
|
||||
default=None,
|
||||
help="Tag the built image with the name %(metavar)s",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--build-arg",
|
||||
metavar="ARG=VALUE",
|
||||
action="append",
|
||||
default=None,
|
||||
help="Set build-time variables",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--platform",
|
||||
metavar="PLAT1,PLAT2",
|
||||
default=None,
|
||||
help="Set platform if server is multi-platform capable",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--buildkit-args",
|
||||
metavar="'ARG1 ARG2'",
|
||||
default=None,
|
||||
help="Extra arguments for Buildkit (Podman only)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--buildx-args",
|
||||
metavar="'ARG1 ARG2'",
|
||||
default=None,
|
||||
help="Extra arguments for Docker Buildx (Docker only)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dry",
|
||||
default=False,
|
||||
action="store_true",
|
||||
help="Do not run any commands, just print what would happen",
|
||||
)
|
||||
parser.add_argument(
|
||||
"context",
|
||||
metavar="CONTEXT",
|
||||
help="Path to the build context",
|
||||
)
|
||||
|
||||
|
||||
def parse_args() -> dict:
|
||||
parser = argparse.ArgumentParser()
|
||||
subparsers = parser.add_subparsers(dest="command", help="Available commands")
|
||||
|
||||
build_parser = subparsers.add_parser("build", help="Perform a build operation")
|
||||
build_parser.set_defaults(func=build)
|
||||
define_build_cmd_args(build_parser)
|
||||
|
||||
analyze_parser = subparsers.add_parser("analyze", help="Analyze an OCI tarball")
|
||||
analyze_parser.set_defaults(func=analyze)
|
||||
analyze_parser.add_argument(
|
||||
"tarball",
|
||||
metavar="FILE",
|
||||
help="Path to OCI image in .tar format",
|
||||
)
|
||||
analyze_parser.add_argument(
|
||||
"--expected-image-digest",
|
||||
metavar="DIGEST",
|
||||
default=None,
|
||||
help="The expected digest for the provided image",
|
||||
)
|
||||
analyze_parser.add_argument(
|
||||
"--show-contents",
|
||||
default=False,
|
||||
action="store_true",
|
||||
help="Show full file contents",
|
||||
)
|
||||
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
logging.basicConfig(
|
||||
level=logging.DEBUG,
|
||||
format="%(asctime)s - %(levelname)s - %(message)s",
|
||||
datefmt="%Y-%m-%d %H:%M:%S",
|
||||
)
|
||||
args = parse_args()
|
||||
|
||||
if not hasattr(args, "func"):
|
||||
args.func = build
|
||||
args.func(args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
|
@ -13,8 +13,6 @@ if platform.system() in ["Darwin", "Windows"]:
|
|||
elif platform.system() == "Linux":
|
||||
CONTAINER_RUNTIME = "podman"
|
||||
|
||||
ARCH = platform.machine()
|
||||
|
||||
|
||||
def str2bool(v):
|
||||
if isinstance(v, bool):
|
||||
|
@ -50,6 +48,16 @@ def determine_git_tag():
|
|||
)
|
||||
|
||||
|
||||
def determine_debian_archive_date():
|
||||
"""Get the date of the Debian archive from Dockerfile.env."""
|
||||
for env in Path("Dockerfile.env").read_text().split("\n"):
|
||||
if env.startswith("DEBIAN_ARCHIVE_DATE"):
|
||||
return env.split("=")[1]
|
||||
raise Exception(
|
||||
"Could not find 'DEBIAN_ARCHIVE_DATE' build argument in Dockerfile.env"
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
|
@ -64,22 +72,23 @@ def main():
|
|||
help=f"The platform for building the image (default: current platform)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--no-save",
|
||||
action="store_true",
|
||||
help="Do not save the container image as a tarball in share/container.tar.gz",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--buildx",
|
||||
action="store_true",
|
||||
help="Use the buildx platform of Docker or Podman",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--compress-level",
|
||||
type=int,
|
||||
choices=range(0, 10),
|
||||
default=9,
|
||||
help="The Gzip compression level, from 0 (lowest) to 9 (highest, default)",
|
||||
"--output",
|
||||
"-o",
|
||||
default=str(Path("share") / "container.tar"),
|
||||
help="Path to store the container image",
|
||||
)
|
||||
# parser.add_argument(
|
||||
# "--buildx",
|
||||
# action="store_true",
|
||||
# help="Use the buildx platform of Docker or Podman",
|
||||
# )
|
||||
# parser.add_argument(
|
||||
# "--compress-level",
|
||||
# type=int,
|
||||
# choices=range(0, 10),
|
||||
# default=9,
|
||||
# help="The Gzip compression level, from 0 (lowest) to 9 (highest, default)",
|
||||
# )
|
||||
parser.add_argument(
|
||||
"--use-cache",
|
||||
type=str2bool,
|
||||
|
@ -95,84 +104,59 @@ def main():
|
|||
)
|
||||
parser.add_argument(
|
||||
"--debian-archive-date",
|
||||
default=None,
|
||||
help="Use a specific Debian snapshot archive, by its date",
|
||||
"-d",
|
||||
default=determine_debian_archive_date(),
|
||||
help="Use a specific Debian snapshot archive, by its date (default %(default)s)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dry",
|
||||
default=False,
|
||||
action="store_true",
|
||||
help="Do not run any commands, just print what would happen",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
tarball_path = Path("share") / "container.tar.gz"
|
||||
image_id_path = Path("share") / "image-id.txt"
|
||||
|
||||
print(f"Building for architecture '{ARCH}'")
|
||||
|
||||
tag = args.tag or determine_git_tag()
|
||||
date = f"{args.debian_archive_date}-" if args.debian_archive_date and not args.tag else ""
|
||||
image_name_tagged = IMAGE_NAME + ":" + date + tag
|
||||
image_name_tagged = f"{IMAGE_NAME}:{args.debian_archive_date}-{tag}"
|
||||
|
||||
print(f"Will tag the container image as '{image_name_tagged}'")
|
||||
with open(image_id_path, "w") as f:
|
||||
f.write(tag)
|
||||
image_id_path = Path("share") / "image-id.txt"
|
||||
if not args.dry:
|
||||
with open(image_id_path, "w") as f:
|
||||
f.write(tag)
|
||||
|
||||
# Build the container image, and tag it with the calculated tag
|
||||
print("Building container image")
|
||||
cache_args = [] if args.use_cache else ["--no-cache"]
|
||||
platform_args = [] if not args.platform else ["--platform", args.platform]
|
||||
build_args = []
|
||||
if args.debian_archive_date:
|
||||
build_args = ["--build-arg", f"DEBIAN_ARCHIVE_DATE={args.debian_archive_date}"]
|
||||
buildx_args = []
|
||||
if args.buildx:
|
||||
buildx_args = [
|
||||
"buildx",
|
||||
"build",
|
||||
"--output",
|
||||
"type=image,name-canonical=true,push=false,push-by-digest=false",
|
||||
]
|
||||
rootless_args = [] if args.runtime == "docker" else ["--rootless"]
|
||||
dry_args = [] if not args.dry else ["--dry"]
|
||||
|
||||
subprocess.run(
|
||||
[
|
||||
"./dev_scripts/repro-build",
|
||||
"build",
|
||||
"--runtime",
|
||||
args.runtime,
|
||||
*buildx_args,
|
||||
BUILD_CONTEXT,
|
||||
*build_args,
|
||||
"--build-arg",
|
||||
f"DEBIAN_ARCHIVE_DATE={args.debian_archive_date}",
|
||||
"--datetime",
|
||||
args.debian_archive_date,
|
||||
*dry_args,
|
||||
*cache_args,
|
||||
*platform_args,
|
||||
"--provenance",
|
||||
"false",
|
||||
"-f",
|
||||
"Dockerfile",
|
||||
*rootless_args,
|
||||
"--tag",
|
||||
image_name_tagged,
|
||||
"--output",
|
||||
args.output,
|
||||
"-f",
|
||||
"Dockerfile",
|
||||
BUILD_CONTEXT,
|
||||
],
|
||||
check=True,
|
||||
)
|
||||
|
||||
if not args.no_save:
|
||||
print("Saving container image")
|
||||
cmd = subprocess.Popen(
|
||||
[
|
||||
CONTAINER_RUNTIME,
|
||||
"save",
|
||||
image_name_tagged,
|
||||
],
|
||||
stdout=subprocess.PIPE,
|
||||
)
|
||||
|
||||
print("Compressing container image")
|
||||
chunk_size = 4 << 20
|
||||
with gzip.open(
|
||||
tarball_path,
|
||||
"wb",
|
||||
compresslevel=args.compress_level,
|
||||
) as gzip_f:
|
||||
while True:
|
||||
chunk = cmd.stdout.read(chunk_size)
|
||||
if len(chunk) > 0:
|
||||
gzip_f.write(chunk)
|
||||
else:
|
||||
break
|
||||
cmd.wait(5)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
|
|
Loading…
Reference in a new issue