diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d5c1f09..8e53232 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -85,7 +85,7 @@ jobs: id: cache-container-image uses: actions/cache@v4 with: - key: v3-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/*.py', 'dangerzone/container/*', 'install/common/build-image.py', 'poetry.lock') }} + key: v3-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/*.py', 'dangerzone/container/*', 'install/common/build-image.py') }} path: | share/container.tar.gz share/image-id.txt diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 34fb410..b6bb54d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,7 +59,7 @@ jobs: id: cache-container-image uses: actions/cache@v4 with: - key: v3-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/*.py', 'dangerzone/container/*', 'install/common/build-image.py', 'poetry.lock') }} + key: v3-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/*.py', 'dangerzone/container/*', 'install/common/build-image.py') }} path: |- share/container.tar.gz share/image-id.txt @@ -67,7 +67,6 @@ jobs: - name: Build Dangerzone container image if: ${{ steps.cache-container-image.outputs.cache-hit != 'true' }} run: | - sudo apt-get install -y python3-poetry python3 ./install/common/build-image.py - name: Upload container image @@ -227,7 +226,7 @@ jobs: - name: Restore container cache uses: actions/cache/restore@v4 with: - key: v3-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/*.py', 'dangerzone/container/*', 'install/common/build-image.py', 'poetry.lock') }} + key: v3-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/*.py', 'dangerzone/container/*', 'install/common/build-image.py') }} path: |- share/container.tar.gz share/image-id.txt @@ -334,7 +333,7 @@ jobs: - name: Restore container image uses: actions/cache/restore@v4 with: - key: v3-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/*.py', 'dangerzone/container/*', 'install/common/build-image.py', 'poetry.lock') }} + key: v3-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/*.py', 'dangerzone/container/*', 'install/common/build-image.py') }} path: |- share/container.tar.gz share/image-id.txt @@ -429,7 +428,7 @@ jobs: - name: Restore container image uses: actions/cache/restore@v4 with: - key: v3-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/*.py', 'dangerzone/container/*', 'install/common/build-image.py', 'poetry.lock') }} + key: v3-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/*.py', 'dangerzone/container/*', 'install/common/build-image.py') }} path: |- share/container.tar.gz share/image-id.txt diff --git a/dodo.py b/dodo.py index 8fca92a..9172009 100644 --- a/dodo.py +++ b/dodo.py @@ -63,7 +63,6 @@ TESSDATA_TARGETS = list_language_data() IMAGE_DEPS = [ "Dockerfile", - "poetry.lock", *list_files("dangerzone/conversion"), *list_files("dangerzone/container"), "install/common/build-image.py", diff --git a/install/common/build-image.py b/install/common/build-image.py index 6d99877..e2e3b01 100644 --- a/install/common/build-image.py +++ b/install/common/build-image.py @@ -1,6 +1,5 @@ import argparse import gzip -import os import platform import secrets import subprocess @@ -9,7 +8,6 @@ from pathlib import Path BUILD_CONTEXT = "dangerzone/" IMAGE_NAME = "dangerzone.rocks/dangerzone" -REQUIREMENTS_TXT = "container-pip-requirements.txt" if platform.system() in ["Darwin", "Windows"]: CONTAINER_RUNTIME = "docker" elif platform.system() == "Linux": @@ -84,91 +82,48 @@ def main(): with open(image_id_path, "w") as f: f.write(tag) - print("Exporting container pip dependencies") - with ContainerPipDependencies(): - if not args.use_cache: - print("Pulling base image") - subprocess.run( - [ - args.runtime, - "pull", - "alpine:latest", - ], - check=True, - ) + # Build the container image, and tag it with the calculated tag + print("Building container image") + cache_args = [] if args.use_cache else ["--no-cache"] + subprocess.run( + [ + args.runtime, + "build", + BUILD_CONTEXT, + *cache_args, + "-f", + "Dockerfile", + "--tag", + image_name_tagged, + ], + check=True, + ) - # Build the container image, and tag it with the calculated tag - print("Building container image") - cache_args = [] if args.use_cache else ["--no-cache"] - subprocess.run( + if not args.no_save: + print("Saving container image") + cmd = subprocess.Popen( [ - args.runtime, - "build", - BUILD_CONTEXT, - *cache_args, - "--build-arg", - f"REQUIREMENTS_TXT={REQUIREMENTS_TXT}", - "--build-arg", - f"ARCH={ARCH}", - "-f", - "Dockerfile", - "--tag", + CONTAINER_RUNTIME, + "save", image_name_tagged, ], - check=True, + stdout=subprocess.PIPE, ) - if not args.no_save: - print("Saving container image") - cmd = subprocess.Popen( - [ - CONTAINER_RUNTIME, - "save", - image_name_tagged, - ], - stdout=subprocess.PIPE, - ) - - print("Compressing container image") - chunk_size = 4 << 20 - with gzip.open( - tarball_path, - "wb", - compresslevel=args.compress_level, - ) as gzip_f: - while True: - chunk = cmd.stdout.read(chunk_size) - if len(chunk) > 0: - gzip_f.write(chunk) - else: - break - cmd.wait(5) - - -class ContainerPipDependencies: - """Generates PIP dependencies within container""" - - def __enter__(self): - try: - container_requirements_txt = subprocess.check_output( - ["poetry", "export", "--only", "container"], universal_newlines=True - ) - except subprocess.CalledProcessError as e: - print("FAILURE", e.returncode, e.output) - print(f"REQUIREMENTS: {container_requirements_txt}") - # XXX Export container dependencies and exclude pymupdfb since it is not needed in container - req_txt_pymupdfb_stripped = container_requirements_txt.split("pymupdfb")[0] - with open(Path(BUILD_CONTEXT) / REQUIREMENTS_TXT, "w") as f: - if ARCH == "arm64": - # PyMuPDF needs to be built on ARM64 machines - # But is already provided as a prebuilt-wheel on other architectures - f.write(req_txt_pymupdfb_stripped) - else: - f.write(container_requirements_txt) - - def __exit__(self, exc_type, exc_value, exc_tb): - print("Leaving the context...") - os.remove(Path(BUILD_CONTEXT) / REQUIREMENTS_TXT) + print("Compressing container image") + chunk_size = 4 << 20 + with gzip.open( + tarball_path, + "wb", + compresslevel=args.compress_level, + ) as gzip_f: + while True: + chunk = cmd.stdout.read(chunk_size) + if len(chunk) > 0: + gzip_f.write(chunk) + else: + break + cmd.wait(5) if __name__ == "__main__": diff --git a/install/linux/vendor-pymupdf.py b/install/linux/vendor-pymupdf.py index 0c49720..9cb5ccc 100755 --- a/install/linux/vendor-pymupdf.py +++ b/install/linux/vendor-pymupdf.py @@ -28,7 +28,7 @@ def main(): ) logger.info("Getting PyMuPDF deps as requirements.txt") - cmd = ["poetry", "export", "--only", "container"] + cmd = ["poetry", "export", "--only", "debian"] container_requirements_txt = subprocess.check_output(cmd) # XXX: Hack for Ubuntu Focal. diff --git a/poetry.lock b/poetry.lock index 247d955..6c638f7 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1244,4 +1244,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = ">=3.9,<3.13" -content-hash = "68663ce40ba8a7c7f7cc7868e5f771472555773ff2ef04dad7e0150218ca3eb0" +content-hash = "2d7753fa7ee1056d871fe67d718cfa2ea9acdfada1c6c3b1e41f98d5220d3879" diff --git a/pyproject.toml b/pyproject.toml index f69488d..81e0671 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,7 +57,7 @@ strip-ansi = "*" pytest-subprocess = "^1.5.2" pytest-rerunfailures = "^14.0" -[tool.poetry.group.container.dependencies] +[tool.poetry.group.debian.dependencies] pymupdf = "1.24.11" # Last version to support python 3.8 (needed for Ubuntu Focal support) [tool.poetry.group.dev.dependencies]