diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a8b7ab7..108cb37 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -85,7 +85,7 @@ jobs: id: cache-container-image uses: actions/cache@v4 with: - key: v3-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/*.py', 'dangerzone/container_helpers/*', 'install/common/build-image.py', 'poetry.lock') }} + key: v4-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/*.py', 'dangerzone/container_helpers/*', 'install/common/build-image.py') }} path: | share/container.tar.gz share/image-id.txt diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 306de24..f4e7f40 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,7 +59,7 @@ jobs: id: cache-container-image uses: actions/cache@v4 with: - key: v3-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/*.py', 'dangerzone/container_helpers/*', 'install/common/build-image.py', 'poetry.lock') }} + key: v4-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/*.py', 'dangerzone/container_helpers/*', 'install/common/build-image.py') }} path: |- share/container.tar.gz share/image-id.txt @@ -67,7 +67,6 @@ jobs: - name: Build Dangerzone container image if: ${{ steps.cache-container-image.outputs.cache-hit != 'true' }} run: | - sudo apt-get install -y python3-poetry python3 ./install/common/build-image.py - name: Upload container image @@ -227,7 +226,7 @@ jobs: - name: Restore container cache uses: actions/cache/restore@v4 with: - key: v3-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/*.py', 'dangerzone/container_helpers/*', 'install/common/build-image.py', 'poetry.lock') }} + key: v4-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/*.py', 'dangerzone/container_helpers/*', 'install/common/build-image.py') }} path: |- share/container.tar.gz share/image-id.txt @@ -334,7 +333,7 @@ jobs: - name: Restore container image uses: actions/cache/restore@v4 with: - key: v3-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/*.py', 'dangerzone/container_helpers/*', 'install/common/build-image.py', 'poetry.lock') }} + key: v4-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/*.py', 'dangerzone/container_helpers/*', 'install/common/build-image.py') }} path: |- share/container.tar.gz share/image-id.txt @@ -429,7 +428,7 @@ jobs: - name: Restore container image uses: actions/cache/restore@v4 with: - key: v3-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/*.py', 'dangerzone/container_helpers/*', 'install/common/build-image.py', 'poetry.lock') }} + key: v4-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/*.py', 'dangerzone/container_helpers/*', 'install/common/build-image.py') }} path: |- share/container.tar.gz share/image-id.txt diff --git a/dodo.py b/dodo.py index 54a85af..d2f3ad1 100644 --- a/dodo.py +++ b/dodo.py @@ -63,7 +63,6 @@ TESSDATA_TARGETS = list_language_data() IMAGE_DEPS = [ "Dockerfile", - "poetry.lock", *list_files("dangerzone/conversion"), *list_files("dangerzone/container_helpers"), "install/common/build-image.py", diff --git a/install/common/build-image.py b/install/common/build-image.py index 6d99877..e2e3b01 100644 --- a/install/common/build-image.py +++ b/install/common/build-image.py @@ -1,6 +1,5 @@ import argparse import gzip -import os import platform import secrets import subprocess @@ -9,7 +8,6 @@ from pathlib import Path BUILD_CONTEXT = "dangerzone/" IMAGE_NAME = "dangerzone.rocks/dangerzone" -REQUIREMENTS_TXT = "container-pip-requirements.txt" if platform.system() in ["Darwin", "Windows"]: CONTAINER_RUNTIME = "docker" elif platform.system() == "Linux": @@ -84,91 +82,48 @@ def main(): with open(image_id_path, "w") as f: f.write(tag) - print("Exporting container pip dependencies") - with ContainerPipDependencies(): - if not args.use_cache: - print("Pulling base image") - subprocess.run( - [ - args.runtime, - "pull", - "alpine:latest", - ], - check=True, - ) + # Build the container image, and tag it with the calculated tag + print("Building container image") + cache_args = [] if args.use_cache else ["--no-cache"] + subprocess.run( + [ + args.runtime, + "build", + BUILD_CONTEXT, + *cache_args, + "-f", + "Dockerfile", + "--tag", + image_name_tagged, + ], + check=True, + ) - # Build the container image, and tag it with the calculated tag - print("Building container image") - cache_args = [] if args.use_cache else ["--no-cache"] - subprocess.run( + if not args.no_save: + print("Saving container image") + cmd = subprocess.Popen( [ - args.runtime, - "build", - BUILD_CONTEXT, - *cache_args, - "--build-arg", - f"REQUIREMENTS_TXT={REQUIREMENTS_TXT}", - "--build-arg", - f"ARCH={ARCH}", - "-f", - "Dockerfile", - "--tag", + CONTAINER_RUNTIME, + "save", image_name_tagged, ], - check=True, + stdout=subprocess.PIPE, ) - if not args.no_save: - print("Saving container image") - cmd = subprocess.Popen( - [ - CONTAINER_RUNTIME, - "save", - image_name_tagged, - ], - stdout=subprocess.PIPE, - ) - - print("Compressing container image") - chunk_size = 4 << 20 - with gzip.open( - tarball_path, - "wb", - compresslevel=args.compress_level, - ) as gzip_f: - while True: - chunk = cmd.stdout.read(chunk_size) - if len(chunk) > 0: - gzip_f.write(chunk) - else: - break - cmd.wait(5) - - -class ContainerPipDependencies: - """Generates PIP dependencies within container""" - - def __enter__(self): - try: - container_requirements_txt = subprocess.check_output( - ["poetry", "export", "--only", "container"], universal_newlines=True - ) - except subprocess.CalledProcessError as e: - print("FAILURE", e.returncode, e.output) - print(f"REQUIREMENTS: {container_requirements_txt}") - # XXX Export container dependencies and exclude pymupdfb since it is not needed in container - req_txt_pymupdfb_stripped = container_requirements_txt.split("pymupdfb")[0] - with open(Path(BUILD_CONTEXT) / REQUIREMENTS_TXT, "w") as f: - if ARCH == "arm64": - # PyMuPDF needs to be built on ARM64 machines - # But is already provided as a prebuilt-wheel on other architectures - f.write(req_txt_pymupdfb_stripped) - else: - f.write(container_requirements_txt) - - def __exit__(self, exc_type, exc_value, exc_tb): - print("Leaving the context...") - os.remove(Path(BUILD_CONTEXT) / REQUIREMENTS_TXT) + print("Compressing container image") + chunk_size = 4 << 20 + with gzip.open( + tarball_path, + "wb", + compresslevel=args.compress_level, + ) as gzip_f: + while True: + chunk = cmd.stdout.read(chunk_size) + if len(chunk) > 0: + gzip_f.write(chunk) + else: + break + cmd.wait(5) if __name__ == "__main__": diff --git a/install/linux/vendor-pymupdf.py b/install/linux/vendor-pymupdf.py index 0c49720..9cb5ccc 100755 --- a/install/linux/vendor-pymupdf.py +++ b/install/linux/vendor-pymupdf.py @@ -28,7 +28,7 @@ def main(): ) logger.info("Getting PyMuPDF deps as requirements.txt") - cmd = ["poetry", "export", "--only", "container"] + cmd = ["poetry", "export", "--only", "debian"] container_requirements_txt = subprocess.check_output(cmd) # XXX: Hack for Ubuntu Focal. diff --git a/poetry.lock b/poetry.lock index 543f210..1acd2a7 100644 --- a/poetry.lock +++ b/poetry.lock @@ -767,6 +767,9 @@ optional = false python-versions = "<3.14,>=3.9" files = [ {file = "PySide6-6.8.1.1-cp39-abi3-macosx_12_0_universal2.whl", hash = "sha256:805728a7ed58352a02689b953ddbe29af1c8944f8c7f2c28312dc0b69f64b85e"}, + {file = "PySide6-6.8.1.1-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:70f8c4745d981ebb5bb93d7b825222532d553373b68b9db7a42cfcee25cafc9a"}, + {file = "PySide6-6.8.1.1-cp39-abi3-manylinux_2_39_aarch64.whl", hash = "sha256:de80ac62087a716b2bada2e3ddd739c5d176dc4be819abef91274d53d75f4e58"}, + {file = "PySide6-6.8.1.1-cp39-abi3-win_amd64.whl", hash = "sha256:60a2551053fa69845b893fb821507e2cc89d3a8a8b43726d568acd1250ad44fb"}, ] [package.dependencies] @@ -799,6 +802,9 @@ optional = false python-versions = "<3.14,>=3.9" files = [ {file = "PySide6_Addons-6.8.1.1-cp39-abi3-macosx_12_0_universal2.whl", hash = "sha256:83d35d7a1a7dbd1a16b4040a26ad4d5cc030a2aed4d439241babee1225d6e58a"}, + {file = "PySide6_Addons-6.8.1.1-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:5ef45aeadca37d658e44a41e11f2b2e43dfc34c780a6be1cd09d96a7696e6cc6"}, + {file = "PySide6_Addons-6.8.1.1-cp39-abi3-manylinux_2_39_aarch64.whl", hash = "sha256:e1b4a20b0bcbc2e440faba62e0d164223b8fd6f041d749543bc3812979116c4c"}, + {file = "PySide6_Addons-6.8.1.1-cp39-abi3-win_amd64.whl", hash = "sha256:d8ae86944ac48cc9891666cf71565acebd403a953d0e050be4d41ac490788d0a"}, ] [package.dependencies] @@ -829,6 +835,9 @@ optional = false python-versions = "<3.14,>=3.9" files = [ {file = "PySide6_Essentials-6.8.1.1-cp39-abi3-macosx_12_0_universal2.whl", hash = "sha256:25f3fdb281ac3b442f08250e3284d3b1944f7c64c62ed93b57678a62c199cf46"}, + {file = "PySide6_Essentials-6.8.1.1-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:62b64842a91114c224c41eeb6a8c8f255ba60268bc5ac19724f944d60e2277c6"}, + {file = "PySide6_Essentials-6.8.1.1-cp39-abi3-manylinux_2_39_aarch64.whl", hash = "sha256:e0c1cc3cfb2ea5eea70748da7d22032a59ea641e24988f543d5b274c0adab065"}, + {file = "PySide6_Essentials-6.8.1.1-cp39-abi3-win_amd64.whl", hash = "sha256:085f12e16db31eb0e802b21c64eabf582f54db6c44463a1f5e1814d897b1f2c0"}, ] [package.dependencies] @@ -1071,6 +1080,9 @@ optional = false python-versions = "<3.14,>=3.9" files = [ {file = "shiboken6-6.8.1.1-cp39-abi3-macosx_12_0_universal2.whl", hash = "sha256:42fbb173a772c4e059dbeafb302e96f6ea8e1c9bacf05fab71ea7eb0d8f97b01"}, + {file = "shiboken6-6.8.1.1-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:d672df0f29dc5f44de7205c1acae4d0471ba8371bb1d68fdacbf1686f4d22a96"}, + {file = "shiboken6-6.8.1.1-cp39-abi3-manylinux_2_39_aarch64.whl", hash = "sha256:ff1b22a66476b042d3dc09870edca353fdac1c1f517a4cdc364b24e296213ecd"}, + {file = "shiboken6-6.8.1.1-cp39-abi3-win_amd64.whl", hash = "sha256:26f7041c77058a8ecfb9345caa187250b199de79cfb37e33936e5fbd468a7780"}, ] [[package]] @@ -1270,4 +1282,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = ">=3.9,<3.13" -content-hash = "30751b4a27cebd020b5222de2abef4e38fb5f6676fdf936693060e42cd35c25f" +content-hash = "035acbf0d18e9eac2f2c338c56b55ea84cfb0c59db64ac014c55607c2ea7cfb2" diff --git a/pyproject.toml b/pyproject.toml index d021422..05a57b8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,7 +64,7 @@ strip-ansi = "*" pytest-subprocess = "^1.5.2" pytest-rerunfailures = "^14.0" -[tool.poetry.group.container.dependencies] +[tool.poetry.group.debian.dependencies] pymupdf = "1.24.11" # Last version to support python 3.8 (needed for Ubuntu Focal support) [tool.poetry.group.dev.dependencies]