mirror of
https://github.com/freedomofpress/dangerzone.git
synced 2025-04-28 09:52:37 +02:00
Do not use poetry.lock when building the container image
Remove all the scaffolding in our `build-image.py` script for using the `poetry.lock` file, now that we install PyMuPDF from the Debian repos.
This commit is contained in:
Alex Pyrgiotis
parent
033ce0986d
commit
14bb6c0e39
7 changed files with 56 additions and 91 deletions
2
.github/workflows/build.yml
vendored
2
.github/workflows/build.yml
vendored
|
|
@ -85,7 +85,7 @@ jobs:
|
|||
id: cache-container-image
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
key: v3-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/*.py', 'dangerzone/container_helpers/*', 'install/common/build-image.py', 'poetry.lock') }}
|
||||
key: v4-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/*.py', 'dangerzone/container_helpers/*', 'install/common/build-image.py') }}
|
||||
path: |
|
||||
share/container.tar.gz
|
||||
share/image-id.txt
|
||||
|
|
|
|||
9
.github/workflows/ci.yml
vendored
9
.github/workflows/ci.yml
vendored
|
|
@ -59,7 +59,7 @@ jobs:
|
|||
id: cache-container-image
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
key: v3-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/*.py', 'dangerzone/container_helpers/*', 'install/common/build-image.py', 'poetry.lock') }}
|
||||
key: v4-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/*.py', 'dangerzone/container_helpers/*', 'install/common/build-image.py') }}
|
||||
path: |-
|
||||
share/container.tar.gz
|
||||
share/image-id.txt
|
||||
|
|
@ -67,7 +67,6 @@ jobs:
|
|||
- name: Build Dangerzone container image
|
||||
if: ${{ steps.cache-container-image.outputs.cache-hit != 'true' }}
|
||||
run: |
|
||||
sudo apt-get install -y python3-poetry
|
||||
python3 ./install/common/build-image.py
|
||||
|
||||
- name: Upload container image
|
||||
|
|
@ -227,7 +226,7 @@ jobs:
|
|||
- name: Restore container cache
|
||||
uses: actions/cache/restore@v4
|
||||
with:
|
||||
key: v3-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/*.py', 'dangerzone/container_helpers/*', 'install/common/build-image.py', 'poetry.lock') }}
|
||||
key: v4-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/*.py', 'dangerzone/container_helpers/*', 'install/common/build-image.py') }}
|
||||
path: |-
|
||||
share/container.tar.gz
|
||||
share/image-id.txt
|
||||
|
|
@ -334,7 +333,7 @@ jobs:
|
|||
- name: Restore container image
|
||||
uses: actions/cache/restore@v4
|
||||
with:
|
||||
key: v3-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/*.py', 'dangerzone/container_helpers/*', 'install/common/build-image.py', 'poetry.lock') }}
|
||||
key: v4-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/*.py', 'dangerzone/container_helpers/*', 'install/common/build-image.py') }}
|
||||
path: |-
|
||||
share/container.tar.gz
|
||||
share/image-id.txt
|
||||
|
|
@ -429,7 +428,7 @@ jobs:
|
|||
- name: Restore container image
|
||||
uses: actions/cache/restore@v4
|
||||
with:
|
||||
key: v3-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/*.py', 'dangerzone/container_helpers/*', 'install/common/build-image.py', 'poetry.lock') }}
|
||||
key: v4-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/*.py', 'dangerzone/container_helpers/*', 'install/common/build-image.py') }}
|
||||
path: |-
|
||||
share/container.tar.gz
|
||||
share/image-id.txt
|
||||
|
|
|
|||
1
dodo.py
1
dodo.py
|
|
@ -63,7 +63,6 @@ TESSDATA_TARGETS = list_language_data()
|
|||
|
||||
IMAGE_DEPS = [
|
||||
"Dockerfile",
|
||||
"poetry.lock",
|
||||
*list_files("dangerzone/conversion"),
|
||||
*list_files("dangerzone/container_helpers"),
|
||||
"install/common/build-image.py",
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
import argparse
|
||||
import gzip
|
||||
import os
|
||||
import platform
|
||||
import secrets
|
||||
import subprocess
|
||||
|
|
@ -9,7 +8,6 @@ from pathlib import Path
|
|||
|
||||
BUILD_CONTEXT = "dangerzone/"
|
||||
IMAGE_NAME = "dangerzone.rocks/dangerzone"
|
||||
REQUIREMENTS_TXT = "container-pip-requirements.txt"
|
||||
if platform.system() in ["Darwin", "Windows"]:
|
||||
CONTAINER_RUNTIME = "docker"
|
||||
elif platform.system() == "Linux":
|
||||
|
|
@ -84,91 +82,48 @@ def main():
|
|||
with open(image_id_path, "w") as f:
|
||||
f.write(tag)
|
||||
|
||||
print("Exporting container pip dependencies")
|
||||
with ContainerPipDependencies():
|
||||
if not args.use_cache:
|
||||
print("Pulling base image")
|
||||
subprocess.run(
|
||||
[
|
||||
args.runtime,
|
||||
"pull",
|
||||
"alpine:latest",
|
||||
],
|
||||
check=True,
|
||||
)
|
||||
# Build the container image, and tag it with the calculated tag
|
||||
print("Building container image")
|
||||
cache_args = [] if args.use_cache else ["--no-cache"]
|
||||
subprocess.run(
|
||||
[
|
||||
args.runtime,
|
||||
"build",
|
||||
BUILD_CONTEXT,
|
||||
*cache_args,
|
||||
"-f",
|
||||
"Dockerfile",
|
||||
"--tag",
|
||||
image_name_tagged,
|
||||
],
|
||||
check=True,
|
||||
)
|
||||
|
||||
# Build the container image, and tag it with the calculated tag
|
||||
print("Building container image")
|
||||
cache_args = [] if args.use_cache else ["--no-cache"]
|
||||
subprocess.run(
|
||||
if not args.no_save:
|
||||
print("Saving container image")
|
||||
cmd = subprocess.Popen(
|
||||
[
|
||||
args.runtime,
|
||||
"build",
|
||||
BUILD_CONTEXT,
|
||||
*cache_args,
|
||||
"--build-arg",
|
||||
f"REQUIREMENTS_TXT={REQUIREMENTS_TXT}",
|
||||
"--build-arg",
|
||||
f"ARCH={ARCH}",
|
||||
"-f",
|
||||
"Dockerfile",
|
||||
"--tag",
|
||||
CONTAINER_RUNTIME,
|
||||
"save",
|
||||
image_name_tagged,
|
||||
],
|
||||
check=True,
|
||||
stdout=subprocess.PIPE,
|
||||
)
|
||||
|
||||
if not args.no_save:
|
||||
print("Saving container image")
|
||||
cmd = subprocess.Popen(
|
||||
[
|
||||
CONTAINER_RUNTIME,
|
||||
"save",
|
||||
image_name_tagged,
|
||||
],
|
||||
stdout=subprocess.PIPE,
|
||||
)
|
||||
|
||||
print("Compressing container image")
|
||||
chunk_size = 4 << 20
|
||||
with gzip.open(
|
||||
tarball_path,
|
||||
"wb",
|
||||
compresslevel=args.compress_level,
|
||||
) as gzip_f:
|
||||
while True:
|
||||
chunk = cmd.stdout.read(chunk_size)
|
||||
if len(chunk) > 0:
|
||||
gzip_f.write(chunk)
|
||||
else:
|
||||
break
|
||||
cmd.wait(5)
|
||||
|
||||
|
||||
class ContainerPipDependencies:
|
||||
"""Generates PIP dependencies within container"""
|
||||
|
||||
def __enter__(self):
|
||||
try:
|
||||
container_requirements_txt = subprocess.check_output(
|
||||
["poetry", "export", "--only", "container"], universal_newlines=True
|
||||
)
|
||||
except subprocess.CalledProcessError as e:
|
||||
print("FAILURE", e.returncode, e.output)
|
||||
print(f"REQUIREMENTS: {container_requirements_txt}")
|
||||
# XXX Export container dependencies and exclude pymupdfb since it is not needed in container
|
||||
req_txt_pymupdfb_stripped = container_requirements_txt.split("pymupdfb")[0]
|
||||
with open(Path(BUILD_CONTEXT) / REQUIREMENTS_TXT, "w") as f:
|
||||
if ARCH == "arm64":
|
||||
# PyMuPDF needs to be built on ARM64 machines
|
||||
# But is already provided as a prebuilt-wheel on other architectures
|
||||
f.write(req_txt_pymupdfb_stripped)
|
||||
else:
|
||||
f.write(container_requirements_txt)
|
||||
|
||||
def __exit__(self, exc_type, exc_value, exc_tb):
|
||||
print("Leaving the context...")
|
||||
os.remove(Path(BUILD_CONTEXT) / REQUIREMENTS_TXT)
|
||||
print("Compressing container image")
|
||||
chunk_size = 4 << 20
|
||||
with gzip.open(
|
||||
tarball_path,
|
||||
"wb",
|
||||
compresslevel=args.compress_level,
|
||||
) as gzip_f:
|
||||
while True:
|
||||
chunk = cmd.stdout.read(chunk_size)
|
||||
if len(chunk) > 0:
|
||||
gzip_f.write(chunk)
|
||||
else:
|
||||
break
|
||||
cmd.wait(5)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ def main():
|
|||
)
|
||||
|
||||
logger.info("Getting PyMuPDF deps as requirements.txt")
|
||||
cmd = ["poetry", "export", "--only", "container"]
|
||||
cmd = ["poetry", "export", "--only", "debian"]
|
||||
container_requirements_txt = subprocess.check_output(cmd)
|
||||
|
||||
# XXX: Hack for Ubuntu Focal.
|
||||
|
|
|
|||
14
poetry.lock
generated
14
poetry.lock
generated
|
|
@ -767,6 +767,9 @@ optional = false
|
|||
python-versions = "<3.14,>=3.9"
|
||||
files = [
|
||||
{file = "PySide6-6.8.1.1-cp39-abi3-macosx_12_0_universal2.whl", hash = "sha256:805728a7ed58352a02689b953ddbe29af1c8944f8c7f2c28312dc0b69f64b85e"},
|
||||
{file = "PySide6-6.8.1.1-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:70f8c4745d981ebb5bb93d7b825222532d553373b68b9db7a42cfcee25cafc9a"},
|
||||
{file = "PySide6-6.8.1.1-cp39-abi3-manylinux_2_39_aarch64.whl", hash = "sha256:de80ac62087a716b2bada2e3ddd739c5d176dc4be819abef91274d53d75f4e58"},
|
||||
{file = "PySide6-6.8.1.1-cp39-abi3-win_amd64.whl", hash = "sha256:60a2551053fa69845b893fb821507e2cc89d3a8a8b43726d568acd1250ad44fb"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
|
|
@ -799,6 +802,9 @@ optional = false
|
|||
python-versions = "<3.14,>=3.9"
|
||||
files = [
|
||||
{file = "PySide6_Addons-6.8.1.1-cp39-abi3-macosx_12_0_universal2.whl", hash = "sha256:83d35d7a1a7dbd1a16b4040a26ad4d5cc030a2aed4d439241babee1225d6e58a"},
|
||||
{file = "PySide6_Addons-6.8.1.1-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:5ef45aeadca37d658e44a41e11f2b2e43dfc34c780a6be1cd09d96a7696e6cc6"},
|
||||
{file = "PySide6_Addons-6.8.1.1-cp39-abi3-manylinux_2_39_aarch64.whl", hash = "sha256:e1b4a20b0bcbc2e440faba62e0d164223b8fd6f041d749543bc3812979116c4c"},
|
||||
{file = "PySide6_Addons-6.8.1.1-cp39-abi3-win_amd64.whl", hash = "sha256:d8ae86944ac48cc9891666cf71565acebd403a953d0e050be4d41ac490788d0a"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
|
|
@ -829,6 +835,9 @@ optional = false
|
|||
python-versions = "<3.14,>=3.9"
|
||||
files = [
|
||||
{file = "PySide6_Essentials-6.8.1.1-cp39-abi3-macosx_12_0_universal2.whl", hash = "sha256:25f3fdb281ac3b442f08250e3284d3b1944f7c64c62ed93b57678a62c199cf46"},
|
||||
{file = "PySide6_Essentials-6.8.1.1-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:62b64842a91114c224c41eeb6a8c8f255ba60268bc5ac19724f944d60e2277c6"},
|
||||
{file = "PySide6_Essentials-6.8.1.1-cp39-abi3-manylinux_2_39_aarch64.whl", hash = "sha256:e0c1cc3cfb2ea5eea70748da7d22032a59ea641e24988f543d5b274c0adab065"},
|
||||
{file = "PySide6_Essentials-6.8.1.1-cp39-abi3-win_amd64.whl", hash = "sha256:085f12e16db31eb0e802b21c64eabf582f54db6c44463a1f5e1814d897b1f2c0"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
|
|
@ -1071,6 +1080,9 @@ optional = false
|
|||
python-versions = "<3.14,>=3.9"
|
||||
files = [
|
||||
{file = "shiboken6-6.8.1.1-cp39-abi3-macosx_12_0_universal2.whl", hash = "sha256:42fbb173a772c4e059dbeafb302e96f6ea8e1c9bacf05fab71ea7eb0d8f97b01"},
|
||||
{file = "shiboken6-6.8.1.1-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:d672df0f29dc5f44de7205c1acae4d0471ba8371bb1d68fdacbf1686f4d22a96"},
|
||||
{file = "shiboken6-6.8.1.1-cp39-abi3-manylinux_2_39_aarch64.whl", hash = "sha256:ff1b22a66476b042d3dc09870edca353fdac1c1f517a4cdc364b24e296213ecd"},
|
||||
{file = "shiboken6-6.8.1.1-cp39-abi3-win_amd64.whl", hash = "sha256:26f7041c77058a8ecfb9345caa187250b199de79cfb37e33936e5fbd468a7780"},
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1270,4 +1282,4 @@ type = ["pytest-mypy"]
|
|||
[metadata]
|
||||
lock-version = "2.0"
|
||||
python-versions = ">=3.9,<3.13"
|
||||
content-hash = "30751b4a27cebd020b5222de2abef4e38fb5f6676fdf936693060e42cd35c25f"
|
||||
content-hash = "035acbf0d18e9eac2f2c338c56b55ea84cfb0c59db64ac014c55607c2ea7cfb2"
|
||||
|
|
|
|||
|
|
@ -64,7 +64,7 @@ strip-ansi = "*"
|
|||
pytest-subprocess = "^1.5.2"
|
||||
pytest-rerunfailures = "^14.0"
|
||||
|
||||
[tool.poetry.group.container.dependencies]
|
||||
[tool.poetry.group.debian.dependencies]
|
||||
pymupdf = "1.24.11" # Last version to support python 3.8 (needed for Ubuntu Focal support)
|
||||
|
||||
[tool.poetry.group.dev.dependencies]
|
||||
|
|
|
|||
Loading…
Reference in a new issue