From 96e64deae750d25769af7ff42e72854c9962151c Mon Sep 17 00:00:00 2001 From: Alex Pyrgiotis Date: Mon, 2 Dec 2024 18:12:16 +0200 Subject: [PATCH 01/12] Move container-specific method from base class Move the `is_runtime_available()` method from the base `IsolationProvider` class, and into the `Dummy` provider class. This method was originally defined in the base class, in order to be mocked in our tests for the `Dummy` provider. There's no reason for the `Qubes` class to have it though, so we can just move it to the `Dummy` provider. --- dangerzone/gui/main_window.py | 1 + dangerzone/isolation_provider/base.py | 4 ---- dangerzone/isolation_provider/dummy.py | 4 ++++ tests/gui/test_main_window.py | 7 ++++--- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/dangerzone/gui/main_window.py b/dangerzone/gui/main_window.py index e292ff0..788ad6a 100644 --- a/dangerzone/gui/main_window.py +++ b/dangerzone/gui/main_window.py @@ -500,6 +500,7 @@ class WaitingWidgetContainer(WaitingWidget): error: Optional[str] = None try: + assert isinstance(self.dangerzone.isolation_provider, (Dummy, Container)) self.dangerzone.isolation_provider.is_runtime_available() except NoContainerTechException as e: log.error(str(e)) diff --git a/dangerzone/isolation_provider/base.py b/dangerzone/isolation_provider/base.py index 6a55a20..9404cee 100644 --- a/dangerzone/isolation_provider/base.py +++ b/dangerzone/isolation_provider/base.py @@ -93,10 +93,6 @@ class IsolationProvider(ABC): else: self.proc_stderr = subprocess.DEVNULL - @staticmethod - def is_runtime_available() -> bool: - return True - @abstractmethod def install(self) -> bool: pass diff --git a/dangerzone/isolation_provider/dummy.py b/dangerzone/isolation_provider/dummy.py index 9ebc345..b8e3b87 100644 --- a/dangerzone/isolation_provider/dummy.py +++ b/dangerzone/isolation_provider/dummy.py @@ -39,6 +39,10 @@ class Dummy(IsolationProvider): def install(self) -> bool: return True + @staticmethod + def is_runtime_available() -> bool: + return True + def start_doc_to_pixels_proc(self, document: Document) -> subprocess.Popen: cmd = [ sys.executable, diff --git a/tests/gui/test_main_window.py b/tests/gui/test_main_window.py index ff45075..7e96d22 100644 --- a/tests/gui/test_main_window.py +++ b/tests/gui/test_main_window.py @@ -30,6 +30,7 @@ from dangerzone.isolation_provider.container import ( NoContainerTechException, NotAvailableContainerTechException, ) +from dangerzone.isolation_provider.dummy import Dummy from .test_updater import assert_report_equal, default_updater_settings @@ -510,9 +511,9 @@ def test_not_available_container_tech_exception( ) -> None: # Setup mock_app = mocker.MagicMock() - dummy = mocker.MagicMock() - - dummy.is_runtime_available.side_effect = NotAvailableContainerTechException( + dummy = Dummy() + fn = mocker.patch.object(dummy, "is_runtime_available") + fn.side_effect = NotAvailableContainerTechException( "podman", "podman image ls logs" ) From f31fbfefc66375becbcb1d4766e461c482484b3f Mon Sep 17 00:00:00 2001 From: Alex Pyrgiotis Date: Mon, 2 Dec 2024 16:12:56 +0200 Subject: [PATCH 02/12] container: Manipulate Dangerzone image tags Add the following methods that allow the `Container` isolation provider to work with tags for the Dangerzone image: * `list_image_tag()` * `delete_image_tag()` * `add_image_tag()` --- dangerzone/isolation_provider/container.py | 70 +++++++++++++++++++++- 1 file changed, 69 insertions(+), 1 deletion(-) diff --git a/dangerzone/isolation_provider/container.py b/dangerzone/isolation_provider/container.py index 94f894d..3f855e0 100644 --- a/dangerzone/isolation_provider/container.py +++ b/dangerzone/isolation_provider/container.py @@ -1,11 +1,12 @@ import gzip +import json import logging import os import platform import shlex import shutil import subprocess -from typing import List, Tuple +from typing import Dict, List, Tuple from ..document import Document from ..util import get_resource_path, get_subprocess_startupinfo @@ -154,6 +155,73 @@ class Container(IsolationProvider): return security_args + @staticmethod + def list_image_tags() -> Dict[str, str]: + """Get the tags of all loaded Dangerzone images. + + This method returns a mapping of image tags to image IDs, for all Dangerzone + images. This can be useful when we want to find which are the local image tags, + and which image ID does the "latest" tag point to. + """ + images = json.loads( + subprocess.check_output( + [ + Container.get_runtime(), + "image", + "list", + "--format", + "json", + Container.CONTAINER_NAME, + ], + text=True, + startupinfo=get_subprocess_startupinfo(), + ) + ) + + # Grab every image name and associate it with an image ID. + tags = {} + for image in images: + for name in image["Names"]: + tag = name.split(":")[1] + tags[tag] = image["Id"] + + return tags + + @staticmethod + def delete_image_tag(tag: str) -> None: + """Delete a Dangerzone image tag.""" + name = Container.CONTAINER_NAME + ":" + tag + log.warning(f"Deleting old container image: {name}") + try: + subprocess.check_output( + [Container.get_runtime(), "rmi", "--force", name], + startupinfo=get_subprocess_startupinfo(), + ) + except Exception as e: + log.warning( + f"Couldn't delete old container image '{name}', so leaving it there." + f" Original error: {e}" + ) + + @staticmethod + def add_image_tag(cur_tag: str, new_tag: str) -> None: + """Add a tag to an existing Dangerzone image.""" + cur_image_name = Container.CONTAINER_NAME + ":" + cur_tag + new_image_name = Container.CONTAINER_NAME + ":" + new_tag + subprocess.check_output( + [ + Container.get_runtime(), + "tag", + cur_image_name, + new_image_name, + ], + startupinfo=get_subprocess_startupinfo(), + ) + + log.info( + f"Successfully tagged container image '{cur_image_name}' as '{new_image_name}'" + ) + @staticmethod def install() -> bool: """ From 7f7fe43711d3bed3e640d58d125279d944e80585 Mon Sep 17 00:00:00 2001 From: Alex Pyrgiotis Date: Mon, 2 Dec 2024 16:30:43 +0200 Subject: [PATCH 03/12] container: Factor out loading an image tarball --- dangerzone/isolation_provider/container.py | 26 +++++++++++++++------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/dangerzone/isolation_provider/container.py b/dangerzone/isolation_provider/container.py index 3f855e0..0954a53 100644 --- a/dangerzone/isolation_provider/container.py +++ b/dangerzone/isolation_provider/container.py @@ -223,16 +223,14 @@ class Container(IsolationProvider): ) @staticmethod - def install() -> bool: - """ - Make sure the podman container is installed. Linux only. - """ - if Container.is_container_installed(): - return True + def get_expected_tag() -> str: + """Get the tag of the Dangerzone image tarball from the image-id.txt file.""" + with open(get_resource_path("image-id.txt")) as f: + return f.read.strip() - # Load the container into podman + @staticmethod + def load_image_tarball() -> None: log.info("Installing Dangerzone container image...") - p = subprocess.Popen( [Container.get_runtime(), "load"], stdin=subprocess.PIPE, @@ -259,6 +257,18 @@ class Container(IsolationProvider): f"Could not install container image: {error}" ) + log.info("Successfully installed container image from") + + @staticmethod + def install() -> bool: + """ + Make sure the podman container is installed. Linux only. + """ + if Container.is_container_installed(): + return True + + Container.load_image_tarball() + if not Container.is_container_installed(raise_on_error=True): return False From 53214d33d8fc9b59f61d07c971ae8aef50454018 Mon Sep 17 00:00:00 2001 From: Alex Pyrgiotis Date: Mon, 2 Dec 2024 16:51:26 +0200 Subject: [PATCH 04/12] Build and tag Dangerzone images Build Dangerzone images and tag them with a unique ID that stems from the Git reop. Note that using tags as image IDs instead of regular image IDs breaks the current Dangerzone expectations, but this will be addressed in subsequent commits. --- .github/workflows/build.yml | 2 ++ .github/workflows/ci.yml | 4 ++- .github/workflows/scan.yml | 2 ++ install/common/build-image.py | 52 ++++++++++++++++++++++------------- 4 files changed, 40 insertions(+), 20 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 92440e7..1a270a8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -74,6 +74,8 @@ jobs: runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 + with: + fetch-depth: 0 - name: Get current date id: date diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5c72dd1..e694f31 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,6 +48,8 @@ jobs: runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 + with: + fetch-depth: 0 - name: Get current date id: date @@ -245,7 +247,7 @@ jobs: install-deb: name: "install-deb (${{ matrix.distro }} ${{ matrix.version }})" runs-on: ubuntu-latest - needs: + needs: - build-deb strategy: matrix: diff --git a/.github/workflows/scan.yml b/.github/workflows/scan.yml index d9f397b..3080476 100644 --- a/.github/workflows/scan.yml +++ b/.github/workflows/scan.yml @@ -14,6 +14,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 + with: + fetch-depth: 0 - name: Install container build dependencies run: sudo apt install pipx && pipx install poetry - name: Build container image diff --git a/install/common/build-image.py b/install/common/build-image.py index 9f2dcc8..921a520 100644 --- a/install/common/build-image.py +++ b/install/common/build-image.py @@ -2,12 +2,13 @@ import argparse import gzip import os import platform +import secrets import subprocess import sys from pathlib import Path BUILD_CONTEXT = "dangerzone/" -TAG = "dangerzone.rocks/dangerzone:latest" +IMAGE_NAME = "dangerzone.rocks/dangerzone" REQUIREMENTS_TXT = "container-pip-requirements.txt" if platform.system() in ["Darwin", "Windows"]: CONTAINER_RUNTIME = "docker" @@ -44,8 +45,31 @@ def main(): ) args = parser.parse_args() + tarball_path = Path("share") / "container.tar.gz" + image_id_path = Path("share") / "image-id.txt" + print(f"Building for architecture '{ARCH}'") + # Designate a unique tag for this image, depending on the Git commit it was created + # from: + # 1. If created from a Git tag (e.g., 0.8.0), the image tag will be `0.8.0`. + # 2. If created from a commit, it will be something like `0.8.0-31-g6bdaa7a`. + # 3. If the contents of the Git repo are dirty, we will append a unique identifier + # for this run, something like `0.8.0-31-g6bdaa7a-fdcb` or `0.8.0-fdcb`. + dirty_ident = secrets.token_hex(2) + tag = ( + subprocess.check_output( + ["git", "describe", "--first-parent", f"--dirty=-{dirty_ident}"], + ) + .decode() + .strip()[1:] # remove the "v" prefix of the tag. + ) + image_name_tagged = IMAGE_NAME + ":" + tag + + print(f"Will tag the container image as '{image_name_tagged}'") + with open(image_id_path, "w") as f: + f.write(tag) + print("Exporting container pip dependencies") with ContainerPipDependencies(): if not args.use_cache: @@ -59,8 +83,11 @@ def main(): check=True, ) + # Build the container image, and tag it with two tags; the one we calculated + # above, and the "latest" tag. print("Building container image") cache_args = [] if args.use_cache else ["--no-cache"] + image_name_latest = IMAGE_NAME + ":latest" subprocess.run( [ args.runtime, @@ -74,7 +101,9 @@ def main(): "-f", "Dockerfile", "--tag", - TAG, + image_name_latest, + "--tag", + image_name_tagged, ], check=True, ) @@ -85,7 +114,7 @@ def main(): [ CONTAINER_RUNTIME, "save", - TAG, + image_name_tagged, ], stdout=subprocess.PIPE, ) @@ -93,7 +122,7 @@ def main(): print("Compressing container image") chunk_size = 4 << 20 with gzip.open( - "share/container.tar.gz", + tarball_path, "wb", compresslevel=args.compress_level, ) as gzip_f: @@ -105,21 +134,6 @@ def main(): break cmd.wait(5) - print("Looking up the image id") - image_id = subprocess.check_output( - [ - args.runtime, - "image", - "list", - "--format", - "{{.ID}}", - TAG, - ], - text=True, - ) - with open("share/image-id.txt", "w") as f: - f.write(image_id) - class ContainerPipDependencies: """Generates PIP dependencies within container""" From 5b1fe4d7ad12876ddf187efdaf04e24f05bdb844 Mon Sep 17 00:00:00 2001 From: Alex Pyrgiotis Date: Mon, 2 Dec 2024 17:05:38 +0200 Subject: [PATCH 05/12] container: Revamp container image installation Revamp the container image installation process in a way that does not involve using image IDs. We don't want to rely on image IDs anymore, since they are brittle (see https://github.com/freedomofpress/dangerzone/issues/933). Instead, we use image tags, as provided in the `image-id.txt` file. This allows us to check fast if an image is up to date, and we no longer need to maintain multiple image IDs from various container runtimes. Refs #933 Refs #988 Fixes #1020 --- dangerzone/isolation_provider/container.py | 98 +++++++++------------- tests/isolation_provider/test_container.py | 6 +- 2 files changed, 43 insertions(+), 61 deletions(-) diff --git a/dangerzone/isolation_provider/container.py b/dangerzone/isolation_provider/container.py index 0954a53..36376de 100644 --- a/dangerzone/isolation_provider/container.py +++ b/dangerzone/isolation_provider/container.py @@ -226,7 +226,7 @@ class Container(IsolationProvider): def get_expected_tag() -> str: """Get the tag of the Dangerzone image tarball from the image-id.txt file.""" with open(get_resource_path("image-id.txt")) as f: - return f.read.strip() + return f.read().strip() @staticmethod def load_image_tarball() -> None: @@ -261,18 +261,50 @@ class Container(IsolationProvider): @staticmethod def install() -> bool: + """Install the container image tarball, or verify that it's already installed. + + Perform the following actions: + 1. Get the tags of any locally available images that match Dangerzone's image + name. + 2. Get the expected image tag from the image-id.txt file. + - If this tag is present in the local images, and that image is also tagged + as "latest", then we can return. + - Else, prune the older container images and continue. + 3. Load the image tarball and make sure it matches the expected tag. + 4. Tag that image as "latest", and mark the installation as finished. """ - Make sure the podman container is installed. Linux only. - """ - if Container.is_container_installed(): + old_tags = Container.list_image_tags() + expected_tag = Container.get_expected_tag() + + if expected_tag not in old_tags: + # Prune older container images. + log.info( + f"Could not find a Dangerzone container image with tag '{expected_tag}'" + ) + for tag in old_tags.keys(): + Container.delete_image_tag(tag) + elif old_tags[expected_tag] != old_tags.get("latest"): + log.info(f"The expected tag '{expected_tag}' is not the latest one") + Container.add_image_tag(expected_tag, "latest") + return True + else: return True + # Load the image tarball into the container runtime. Container.load_image_tarball() - if not Container.is_container_installed(raise_on_error=True): - return False + # Check that the container image has the expected image tag. + # See https://github.com/freedomofpress/dangerzone/issues/988 for an example + # where this was not the case. + new_tags = Container.list_image_tags() + if expected_tag not in new_tags: + raise ImageNotPresentException( + f"Could not find expected tag '{expected_tag}' after loading the" + " container image tarball" + ) - log.info("Container image installed") + # Mark the expected tag as "latest". + Container.add_image_tag(expected_tag, "latest") return True @staticmethod @@ -291,58 +323,6 @@ class Container(IsolationProvider): raise NotAvailableContainerTechException(runtime_name, stderr.decode()) return True - @staticmethod - def is_container_installed(raise_on_error: bool = False) -> bool: - """ - See if the container is installed. - """ - # Get the image id - with open(get_resource_path("image-id.txt")) as f: - expected_image_ids = f.read().strip().split() - - # See if this image is already installed - installed = False - found_image_id = subprocess.check_output( - [ - Container.get_runtime(), - "image", - "list", - "--format", - "{{.ID}}", - Container.CONTAINER_NAME, - ], - text=True, - startupinfo=get_subprocess_startupinfo(), - ) - found_image_id = found_image_id.strip() - - if found_image_id in expected_image_ids: - installed = True - elif found_image_id == "": - if raise_on_error: - raise ImageNotPresentException( - "Image is not listed after installation. Bailing out." - ) - else: - msg = ( - f"{Container.CONTAINER_NAME} images found, but IDs do not match." - f" Found: {found_image_id}, Expected: {','.join(expected_image_ids)}" - ) - if raise_on_error: - raise ImageNotPresentException(msg) - log.info(msg) - log.info("Deleting old dangerzone container image") - - try: - subprocess.check_output( - [Container.get_runtime(), "rmi", "--force", found_image_id], - startupinfo=get_subprocess_startupinfo(), - ) - except Exception: - log.warning("Couldn't delete old container image, so leaving it there") - - return installed - def doc_to_pixels_container_name(self, document: Document) -> str: """Unique container name for the doc-to-pixels phase.""" return f"dangerzone-doc-to-pixels-{document.id}" diff --git a/tests/isolation_provider/test_container.py b/tests/isolation_provider/test_container.py index 3fb3243..a1a844d 100644 --- a/tests/isolation_provider/test_container.py +++ b/tests/isolation_provider/test_container.py @@ -69,10 +69,11 @@ class TestContainer(IsolationProviderTest): "image", "list", "--format", - "{{.ID}}", + "json", "dangerzone.rocks/dangerzone", ], occurrences=2, + stdout="{}", ) # Make podman load fail @@ -102,10 +103,11 @@ class TestContainer(IsolationProviderTest): "image", "list", "--format", - "{{.ID}}", + "json", "dangerzone.rocks/dangerzone", ], occurrences=2, + stdout="{}", ) # Patch gzip.open and podman load so that it works From f400205c74c1c4b83496d462fd69524f40c7b549 Mon Sep 17 00:00:00 2001 From: Alex Pyrgiotis Date: Mon, 2 Dec 2024 18:38:33 +0200 Subject: [PATCH 06/12] Update our release instructions --- QA.md | 6 ++++-- RELEASE.md | 4 ---- dev_scripts/qa.py | 6 ++++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/QA.md b/QA.md index dff7780..5302f92 100644 --- a/QA.md +++ b/QA.md @@ -107,9 +107,10 @@ Close the Dangerzone application and get the container image for that version. For example: ``` -$ docker images dangerzone.rocks/dangerzone:latest +$ docker images dangerzone.rocks/dangerzone REPOSITORY TAG IMAGE ID CREATED SIZE dangerzone.rocks/dangerzone latest +dangerzone.rocks/dangerzone ``` Then run the version under QA and ensure that the settings remain changed. @@ -118,9 +119,10 @@ Afterwards check that new docker image was installed by running the same command and seeing the following differences: ``` -$ docker images dangerzone.rocks/dangerzone:latest +$ docker images dangerzone.rocks/dangerzone REPOSITORY TAG IMAGE ID CREATED SIZE dangerzone.rocks/dangerzone latest +dangerzone.rocks/dangerzone ``` #### 4. Dangerzone successfully installs the container image diff --git a/RELEASE.md b/RELEASE.md index 8d4c0ba..e13012c 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -142,7 +142,6 @@ Here is what you need to do: poetry run ./install/macos/build-app.py ``` -- [ ] Make sure that the build application works with the containerd graph driver (see [#933](https://github.com/freedomofpress/dangerzone/issues/933)) - [ ] Sign the application bundle, and notarize it @@ -212,9 +211,6 @@ The Windows release is performed in a Windows 11 virtual machine (as opposed to - [ ] Copy the container image into the VM > [!IMPORTANT] > Instead of running `python .\install\windows\build-image.py` in the VM, run the build image script on the host (making sure to build for `linux/amd64`). Copy `share/container.tar.gz` and `share/image-id.txt` from the host into the `share` folder in the VM. - > Also, don't forget to add the supplementary image ID (see - > [#933](https://github.com/freedomofpress/dangerzone/issues/933)) in - > `share/image-id.txt`) - [ ] Run `poetry run .\install\windows\build-app.bat` - [ ] When you're done you will have `dist\Dangerzone.msi` diff --git a/dev_scripts/qa.py b/dev_scripts/qa.py index 5039bbd..ebd6099 100755 --- a/dev_scripts/qa.py +++ b/dev_scripts/qa.py @@ -127,9 +127,10 @@ Close the Dangerzone application and get the container image for that version. For example: ``` -$ docker images dangerzone.rocks/dangerzone:latest +$ docker images dangerzone.rocks/dangerzone REPOSITORY TAG IMAGE ID CREATED SIZE dangerzone.rocks/dangerzone latest +dangerzone.rocks/dangerzone ``` Then run the version under QA and ensure that the settings remain changed. @@ -138,9 +139,10 @@ Afterwards check that new docker image was installed by running the same command and seeing the following differences: ``` -$ docker images dangerzone.rocks/dangerzone:latest +$ docker images dangerzone.rocks/dangerzone REPOSITORY TAG IMAGE ID CREATED SIZE dangerzone.rocks/dangerzone latest +dangerzone.rocks/dangerzone ``` #### 4. Dangerzone successfully installs the container image From 02261b112e58132f44eeba66a890a6cb4567d560 Mon Sep 17 00:00:00 2001 From: Alex Pyrgiotis Date: Wed, 4 Dec 2024 13:59:19 +0200 Subject: [PATCH 07/12] Fix some small typos in our release docs --- RELEASE.md | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/RELEASE.md b/RELEASE.md index e13012c..9a1e89f 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -7,7 +7,11 @@ This section documents how we currently release Dangerzone for the different dis Here is a list of tasks that should be done before issuing the release: - [ ] Create a new issue named **QA and Release for version \**, to track the general progress. - You can generate its content with the the `poetry run ./dev_scripts/generate-release-tasks.py` command. + You can generate its content with: + + ``` + poetry run ./dev_scripts/generate-release-tasks.py` + ``` - [ ] [Add new Linux platforms and remove obsolete ones](https://github.com/freedomofpress/dangerzone/blob/main/RELEASE.md#add-new-platforms-and-remove-obsolete-ones) - [ ] Bump the Python dependencies using `poetry lock` - [ ] Update `version` in `pyproject.toml` @@ -126,7 +130,7 @@ Here is what you need to do: ``` - [ ] Build the container image and the OCR language data - + ```bash poetry run ./install/common/build-image.py poetry run ./install/common/download-tessdata.py @@ -142,11 +146,10 @@ Here is what you need to do: poetry run ./install/macos/build-app.py ``` - driver (see [#933](https://github.com/freedomofpress/dangerzone/issues/933)) - [ ] Sign the application bundle, and notarize it - + You need to run this command as the account that has access to the code signing certificate - + This command assumes that you have created, and stored in the Keychain, an application password associated with your Apple Developer ID, which will be used specifically for `notarytool`. From eec4e6a5c3db3d808956b9a87fff6284f0149720 Mon Sep 17 00:00:00 2001 From: Alex Pyrgiotis Date: Wed, 4 Dec 2024 13:30:56 +0200 Subject: [PATCH 08/12] Allow passing true/false to --use-cache build arg --- install/common/build-image.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/install/common/build-image.py b/install/common/build-image.py index 921a520..a49878f 100644 --- a/install/common/build-image.py +++ b/install/common/build-image.py @@ -18,6 +18,17 @@ elif platform.system() == "Linux": ARCH = platform.machine() +def str2bool(v): + if isinstance(v, bool): + return v + if v.lower() in ("yes", "true", "t", "y", "1"): + return True + elif v.lower() in ("no", "false", "f", "n", "0"): + return False + else: + raise argparse.ArgumentTypeError("Boolean value expected.") + + def main(): parser = argparse.ArgumentParser() parser.add_argument( @@ -40,7 +51,10 @@ def main(): ) parser.add_argument( "--use-cache", - action="store_true", + type=str2bool, + nargs="?", + default=False, + const=True, help="Use the builder's cache to speed up the builds (not suitable for release builds)", ) args = parser.parse_args() From ece58cba06952c806a3ff9b125f002089590b107 Mon Sep 17 00:00:00 2001 From: Alex Pyrgiotis Date: Wed, 4 Dec 2024 13:33:03 +0200 Subject: [PATCH 09/12] Add doit in Poetry as package dependency Add the doit automation tool in our `pyproject.toml` and `poetry.lock` file as a package-related dependency, since we don't want to ship it to our end users. --- poetry.lock | 39 ++++++++++++++++++++++++++++++--------- pyproject.toml | 1 + 2 files changed, 31 insertions(+), 9 deletions(-) diff --git a/poetry.lock b/poetry.lock index 43be666..e3bae66 100644 --- a/poetry.lock +++ b/poetry.lock @@ -229,6 +229,17 @@ files = [ [package.dependencies] colorama = {version = "*", markers = "platform_system == \"Windows\""} +[[package]] +name = "cloudpickle" +version = "3.1.0" +description = "Pickler class to extend the standard pickle.Pickler functionality" +optional = false +python-versions = ">=3.8" +files = [ + {file = "cloudpickle-3.1.0-py3-none-any.whl", hash = "sha256:fe11acda67f61aaaec473e3afe030feb131d78a43461b718185363384f1ba12e"}, + {file = "cloudpickle-3.1.0.tar.gz", hash = "sha256:81a929b6e3c7335c863c771d673d105f02efdb89dfaba0c90495d1c64796601b"}, +] + [[package]] name = "colorama" version = "0.4.6" @@ -412,6 +423,24 @@ files = [ {file = "cx_logging-3.2.1.tar.gz", hash = "sha256:812665ae5012680a6fe47095c3772bce638e47cf05b2c3483db3bdbe6b06da44"}, ] +[[package]] +name = "doit" +version = "0.36.0" +description = "doit - Automation Tool" +optional = false +python-versions = ">=3.8" +files = [ + {file = "doit-0.36.0-py3-none-any.whl", hash = "sha256:ebc285f6666871b5300091c26eafdff3de968a6bd60ea35dd1e3fc6f2e32479a"}, + {file = "doit-0.36.0.tar.gz", hash = "sha256:71d07ccc9514cb22fe59d98999577665eaab57e16f644d04336ae0b4bae234bc"}, +] + +[package.dependencies] +cloudpickle = "*" +importlib-metadata = ">=4.4" + +[package.extras] +toml = ["tomli"] + [[package]] name = "exceptiongroup" version = "1.2.2" @@ -554,7 +583,6 @@ python-versions = ">=3.8" files = [ {file = "lief-0.15.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a80246b96501b2b1d4927ceb3cb817eda9333ffa9e07101358929a6cffca5dae"}, {file = "lief-0.15.1-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:84bf310710369544e2bb82f83d7fdab5b5ac422651184fde8bf9e35f14439691"}, - {file = "lief-0.15.1-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:517dc5dad31c754720a80a87ad9e6cb1e48223d4505980c2fd86072bd4f69001"}, {file = "lief-0.15.1-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:8fb58efb77358291109d2675d5459399c0794475b497992d0ecee18a4a46a207"}, {file = "lief-0.15.1-cp310-cp310-manylinux_2_33_aarch64.whl", hash = "sha256:d5852a246361bbefa4c1d5930741765a2337638d65cfe30de1b7d61f9a54b865"}, {file = "lief-0.15.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:12e53dc0253c303df386ae45487a2f0078026602b36d0e09e838ae1d4dbef958"}, @@ -562,7 +590,6 @@ files = [ {file = "lief-0.15.1-cp310-cp310-win_amd64.whl", hash = "sha256:ddf2ebd73766169594d631b35f84c49ef42871de552ad49f36002c60164d0aca"}, {file = "lief-0.15.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:20508c52de0dffcee3242253541609590167a3e56150cbacb506fdbb822206ef"}, {file = "lief-0.15.1-cp311-cp311-macosx_11_0_x86_64.whl", hash = "sha256:0750c892fd3b7161a3c2279f25fe1844427610c3a5a4ae23f65674ced6f93ea5"}, - {file = "lief-0.15.1-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:3e49bd595a8548683bead982bc15b064257fea3110fd15e22fb3feb17d97ad1c"}, {file = "lief-0.15.1-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:a8634ea79d6d9862297fadce025519ab25ff01fcadb333cf42967c6295f0d057"}, {file = "lief-0.15.1-cp311-cp311-manylinux_2_33_aarch64.whl", hash = "sha256:1e11e046ad71fe8c81e1a8d1d207fe2b99c967d33ce79c3d3915cb8f5ecacf52"}, {file = "lief-0.15.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:674b620cdf1d686f52450fd97c1056d4c92e55af8217ce85a1b2efaf5b32140b"}, @@ -570,15 +597,11 @@ files = [ {file = "lief-0.15.1-cp311-cp311-win_amd64.whl", hash = "sha256:e9b96a37bf11ca777ff305d85d957eabad2a92a6e577b6e2fb3ab79514e5a12e"}, {file = "lief-0.15.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1a96f17c2085ef38d12ad81427ae8a5d6ad76f0bc62a1e1f5fe384255cd2cc94"}, {file = "lief-0.15.1-cp312-cp312-macosx_11_0_x86_64.whl", hash = "sha256:d780af1762022b8e01b613253af490afea3864fbd6b5a49c6de7cea8fde0443d"}, - {file = "lief-0.15.1-cp312-cp312-manylinux2014_aarch64.whl", hash = "sha256:536a4ecd46b295b3acac0d60a68d1646480b7761ade862c6c87ccbb41229fae3"}, {file = "lief-0.15.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:d0f10d80202de9634a16786b53ba3a8f54ae8b9a9e124a964d83212444486087"}, {file = "lief-0.15.1-cp312-cp312-manylinux_2_33_aarch64.whl", hash = "sha256:864f17ecf1736296e6d5fc38b11983f9d19a5e799f094e21e20d58bfb1b95b80"}, {file = "lief-0.15.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c2ec738bcafee8a569741f4a749f0596823b12f10713306c7d0cbbf85759f51c"}, {file = "lief-0.15.1-cp312-cp312-win32.whl", hash = "sha256:db38619edf70e27fb3686b8c0f0bec63ad494ac88ab51660c5ecd2720b506e41"}, {file = "lief-0.15.1-cp312-cp312-win_amd64.whl", hash = "sha256:28bf0922de5fb74502a29cc47930d3a052df58dc23ab6519fa590e564f194a60"}, - {file = "lief-0.15.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0805301e8fef9b13da00c33c831fb0c05ea892309230f3a35551c2dfaf69b11d"}, - {file = "lief-0.15.1-cp313-cp313-macosx_11_0_x86_64.whl", hash = "sha256:7580defe140e921bc4f210e8a6cb115fcf2923f00d37800b1626168cbca95108"}, - {file = "lief-0.15.1-cp313-cp313-manylinux2014_aarch64.whl", hash = "sha256:c0119306b6a38759483136de7242b7c2e0a23f1de1d4ae53f12792c279607410"}, {file = "lief-0.15.1-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:0616e6048f269d262ff93d67c497ebff3c1d3965ffb9427b0f2b474764fd2e8c"}, {file = "lief-0.15.1-cp313-cp313-manylinux_2_33_aarch64.whl", hash = "sha256:6a08b2e512a80040429febddc777768c949bcd53f6f580e902e41ec0d9d936b8"}, {file = "lief-0.15.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:fcd489ff80860bcc2b2689faa330a46b6d66f0ee3e0f6ef9e643e2b996128a06"}, @@ -586,7 +609,6 @@ files = [ {file = "lief-0.15.1-cp313-cp313-win_amd64.whl", hash = "sha256:5af7dcb9c3f44baaf60875df6ba9af6777db94776cc577ee86143bcce105ba2f"}, {file = "lief-0.15.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:f9757ff0c7c3d6f66e5fdcc6a9df69680fad0dc2707d64a3428f0825dfce1a85"}, {file = "lief-0.15.1-cp38-cp38-macosx_11_0_x86_64.whl", hash = "sha256:8ac3cd099be2580d0e15150b1d2f5095c38f150af89993ddf390d7897ee8135f"}, - {file = "lief-0.15.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:e732619acc34943b504c867258fc0196f1931f72c2a627219d4f116a7acc726d"}, {file = "lief-0.15.1-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:4dedeab498c312a29b58f16b739895f65fa54b2a21b8d98b111e99ad3f7e30a8"}, {file = "lief-0.15.1-cp38-cp38-manylinux_2_33_aarch64.whl", hash = "sha256:b9217578f7a45f667503b271da8481207fb4edda8d4a53e869fb922df6030484"}, {file = "lief-0.15.1-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:82e6308ad8bd4bc7eadee3502ede13a5bb398725f25513a0396c8dba850f58a1"}, @@ -594,7 +616,6 @@ files = [ {file = "lief-0.15.1-cp38-cp38-win_amd64.whl", hash = "sha256:a079a76bca23aa73c850ab5beb7598871a1bf44662658b952cead2b5ddd31bee"}, {file = "lief-0.15.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:785a3aa14575f046ed9c8d44ea222ea14c697cd03b5331d1717b5b0cf4f72466"}, {file = "lief-0.15.1-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:d7044553cf07c8a2ab6e21874f07585610d996ff911b9af71dc6085a89f59daa"}, - {file = "lief-0.15.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:fa020f3ed6e95bb110a4316af544021b74027d18bf4671339d4cffec27aa5884"}, {file = "lief-0.15.1-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:13285c3ff5ef6de2421d85684c954905af909db0ad3472e33c475e5f0f657dcf"}, {file = "lief-0.15.1-cp39-cp39-manylinux_2_33_aarch64.whl", hash = "sha256:932f880ee8a130d663a97a9099516d8570b1b303af7816e70a02f9931d5ef4c2"}, {file = "lief-0.15.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:de9453f94866e0f2c36b6bd878625880080e7e5800788f5cbc06a76debf283b9"}, @@ -1189,4 +1210,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = ">=3.9,<3.13" -content-hash = "5d1ff28aa04c3a814280e55c0b2a307efe5ca953cd4cb281056c35fd2e53fdf0" +content-hash = "a2937fd8ead7b45da571cb943ab43918a9c6d3dcbc6935dc8d0af3d1d4190371" diff --git a/pyproject.toml b/pyproject.toml index 5acf273..c0e620a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,6 +34,7 @@ setuptools = "*" cx_freeze = {version = "^7.2.5", platform = "win32"} pywin32 = {version = "*", platform = "win32"} pyinstaller = {version = "*", platform = "darwin"} +doit = "^0.36.0" # Dependencies required for linting the code. [tool.poetry.group.lint.dependencies] From 52eae7cd00838af7829a3b6849d2c17c5296041c Mon Sep 17 00:00:00 2001 From: Alex Pyrgiotis Date: Wed, 4 Dec 2024 13:58:20 +0200 Subject: [PATCH 10/12] Automate a large portion of our release tasks Create a `dodo.py` file where we define the dependencies and targets of each release task, as well as how to run it. Currently, we have automated all of our Linux and macOS tasks, except for adding Linux packages to the respective APT/YUM repos. The tasks we have automated follow below: build_image Build the container image using ./install/common/build-image.py check_container_runtime Test that the container runtime is ready. clean_container_runtime Clean the storage space of the container runtime. clean_prompt Make sure that the user really wants to run the clean tasks. debian_deb Build a Debian package for Debian Bookworm. debian_env Build a Debian Bookworm dev environment. download_tessdata Download the Tesseract data using ./install/common/download-tessdata.py fedora_env Build Fedora dev environments. fedora_env:40 Build Fedora 40 dev environments fedora_env:41 Build Fedora 41 dev environments fedora_rpm Build Fedora packages for every supported version. fedora_rpm:40 Build a Fedora 40 package fedora_rpm:40-qubes Build a Fedora 40 package for Qubes fedora_rpm:41 Build a Fedora 41 package fedora_rpm:41-qubes Build a Fedora 41 package for Qubes git_archive Build a Git archive of the repo. init_release_dir Create a directory for release artifacts. macos_build_dmg Build the macOS .dmg file for Dangerzone. macos_check_cert Test that the Apple developer certificate can be used. macos_check_system Run macOS specific system checks, as well as the generic ones. poetry_install Setup the Poetry environment Closes #1016 --- .gitignore | 1 + CHANGELOG.md | 4 + dodo.py | 405 +++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 410 insertions(+) create mode 100644 dodo.py diff --git a/.gitignore b/.gitignore index f45a78b..db5ebd0 100644 --- a/.gitignore +++ b/.gitignore @@ -149,3 +149,4 @@ share/container.tar share/container.tar.gz share/image-id.txt container/container-pip-requirements.txt +.doit.db.db diff --git a/CHANGELOG.md b/CHANGELOG.md index 31f149b..38cb669 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,10 @@ since 0.4.1, and this project adheres to [Semantic Versioning](https://semver.or - Platform support: Drop support for Fedora 39, since it's end-of-life ([#999](https://github.com/freedomofpress/dangerzone/pull/999)) +### Development changes + +- Automate a large portion of our release tasks with `doit` ([#1016](https://github.com/freedomofpress/dangerzone/issues/1016)) + ## [0.8.0](https://github.com/freedomofpress/dangerzone/compare/v0.8.0...0.7.1) ### Added diff --git a/dodo.py b/dodo.py new file mode 100644 index 0000000..b125910 --- /dev/null +++ b/dodo.py @@ -0,0 +1,405 @@ +import json +import os +import platform +import shutil +from pathlib import Path + +from doit import get_var +from doit.action import CmdAction + +ARCH = "arm64" if platform.machine() == "arm64" else "i686" +VERSION = open("share/version.txt").read().strip() +FEDORA_VERSIONS = ["40", "41"] +DEBIAN_VERSIONS = ["bullseye", "focal", "jammy", "mantic", "noble", "trixie"] + +### Global parameters +# +# Read more about global parameters in +# https://pydoit.org/task-args.html#command-line-variables-doit-get-var + +CONTAINER_RUNTIME = get_var("runtime", "podman") +DEFAULT_RELEASE_DIR = Path.home() / "release-assets" / VERSION +# XXX: Workaround for https://github.com/pydoit/doit/issues/164 +RELEASE_DIR = Path(get_var("release_dir", None) or DEFAULT_RELEASE_DIR) +APPLE_ID = get_var("apple_id", None) + +### Task Parameters + +PARAM_APPLE_ID = { + "name": "apple_id", + "long": "apple-id", + "default": APPLE_ID, + "help": "The Apple developer ID that will be used for signing the .dmg", +} + +PARAM_USE_CACHE = { + "name": "use_cache", + "long": "use-cache", + "help": ( + "Whether to use cached results or not. For reproducibility reasons," + " it's best to leave it to false" + ), + "default": False, +} + +### File dependencies +# +# Define all the file dependencies for our tasks in a single place, since some file +# dependencies are shared between tasks. + + +def list_files(path, recursive=False): + """List files in a directory, and optionally traverse into subdirectories.""" + filepaths = [] + for root, _, files in os.walk(path): + for f in files: + if f.endswith(".pyc"): + continue + filepaths.append(Path(root) / f) + if not recursive: + break + return filepaths + + +def list_language_data(): + """List the expected language data that Dangerzone downloads and stores locally.""" + tessdata_dir = Path("share") / "tessdata" + langs = json.loads(open(tessdata_dir.parent / "ocr-languages.json").read()).values() + targets = [tessdata_dir / f"{lang}.traineddata" for lang in langs] + targets.append(tessdata_dir) + return targets + + +TESSDATA_DEPS = ["install/common/download-tessdata.py", "share/ocr-languages.json"] +TESSDATA_TARGETS = list_language_data() + +IMAGE_DEPS = [ + "Dockerfile", + "poetry.lock", + *list_files("dangerzone/conversion"), + "dangerzone/gvisor_wrapper/entrypoint.py", + "install/common/build-image.py", +] +IMAGE_TARGETS = ["share/container.tar.gz", "share/image-id.txt"] + +SOURCE_DEPS = [ + *list_files("assets"), + *list_files("share"), + *list_files("dangerzone", recursive=True), +] + +PYTHON_DEPS = ["poetry.lock", "pyproject.toml"] + +DMG_DEPS = [ + *list_files("install/macos"), + *TESSDATA_TARGETS, + *IMAGE_TARGETS, + *PYTHON_DEPS, + *SOURCE_DEPS, +] + +LINUX_DEPS = [ + *list_files("install/linux"), + *IMAGE_TARGETS, + *PYTHON_DEPS, + *SOURCE_DEPS, +] + +DEB_DEPS = [*LINUX_DEPS, *list_files("debian")] +RPM_DEPS = [*LINUX_DEPS, *list_files("qubes")] + + +def copy_dir(src, dst): + """Copy a directory to a destination dir, and overwrite it if it exists.""" + shutil.rmtree(dst, ignore_errors=True) + shutil.copytree(src, dst) + + +def create_release_dir(): + RELEASE_DIR.mkdir(parents=True, exist_ok=True) + (RELEASE_DIR / "tmp").mkdir(exist_ok=True) + + +def build_linux_pkg(distro, version, cwd, qubes=False): + """Generic command for building a .deb/.rpm in a Dangerzone dev environment.""" + pkg = "rpm" if distro == "fedora" else "deb" + cmd = [ + "python3", + "./dev_scripts/env.py", + "--distro", + distro, + "--version", + version, + "run", + "--no-gui", + "--dev", + f"./dangerzone/install/linux/build-{pkg}.py", + ] + if qubes: + cmd += ["--qubes"] + return CmdAction(" ".join(cmd), cwd=cwd) + + +def build_deb(cwd): + """Build a .deb package on Debian Bookworm.""" + return build_linux_pkg(distro="debian", version="bookworm", cwd=cwd) + + +def build_rpm(version, cwd, qubes=False): + """Build an .rpm package on the requested Fedora distro.""" + return build_linux_pkg(distro="Fedora", version=version, cwd=cwd, qubes=qubes) + + +### Tasks + + +def task_clean_container_runtime(): + """Clean the storage space of the container runtime.""" + return { + "actions": None, + "clean": [ + [CONTAINER_RUNTIME, "system", "prune", "-a", "-f"], + ], + } + + +def task_check_container_runtime(): + """Test that the container runtime is ready.""" + return { + "actions": [ + ["which", CONTAINER_RUNTIME], + [CONTAINER_RUNTIME, "ps"], + ], + } + + +def task_macos_check_cert(): + """Test that the Apple developer certificate can be used.""" + return { + "actions": [ + "xcrun notarytool history --apple-id %(apple_id)s --keychain-profile dz-notarytool-release-key" + ], + "params": [PARAM_APPLE_ID], + } + + +def task_macos_check_system(): + """Run macOS specific system checks, as well as the generic ones.""" + return { + "actions": None, + "task_dep": ["check_container_runtime", "macos_check_cert"], + } + + +def task_init_release_dir(): + """Create a directory for release artifacts.""" + return { + "actions": [create_release_dir], + "clean": [f"rm -rf {RELEASE_DIR}"], + } + + +def task_download_tessdata(): + """Download the Tesseract data using ./install/common/download-tessdata.py""" + return { + "actions": ["python install/common/download-tessdata.py"], + "file_dep": TESSDATA_DEPS, + "targets": TESSDATA_TARGETS, + "clean": True, + } + + +def task_build_image(): + """Build the container image using ./install/common/build-image.py""" + img_src = "share/container.tar.gz" + img_dst = RELEASE_DIR / f"container-{VERSION}-{ARCH}.tar.gz" # FIXME: Add arch + img_id_src = "share/image-id.txt" + img_id_dst = RELEASE_DIR / "image-id.txt" # FIXME: Add arch + + return { + "actions": [ + f"python install/common/build-image.py --use-cache=%(use_cache)s --runtime={CONTAINER_RUNTIME}", + ["cp", img_src, img_dst], + ["cp", img_id_src, img_id_dst], + ], + "params": [PARAM_USE_CACHE], + "file_dep": IMAGE_DEPS, + "targets": [img_src, img_dst, img_id_src, img_id_dst], + "task_dep": ["init_release_dir", "check_container_runtime"], + "clean": True, + } + + +def task_poetry_install(): + """Setup the Poetry environment""" + return {"actions": ["poetry install --sync"], "clean": ["poetry env remove --all"]} + + +def task_macos_build_dmg(): + """Build the macOS .dmg file for Dangerzone.""" + dz_dir = RELEASE_DIR / "tmp" / "macos" + dmg_src = dz_dir / "dist" / "Dangerzone.dmg" + dmg_dst = RELEASE_DIR / f"Dangerzone-{VERSION}-{ARCH}.dmg" # FIXME: Add -arch + + return { + "actions": [ + (copy_dir, [".", dz_dir]), + f"cd {dz_dir} && poetry run install/macos/build-app.py --with-codesign", + ( + "xcrun notarytool submit --wait --apple-id %(apple_id)s" + f" --keychain-profile dz-notarytool-release-key {dmg_src}" + ), + f"xcrun stapler staple {dmg_src}", + ["cp", dmg_src, dmg_dst], + ["rm", "-rf", dz_dir], + ], + "params": [PARAM_APPLE_ID], + "file_dep": DMG_DEPS, + "task_dep": [ + "macos_check_system", + "init_release_dir", + "poetry_install", + "download_tessdata", + ], + "targets": [dmg_src, dmg_dst], + "clean": True, + } + + +def task_debian_env(): + """Build a Debian Bookworm dev environment.""" + return { + "actions": [ + [ + "python3", + "./dev_scripts/env.py", + "--distro", + "debian", + "--version", + "bookworm", + "build-dev", + ] + ], + "task_dep": ["check_container_runtime"], + } + + +def task_debian_deb(): + """Build a Debian package for Debian Bookworm.""" + dz_dir = RELEASE_DIR / "tmp" / "debian" + deb_name = f"dangerzone_{VERSION}-1_amd64.deb" + deb_src = dz_dir / "deb_dist" / deb_name + deb_dst = RELEASE_DIR / deb_name + + return { + "actions": [ + (copy_dir, [".", dz_dir]), + build_deb(cwd=dz_dir), + ["cp", deb_src, deb_dst], + ["rm", "-rf", dz_dir], + ], + "file_dep": DEB_DEPS, + "task_dep": ["init_release_dir", "debian_env"], + "targets": [deb_dst], + "clean": True, + } + + +def task_fedora_env(): + """Build Fedora dev environments.""" + for version in FEDORA_VERSIONS: + yield { + "name": version, + "doc": f"Build Fedora {version} dev environments", + "actions": [ + [ + "python3", + "./dev_scripts/env.py", + "--distro", + "fedora", + "--version", + version, + "build-dev", + ], + ], + "task_dep": ["check_container_runtime"], + } + + +def task_fedora_rpm(): + """Build Fedora packages for every supported version.""" + for version in FEDORA_VERSIONS: + for qubes in (True, False): + qubes_ident = "-qubes" if qubes else "" + qubes_desc = " for Qubes" if qubes else "" + dz_dir = RELEASE_DIR / "tmp" / f"f{version}{qubes_ident}" + rpm_names = [ + f"dangerzone{qubes_ident}-{VERSION}-1.fc{version}.x86_64.rpm", + f"dangerzone{qubes_ident}-{VERSION}-1.fc{version}.src.rpm", + ] + rpm_src = [dz_dir / "dist" / rpm_name for rpm_name in rpm_names] + rpm_dst = [RELEASE_DIR / rpm_name for rpm_name in rpm_names] + + yield { + "name": version + qubes_ident, + "doc": f"Build a Fedora {version} package{qubes_desc}", + "actions": [ + (copy_dir, [".", dz_dir]), + build_rpm(version, cwd=dz_dir, qubes=qubes), + ["cp", *rpm_src, RELEASE_DIR], + ["rm", "-rf", dz_dir], + ], + "file_dep": RPM_DEPS, + "task_dep": ["init_release_dir", f"fedora_env:{version}"], + "targets": rpm_dst, + "clean": True, + } + + +def task_git_archive(): + """Build a Git archive of the repo.""" + target = f"{RELEASE_DIR}/dangerzone-{VERSION}.tar.gz" + return { + "actions": [ + f"git archive --format=tar.gz -o {target} --prefix=dangerzone/ v{VERSION}" + ], + "targets": [target], + "task_dep": ["init_release_dir"], + } + + +####################################################################################### +# +# END OF TASKS +# +# The following task should be the LAST one in the dodo file, so that it runs first when +# running `do clean`. + + +def clean_prompt(): + ans = input( + f""" +You have not specified a target to clean. +This means that doit will clean the following targets: + +* ALL the containers, images, and build cache in {CONTAINER_RUNTIME.capitalize()} +* ALL the built targets and directories + +For a full list of the targets that doit will clean, run: doit clean --dry-run + +Are you sure you want to clean everything (y/N): \ +""" + ) + if ans.lower() in ["yes", "y"]: + return + else: + print("Exiting...") + exit(1) + + +def task_clean_prompt(): + """Make sure that the user really wants to run the clean tasks.""" + return { + "actions": None, + "clean": [clean_prompt], + } From 2f29095b31b24b0631dfbd4b3a68bf0eb9737fa9 Mon Sep 17 00:00:00 2001 From: Alex Pyrgiotis Date: Wed, 4 Dec 2024 14:06:32 +0200 Subject: [PATCH 11/12] docs: Update release instructions Update our release instructions with a way to run manual tasks via `doit`. Also, add developer documentation on how to use `doit`, and some tips and tricks. --- RELEASE.md | 26 ++++++++++++++----- docs/developer/doit.md | 58 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 6 deletions(-) create mode 100644 docs/developer/doit.md diff --git a/RELEASE.md b/RELEASE.md index 9a1e89f..070fbe4 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -80,7 +80,16 @@ Once we are confident that the release will be out shortly, and doesn't need any ### macOS Release -This needs to happen for both Silicon and Intel chipsets. +> [!TIP] +> You can automate these steps from your macOS terminal app with: +> +> ``` +> doit clean +> doit -n 8 apple_id= # for Intel macOS +> doit -n 8 apple_id= macos_build_dmg # for Apple Silicon macOS +> ``` + +The following needs to happen for both Silicon and Intel chipsets. #### Initial Setup @@ -221,12 +230,17 @@ Rename `Dangerzone.msi` to `Dangerzone-$VERSION.msi`. ### Linux release -> [!INFO] -> Below we explain how we build packages for each Linux distribution we support. +> [!TIP] +> You can automate these steps from any Linux distribution with: > -> There is also a `release.sh` script available which creates all -> the `.rpm` and `.deb` files with a single command. +> ``` +> doit clean +> doit -n 8 fedora_rpm debian_deb +> ``` +> +> You can then add the created artifacts to the appropriate APT/YUM repo. +Below we explain how we build packages for each Linux distribution we support. #### Debian/Ubuntu @@ -268,7 +282,7 @@ or create your own locally with: ./dev_scripts/env.py --distro fedora --version 41 build-dev # Build the latest container (skip if already built): -./dev_scripts/env.py --distro fedora --version 41 run --dev bash -c "cd dangerzone && poetry run ./install/common/build-image.py" +./dev_scripts/env.py --distro fedora --version 41 run --dev bash -c "cd dangerzone && poetry run ./install/common/build-image.py" # Create a .rpm: ./dev_scripts/env.py --distro fedora --version 41 run --dev bash -c "cd dangerzone && ./install/linux/build-rpm.py" diff --git a/docs/developer/doit.md b/docs/developer/doit.md new file mode 100644 index 0000000..e702eff --- /dev/null +++ b/docs/developer/doit.md @@ -0,0 +1,58 @@ +# Using the Doit Automation Tool + +Developers can use the [Doit](https://pydoit.org/) automation tool to create +release artifacts. The purpose of the tool is to automate our manual release +instructions in `RELEASE.md` file. Not everything is automated yet, since we're +still experimenting with this tool. You can find our task definitions in this +repo's `dodo.py` file. + +## Why Doit? + +We picked Doit out of the various tools out there for the following reasons: + +* **Pythonic:** The configuration file and tasks can be written in Python. Where + applicable, it's easy to issue shell commands as well. +* **File targets:** Doit borrows the file target concept from Makefiles. Tasks + can have file dependencies, and targets they build. This makes it easy to + define a dependency graph (DAG) for tasks. +* **Hash-based caching:** Unlike Makefiles, doit does not look at the + modification timestamp of source/target files, to figure out if it needs to + run them. Instead, it hashes those files, and will run a task only if the + hash of a file dependency has changed. +* **Parallelization:** Tasks can be run in parallel with the `-n` argument, + which is similar to `make`'s `-j` argument. + +## How to Doit? + +First, enter your Poetry shell. Then, make sure that your environment is clean, +and you have ample disk space. You can run: + +```bash +doit clean --dry-run # if you want to see what would happen +doit clean # you'll be asked to cofirm that you want to clean everything +``` + +Finally, you can build all the release artifacts with `doit`, or a specific task +with: + +``` +doit +``` + +## Tips and tricks + +* You can run `doit list --all -s` to see the full list of tasks, their + dependencies, and whether they are up to date. +* You can run `doit info ` to see which dependencies are missing. +* You can change this line in `pyproject.toml` to `true`, to allow using the + Docker/Podman build cache: + + ``` + use_cache = true + ``` + +* You can pass the following global parameters with `doit =`: + - `runtime`: The container runtime to use. Either `podman` or `docker` + - `release_dir`: Where to store the release artifacts. Default path is + `~/release-assets/` + - `apple_id`: The Apple ID to use when signing/notarizing the macOS DMG. From 396d53b130fbcec7be44c4a560c04e510bece5e1 Mon Sep 17 00:00:00 2001 From: Alex Pyrgiotis Date: Wed, 4 Dec 2024 14:10:44 +0200 Subject: [PATCH 12/12] Add doit configuration options --- pyproject.toml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index c0e620a..9945c35 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,6 +67,13 @@ skip_gitignore = true # This is necessary due to https://github.com/PyCQA/isort/issues/1835 follow_links = false +[tool.doit] +verbosity = 3 + +[tool.doit.tasks.build_image] +# DO NOT change this to 'true' for release artifacts. +use_cache = false + [build-system] requires = ["poetry-core>=1.2.0"] build-backend = "poetry.core.masonry.api"