Merge 178364e3a7 into fbe05065c9

Update our release instructions with a way to run manual tasks via
`doit`. Also, add developer documentation on how to use `doit`, and some
tips and tricks.

.github/workflows/build.yml

@@ -74,6 +74,8 @@ jobs:
     runs-on: ubuntu-24.04
     steps:
       - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
 
       - name: Get current date
         id: date
@@ -83,7 +85,7 @@ jobs:
         id: cache-container-image
         uses: actions/cache@v4
         with:
-          key: v2-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/common.py', 'dangerzone/conversion/doc_to_pixels.py', 'dangerzone/conversion/pixels_to_pdf.py', 'poetry.lock', 'gvisor_wrapper/entrypoint.py') }}
+          key: v3-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/common.py', 'dangerzone/conversion/doc_to_pixels.py', 'dangerzone/conversion/pixels_to_pdf.py', 'poetry.lock', 'gvisor_wrapper/entrypoint.py') }}
           path: |
             share/container.tar.gz
             share/image-id.txt
@@ -95,6 +97,7 @@ jobs:
           python3 ./install/common/build-image.py
           echo ${{ github.token }} | podman login ghcr.io -u USERNAME --password-stdin
           gunzip -c share/container.tar.gz | podman load
+          tag=$(cat share/image-id.txt)
           podman push \
-                  dangerzone.rocks/dangerzone \
-                  ${{ env.IMAGE_REGISTRY }}/dangerzone/dangerzone
+                  dangerzone.rocks/dangerzone:$tag \
+                  ${{ env.IMAGE_REGISTRY }}/dangerzone/dangerzone:tag

.github/workflows/ci.yml

@@ -48,6 +48,8 @@ jobs:
     runs-on: ubuntu-24.04
     steps:
       - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
 
       - name: Get current date
         id: date
@@ -57,7 +59,7 @@ jobs:
         id: cache-container-image
         uses: actions/cache@v4
         with:
-          key: v2-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/common.py', 'dangerzone/conversion/doc_to_pixels.py', 'dangerzone/conversion/pixels_to_pdf.py', 'poetry.lock', 'gvisor_wrapper/entrypoint.py') }}
+          key: v3-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/common.py', 'dangerzone/conversion/doc_to_pixels.py', 'dangerzone/conversion/pixels_to_pdf.py', 'poetry.lock', 'gvisor_wrapper/entrypoint.py') }}
           path: |-
             share/container.tar.gz
             share/image-id.txt
@@ -225,7 +227,7 @@ jobs:
       - name: Restore container cache
         uses: actions/cache/restore@v4
         with:
-          key: v2-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/common.py', 'dangerzone/conversion/doc_to_pixels.py', 'dangerzone/conversion/pixels_to_pdf.py', 'poetry.lock', 'gvisor_wrapper/entrypoint.py') }}
+          key: v3-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/common.py', 'dangerzone/conversion/doc_to_pixels.py', 'dangerzone/conversion/pixels_to_pdf.py', 'poetry.lock', 'gvisor_wrapper/entrypoint.py') }}
           path: |-
             share/container.tar.gz
             share/image-id.txt
@@ -249,7 +251,7 @@ jobs:
   install-deb:
     name: "install-deb (${{ matrix.distro }} ${{ matrix.version }})"
     runs-on: ubuntu-latest
-    needs: 
+    needs:
       - build-deb
     strategy:
       matrix:
@@ -332,7 +334,7 @@ jobs:
       - name: Restore container image
         uses: actions/cache/restore@v4
         with:
-          key: v2-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/common.py', 'dangerzone/conversion/doc_to_pixels.py', 'dangerzone/conversion/pixels_to_pdf.py', 'poetry.lock', 'gvisor_wrapper/entrypoint.py') }}
+          key: v3-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/common.py', 'dangerzone/conversion/doc_to_pixels.py', 'dangerzone/conversion/pixels_to_pdf.py', 'poetry.lock', 'gvisor_wrapper/entrypoint.py') }}
           path: |-
             share/container.tar.gz
             share/image-id.txt
@@ -427,7 +429,7 @@ jobs:
       - name: Restore container image
         uses: actions/cache/restore@v4
         with:
-          key: v2-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/common.py', 'dangerzone/conversion/doc_to_pixels.py', 'dangerzone/conversion/pixels_to_pdf.py', 'poetry.lock', 'gvisor_wrapper/entrypoint.py') }}
+          key: v3-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/common.py', 'dangerzone/conversion/doc_to_pixels.py', 'dangerzone/conversion/pixels_to_pdf.py', 'poetry.lock', 'gvisor_wrapper/entrypoint.py') }}
           path: |-
             share/container.tar.gz
             share/image-id.txt

.github/workflows/scan.yml

@@ -14,17 +14,24 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
       - name: Install container build dependencies
         run: sudo apt install pipx && pipx install poetry
       - name: Build container image
         run: python3 ./install/common/build-image.py --runtime docker --no-save
+      - name: Get image tag
+        id: tag
+        run: |
+          tag=$(docker images dangerzone.rocks/dangerzone --format '{{ .Tag }}')
+          echo "tag=$tag" >> $GITHUB_OUTPUT
       # NOTE: Scan first without failing, else we won't be able to read the scan
       # report.
       - name: Scan container image (no fail)
         uses: anchore/scan-action@v5
         id: scan_container
         with:
-          image: "dangerzone.rocks/dangerzone:latest"
+          image: "dangerzone.rocks/dangerzone:${{ steps.tag.outputs.tag }}"
           fail-build: false
           only-fixed: false
           severity-cutoff: critical
@@ -38,7 +45,7 @@ jobs:
       - name: Scan container image
         uses: anchore/scan-action@v5
         with:
-          image: "dangerzone.rocks/dangerzone:latest"
+          image: "dangerzone.rocks/dangerzone:${{ steps.tag.outputs.tag }}"
           fail-build: true
           only-fixed: false
           severity-cutoff: critical

.github/workflows/scan_released.yml

@@ -24,13 +24,18 @@ jobs:
           CONTAINER_FILENAME=container-${VERSION:1}-${{ matrix.arch }}.tar.gz
           wget https://github.com/freedomofpress/dangerzone/releases/download/${VERSION}/${CONTAINER_FILENAME} -O ${CONTAINER_FILENAME}
           docker load -i ${CONTAINER_FILENAME}
+      - name: Get image tag
+        id: tag
+        run: |
+          tag=$(docker images dangerzone.rocks/dangerzone --format '{{ .Tag }}')
+          echo "tag=$tag" >> $GITHUB_OUTPUT
       # NOTE: Scan first without failing, else we won't be able to read the scan
       # report.
       - name: Scan container image (no fail)
         uses: anchore/scan-action@v5
         id: scan_container
         with:
-          image: "dangerzone.rocks/dangerzone:latest"
+          image: "dangerzone.rocks/dangerzone:${{ steps.tag.outputs.tag }}"
           fail-build: false
           only-fixed: false
           severity-cutoff: critical
@@ -44,7 +49,7 @@ jobs:
       - name: Scan container image
         uses: anchore/scan-action@v5
         with:
-          image: "dangerzone.rocks/dangerzone:latest"
+          image: "dangerzone.rocks/dangerzone:${{ steps.tag.outputs.tag }}"
           fail-build: true
           only-fixed: false
           severity-cutoff: critical

.gitignore

@@ -149,3 +149,4 @@ share/container.tar
 share/container.tar.gz
 share/image-id.txt
 container/container-pip-requirements.txt
+.doit.db.db

CHANGELOG.md

@@ -18,8 +18,8 @@ since 0.4.1, and this project adheres to [Semantic Versioning](https://semver.or
 
 ### Development changes
 
-- Build Dangerzone MSI with Wix Toolset 5 ([#929](https://github.com/freedomofpress/dangerzone/pull/929)).
   Thanks [@jkarasti](https://github.com/jkarasti) for the contribution.
+- Automate a large portion of our release tasks with `doit` ([#1016](https://github.com/freedomofpress/dangerzone/issues/1016))
 
 ## [0.8.0](https://github.com/freedomofpress/dangerzone/compare/v0.8.0...0.7.1)
 

Makefile

@@ -66,6 +66,22 @@ test-large: test-large-init  ## Run large test set
 	python -m pytest --tb=no tests/test_large_set.py::TestLargeSet -v $(JUNIT_FLAGS) --junitxml=$(TEST_LARGE_RESULTS)
 	python $(TEST_LARGE_RESULTS)/report.py $(TEST_LARGE_RESULTS)
 
+.PHONY: build-clean
+build-clean:
+	doit clean
+
+.PHONY: build-macos-intel
+build-macos-intel: build-clean
+	doit -n 8
+
+.PHONY: build-macos-arm
+build-macos-arm: build-clean
+	doit -n 8 macos_build_dmg
+
+.PHONY: build-linux
+build-linux: build-clean
+	doit -n 8 fedora_rpm debian_deb
+
 # Makefile self-help borrowed from the securedrop-client project
 # Explaination of the below shell command should it ever break.
 # 1. Set the field separator to ": ##" and any make targets that might appear between : and ##

QA.md

@@ -107,9 +107,9 @@ Close the Dangerzone application and get the container image for that
 version. For example:
 
 ```
-$ docker images dangerzone.rocks/dangerzone:latest
+$ docker images dangerzone.rocks/dangerzone
 REPOSITORY                   TAG         IMAGE ID      CREATED       SIZE
-dangerzone.rocks/dangerzone  latest      <image ID>    <date>        <size>
+dangerzone.rocks/dangerzone  <tag>       <image ID>    <date>        <size>
 ```
 
 Then run the version under QA and ensure that the settings remain changed.
@@ -118,9 +118,9 @@ Afterwards check that new docker image was installed by running the same command
 and seeing the following differences:
 
 ```
-$ docker images dangerzone.rocks/dangerzone:latest
+$ docker images dangerzone.rocks/dangerzone
 REPOSITORY                   TAG         IMAGE ID        CREATED       SIZE
-dangerzone.rocks/dangerzone  latest      <different ID>  <newer date>  <different size>
+dangerzone.rocks/dangerzone  <other tag> <different ID>  <newer date>  <different size>
 ```
 
 #### 4. Dangerzone successfully installs the container image

RELEASE.md

@@ -76,7 +76,16 @@ Once we are confident that the release will be out shortly, and doesn't need any
 
 ### macOS Release
 
-This needs to happen for both Silicon and Intel chipsets.
+> [!TIP]
+> You can automate these steps from your macOS terminal app with:
+>
+> ```
+> export APPLE_ID=<email>
+> make build-macos-intel  # for Intel macOS
+> make build-macos-arm    # for Apple Silicon macOS
+> ```
+
+The following needs to happen for both Silicon and Intel chipsets.
 
 #### Initial Setup
 
@@ -126,7 +135,7 @@ Here is what you need to do:
   ```
 
 - [ ] Build the container image and the OCR language data
-  
+
   ```bash
   poetry run ./install/common/build-image.py
   poetry run ./install/common/download-tessdata.py
@@ -142,12 +151,10 @@ Here is what you need to do:
   poetry run ./install/macos/build-app.py
   ```
 
-- [ ] Make sure that the build application works with the containerd graph
-  driver (see [#933](https://github.com/freedomofpress/dangerzone/issues/933))
 - [ ] Sign the application bundle, and notarize it
-  
+
   You need to run this command as the account that has access to the code signing certificate
-  
+
   This command assumes that you have created, and stored in the Keychain, an
   application password associated with your Apple Developer ID, which will be
   used specifically for `notarytool`.
@@ -212,9 +219,6 @@ The Windows release is performed in a Windows 11 virtual machine (as opposed to
 - [ ] Copy the container image into the VM
   > [!IMPORTANT]
   > Instead of running `python .\install\windows\build-image.py` in the VM, run the build image script on the host (making sure to build for `linux/amd64`). Copy `share/container.tar.gz` and `share/image-id.txt` from the host into the `share` folder in the VM.
-  > Also, don't forget to add the supplementary image ID (see
-  > [#933](https://github.com/freedomofpress/dangerzone/issues/933)) in
-  > `share/image-id.txt`)
 - [ ] Run `poetry run .\install\windows\build-app.bat`
 - [ ] When you're done you will have `dist\Dangerzone.msi`
 
@@ -222,12 +226,16 @@ Rename `Dangerzone.msi` to `Dangerzone-$VERSION.msi`.
 
 ### Linux release
 
-> [!INFO]
-> Below we explain how we build packages for each Linux distribution we support.
+> [!TIP]
+> You can automate these steps from any Linux distribution with:
 >
-> There is also a `release.sh` script available which creates all
-> the `.rpm` and `.deb` files with a single command.
+> ```
+> make build-linux
+> ```
+>
+> You can then add the created artifacts to the appropriate APT/YUM repo.
 
+Below we explain how we build packages for each Linux distribution we support.
 
 #### Debian/Ubuntu
 
@@ -269,7 +277,7 @@ or create your own locally with:
 ./dev_scripts/env.py --distro fedora --version 41 build-dev
 
 # Build the latest container (skip if already built):
-./dev_scripts/env.py --distro fedora --version 41 run --dev bash -c "cd dangerzone && poetry run ./install/common/build-image.py" 
+./dev_scripts/env.py --distro fedora --version 41 run --dev bash -c "cd dangerzone && poetry run ./install/common/build-image.py"
 
 # Create a .rpm:
 ./dev_scripts/env.py --distro fedora --version 41 run --dev bash -c "cd dangerzone && ./install/linux/build-rpm.py"

dangerzone/container_utils.py

@@ -0,0 +1,149 @@
+import gzip
+import logging
+import platform
+import shutil
+import subprocess
+from typing import List, Tuple
+
+from . import errors
+from .util import get_resource_path, get_subprocess_startupinfo
+
+CONTAINER_NAME = "dangerzone.rocks/dangerzone"
+
+log = logging.getLogger(__name__)
+
+
+def get_runtime_name() -> str:
+    if platform.system() == "Linux":
+        runtime_name = "podman"
+    else:
+        # Windows, Darwin, and unknown use docker for now, dangerzone-vm eventually
+        runtime_name = "docker"
+    return runtime_name
+
+
+def get_runtime_version() -> Tuple[int, int]:
+    """Get the major/minor parts of the Docker/Podman version.
+
+    Some of the operations we perform in this module rely on some Podman features
+    that are not available across all of our platforms. In order to have a proper
+    fallback, we need to know the Podman version. More specifically, we're fine with
+    just knowing the major and minor version, since writing/installing a full-blown
+    semver parser is an overkill.
+    """
+    # Get the Docker/Podman version, using a Go template.
+    runtime = get_runtime_name()
+    if runtime == "podman":
+        query = "{{.Client.Version}}"
+    else:
+        query = "{{.Server.Version}}"
+
+    cmd = [runtime, "version", "-f", query]
+    try:
+        version = subprocess.run(
+            cmd,
+            startupinfo=get_subprocess_startupinfo(),
+            capture_output=True,
+            check=True,
+        ).stdout.decode()
+    except Exception as e:
+        msg = f"Could not get the version of the {runtime.capitalize()} tool: {e}"
+        raise RuntimeError(msg) from e
+
+    # Parse this version and return the major/minor parts, since we don't need the
+    # rest.
+    try:
+        major, minor, _ = version.split(".", 3)
+        return (int(major), int(minor))
+    except Exception as e:
+        msg = (
+            f"Could not parse the version of the {runtime.capitalize()} tool"
+            f" (found: '{version}') due to the following error: {e}"
+        )
+        raise RuntimeError(msg)
+
+
+def get_runtime() -> str:
+    container_tech = get_runtime_name()
+    runtime = shutil.which(container_tech)
+    if runtime is None:
+        raise errors.NoContainerTechException(container_tech)
+    return runtime
+
+
+def list_image_tags() -> List[str]:
+    """Get the tags of all loaded Dangerzone images.
+
+    This method returns a mapping of image tags to image IDs, for all Dangerzone
+    images. This can be useful when we want to find which are the local image tags,
+    and which image ID does the "latest" tag point to.
+    """
+    return (
+        subprocess.check_output(
+            [
+                get_runtime(),
+                "image",
+                "list",
+                "--format",
+                "{{ .Tag }}",
+                CONTAINER_NAME,
+            ],
+            text=True,
+            startupinfo=get_subprocess_startupinfo(),
+        )
+        .strip()
+        .split()
+    )
+
+
+def delete_image_tag(tag: str) -> None:
+    """Delete a Dangerzone image tag."""
+    name = CONTAINER_NAME + ":" + tag
+    log.warning(f"Deleting old container image: {name}")
+    try:
+        subprocess.check_output(
+            [get_runtime(), "rmi", "--force", name],
+            startupinfo=get_subprocess_startupinfo(),
+        )
+    except Exception as e:
+        log.warning(
+            f"Couldn't delete old container image '{name}', so leaving it there."
+            f" Original error: {e}"
+        )
+
+
+def get_expected_tag() -> str:
+    """Get the tag of the Dangerzone image tarball from the image-id.txt file."""
+    with open(get_resource_path("image-id.txt")) as f:
+        return f.read().strip()
+
+
+def load_image_tarball() -> None:
+    log.info("Installing Dangerzone container image...")
+    p = subprocess.Popen(
+        [get_runtime(), "load"],
+        stdin=subprocess.PIPE,
+        startupinfo=get_subprocess_startupinfo(),
+    )
+
+    chunk_size = 4 << 20
+    compressed_container_path = get_resource_path("container.tar.gz")
+    with gzip.open(compressed_container_path) as f:
+        while True:
+            chunk = f.read(chunk_size)
+            if len(chunk) > 0:
+                if p.stdin:
+                    p.stdin.write(chunk)
+            else:
+                break
+    _, err = p.communicate()
+    if p.returncode < 0:
+        if err:
+            error = err.decode()
+        else:
+            error = "No output"
+        raise errors.ImageInstallationException(
+            f"Could not install container image: {error}"
+        )
+
+    log.info("Successfully installed container image from")

dangerzone/errors.py

@@ -117,3 +117,26 @@ def handle_document_errors(func: F) -> F:
             sys.exit(1)
 
     return cast(F, wrapper)
+
+
+#### Container-related errors
+
+
+class ImageNotPresentException(Exception):
+    pass
+
+
+class ImageInstallationException(Exception):
+    pass
+
+
+class NoContainerTechException(Exception):
+    def __init__(self, container_tech: str) -> None:
+        super().__init__(f"{container_tech} is not installed")
+
+
+class NotAvailableContainerTechException(Exception):
+    def __init__(self, container_tech: str, error: str) -> None:
+        self.error = error
+        self.container_tech = container_tech
+        super().__init__(f"{container_tech} is not available")

dangerzone/gui/main_window.py

@@ -25,13 +25,7 @@ else:
 
 from .. import errors
 from ..document import SAFE_EXTENSION, Document
-from ..isolation_provider.container import (
-    Container,
-    NoContainerTechException,
-    NotAvailableContainerTechException,
-)
-from ..isolation_provider.dummy import Dummy
-from ..isolation_provider.qubes import Qubes, is_qubes_native_conversion
+from ..isolation_provider.qubes import is_qubes_native_conversion
 from ..util import format_exception, get_resource_path, get_version
 from .logic import Alert, CollapsibleBox, DangerzoneGui, UpdateDialog
 from .updater import UpdateReport
@@ -197,14 +191,11 @@ class MainWindow(QtWidgets.QMainWindow):
         header_layout.addWidget(self.hamburger_button)
         header_layout.addSpacing(15)
 
-        if isinstance(self.dangerzone.isolation_provider, Container):
+        if self.dangerzone.isolation_provider.should_wait_install():
             # Waiting widget replaces content widget while container runtime isn't available
             self.waiting_widget: WaitingWidget = WaitingWidgetContainer(self.dangerzone)
             self.waiting_widget.finished.connect(self.waiting_finished)
-
-        elif isinstance(self.dangerzone.isolation_provider, Dummy) or isinstance(
-            self.dangerzone.isolation_provider, Qubes
-        ):
+        else:
             # Don't wait with dummy converter and on Qubes.
             self.waiting_widget = WaitingWidget()
             self.dangerzone.is_waiting_finished = True
@@ -500,11 +491,11 @@ class WaitingWidgetContainer(WaitingWidget):
         error: Optional[str] = None
 
         try:
-            self.dangerzone.isolation_provider.is_runtime_available()
-        except NoContainerTechException as e:
+            self.dangerzone.isolation_provider.is_available()
+        except errors.NoContainerTechException as e:
             log.error(str(e))
             state = "not_installed"
-        except NotAvailableContainerTechException as e:
+        except errors.NotAvailableContainerTechException as e:
             log.error(str(e))
             state = "not_running"
             error = e.error

dangerzone/isolation_provider/base.py

@@ -93,10 +93,6 @@ class IsolationProvider(ABC):
         else:
             self.proc_stderr = subprocess.DEVNULL
 
-    @staticmethod
-    def is_runtime_available() -> bool:
-        return True
-
     @abstractmethod
     def install(self) -> bool:
         pass
@@ -258,6 +254,16 @@ class IsolationProvider(ABC):
             )
         return errors.exception_from_error_code(error_code)
 
+    @abstractmethod
+    def should_wait_install(self) -> bool:
+        """Whether this isolation provider takes a lot of time to install."""
+        pass
+
+    @abstractmethod
+    def is_available(self) -> bool:
+        """Whether the backing implementation of the isolation provider is available."""
+        pass
+
     @abstractmethod
     def get_max_parallel_conversions(self) -> int:
         pass

dangerzone/isolation_provider/container.py

@@ -1,12 +1,11 @@
-import gzip
 import logging
 import os
 import platform
 import shlex
-import shutil
 import subprocess
-from typing import List, Tuple
+from typing import List
 
+from .. import container_utils, errors
 from ..document import Document
 from ..util import get_resource_path, get_subprocess_startupinfo
 from .base import IsolationProvider, terminate_process_group
@@ -25,88 +24,8 @@ else:
 log = logging.getLogger(__name__)
 
 
-class NoContainerTechException(Exception):
-    def __init__(self, container_tech: str) -> None:
-        super().__init__(f"{container_tech} is not installed")
-
-
-class NotAvailableContainerTechException(Exception):
-    def __init__(self, container_tech: str, error: str) -> None:
-        self.error = error
-        self.container_tech = container_tech
-        super().__init__(f"{container_tech} is not available")
-
-
-class ImageNotPresentException(Exception):
-    pass
-
-
-class ImageInstallationException(Exception):
-    pass
-
-
 class Container(IsolationProvider):
     # Name of the dangerzone container
-    CONTAINER_NAME = "dangerzone.rocks/dangerzone"
-
-    @staticmethod
-    def get_runtime_name() -> str:
-        if platform.system() == "Linux":
-            runtime_name = "podman"
-        else:
-            # Windows, Darwin, and unknown use docker for now, dangerzone-vm eventually
-            runtime_name = "docker"
-        return runtime_name
-
-    @staticmethod
-    def get_runtime_version() -> Tuple[int, int]:
-        """Get the major/minor parts of the Docker/Podman version.
-
-        Some of the operations we perform in this module rely on some Podman features
-        that are not available across all of our platforms. In order to have a proper
-        fallback, we need to know the Podman version. More specifically, we're fine with
-        just knowing the major and minor version, since writing/installing a full-blown
-        semver parser is an overkill.
-        """
-        # Get the Docker/Podman version, using a Go template.
-        runtime = Container.get_runtime_name()
-        if runtime == "podman":
-            query = "{{.Client.Version}}"
-        else:
-            query = "{{.Server.Version}}"
-
-        cmd = [runtime, "version", "-f", query]
-        try:
-            version = subprocess.run(
-                cmd,
-                startupinfo=get_subprocess_startupinfo(),
-                capture_output=True,
-                check=True,
-            ).stdout.decode()
-        except Exception as e:
-            msg = f"Could not get the version of the {runtime.capitalize()} tool: {e}"
-            raise RuntimeError(msg) from e
-
-        # Parse this version and return the major/minor parts, since we don't need the
-        # rest.
-        try:
-            major, minor, _ = version.split(".", 3)
-            return (int(major), int(minor))
-        except Exception as e:
-            msg = (
-                f"Could not parse the version of the {runtime.capitalize()} tool"
-                f" (found: '{version}') due to the following error: {e}"
-            )
-            raise RuntimeError(msg)
-
-    @staticmethod
-    def get_runtime() -> str:
-        container_tech = Container.get_runtime_name()
-        runtime = shutil.which(container_tech)
-        if runtime is None:
-            raise NoContainerTechException(container_tech)
-        return runtime
-
     @staticmethod
     def get_runtime_security_args() -> List[str]:
         """Security options applicable to the outer Dangerzone container.
@@ -127,12 +46,12 @@ class Container(IsolationProvider):
         * Do not log the container's output.
         * Do not map the host user to the container, with `--userns nomap` (available
           from Podman 4.1 onwards)
-          - This particular argument is specified in `start_doc_to_pixels_proc()`, but
-            should move here once #748 is merged.
         """
-        if Container.get_runtime_name() == "podman":
+        if container_utils.get_runtime_name() == "podman":
             security_args = ["--log-driver", "none"]
             security_args += ["--security-opt", "no-new-privileges"]
+            if container_utils.get_runtime_version() >= (4, 1):
+                security_args += ["--userns", "nomap"]
         else:
             security_args = ["--security-opt=no-new-privileges:true"]
 
@@ -156,51 +75,52 @@ class Container(IsolationProvider):
 
     @staticmethod
     def install() -> bool:
+        """Install the container image tarball, or verify that it's already installed.
+
+        Perform the following actions:
+        1. Get the tags of any locally available images that match Dangerzone's image
+           name.
+        2. Get the expected image tag from the image-id.txt file.
+           - If this tag is present in the local images, then we can return.
+           - Else, prune the older container images and continue.
+        3. Load the image tarball and make sure it matches the expected tag.
         """
-        Make sure the podman container is installed. Linux only.
-        """
-        if Container.is_container_installed():
+        old_tags = container_utils.list_image_tags()
+        expected_tag = container_utils.get_expected_tag()
+
+        if expected_tag not in old_tags:
+            # Prune older container images.
+            log.info(
+                f"Could not find a Dangerzone container image with tag '{expected_tag}'"
+            )
+            for tag in old_tags:
+                container_utils.delete_image_tag(tag)
+        else:
             return True
 
-        # Load the container into podman
-        log.info("Installing Dangerzone container image...")
+        # Load the image tarball into the container runtime.
+        container_utils.load_image_tarball()
 
-        p = subprocess.Popen(
-            [Container.get_runtime(), "load"],
-            stdin=subprocess.PIPE,
-            startupinfo=get_subprocess_startupinfo(),
-        )
-
-        chunk_size = 4 << 20
-        compressed_container_path = get_resource_path("container.tar.gz")
-        with gzip.open(compressed_container_path) as f:
-            while True:
-                chunk = f.read(chunk_size)
-                if len(chunk) > 0:
-                    if p.stdin:
-                        p.stdin.write(chunk)
-                else:
-                    break
-        _, err = p.communicate()
-        if p.returncode < 0:
-            if err:
-                error = err.decode()
-            else:
-                error = "No output"
-            raise ImageInstallationException(
-                f"Could not install container image: {error}"
+        # Check that the container image has the expected image tag.
+        # See https://github.com/freedomofpress/dangerzone/issues/988 for an example
+        # where this was not the case.
+        new_tags = container_utils.list_image_tags()
+        if expected_tag not in new_tags:
+            raise errors.ImageNotPresentException(
+                f"Could not find expected tag '{expected_tag}' after loading the"
+                " container image tarball"
             )
 
-        if not Container.is_container_installed(raise_on_error=True):
-            return False
-
-        log.info("Container image installed")
         return True
 
     @staticmethod
-    def is_runtime_available() -> bool:
-        container_runtime = Container.get_runtime()
-        runtime_name = Container.get_runtime_name()
+    def should_wait_install() -> bool:
+        return True
+
+    @staticmethod
+    def is_available() -> bool:
+        container_runtime = container_utils.get_runtime()
+        runtime_name = container_utils.get_runtime_name()
         # Can we run `docker/podman image ls` without an error
         with subprocess.Popen(
             [container_runtime, "image", "ls"],
@@ -210,61 +130,11 @@ class Container(IsolationProvider):
         ) as p:
             _, stderr = p.communicate()
             if p.returncode != 0:
-                raise NotAvailableContainerTechException(runtime_name, stderr.decode())
+                raise errors.NotAvailableContainerTechException(
+                    runtime_name, stderr.decode()
+                )
             return True
 
-    @staticmethod
-    def is_container_installed(raise_on_error: bool = False) -> bool:
-        """
-        See if the container is installed.
-        """
-        # Get the image id
-        with open(get_resource_path("image-id.txt")) as f:
-            expected_image_ids = f.read().strip().split()
-
-        # See if this image is already installed
-        installed = False
-        found_image_id = subprocess.check_output(
-            [
-                Container.get_runtime(),
-                "image",
-                "list",
-                "--format",
-                "{{.ID}}",
-                Container.CONTAINER_NAME,
-            ],
-            text=True,
-            startupinfo=get_subprocess_startupinfo(),
-        )
-        found_image_id = found_image_id.strip()
-
-        if found_image_id in expected_image_ids:
-            installed = True
-        elif found_image_id == "":
-            if raise_on_error:
-                raise ImageNotPresentException(
-                    "Image is not listed after installation. Bailing out."
-                )
-        else:
-            msg = (
-                f"{Container.CONTAINER_NAME} images found, but IDs do not match."
-                f" Found: {found_image_id}, Expected: {','.join(expected_image_ids)}"
-            )
-            if raise_on_error:
-                raise ImageNotPresentException(msg)
-            log.info(msg)
-            log.info("Deleting old dangerzone container image")
-
-            try:
-                subprocess.check_output(
-                    [Container.get_runtime(), "rmi", "--force", found_image_id],
-                    startupinfo=get_subprocess_startupinfo(),
-                )
-            except Exception:
-                log.warning("Couldn't delete old container image, so leaving it there")
-
-        return installed
-
     def doc_to_pixels_container_name(self, document: Document) -> str:
         """Unique container name for the doc-to-pixels phase."""
         return f"dangerzone-doc-to-pixels-{document.id}"
@@ -295,21 +165,22 @@ class Container(IsolationProvider):
         self,
         command: List[str],
         name: str,
-        extra_args: List[str] = [],
     ) -> subprocess.Popen:
-        container_runtime = self.get_runtime()
+        container_runtime = container_utils.get_runtime()
         security_args = self.get_runtime_security_args()
         enable_stdin = ["-i"]
         set_name = ["--name", name]
         prevent_leakage_args = ["--rm"]
+        image_name = [
+            container_utils.CONTAINER_NAME + ":" + container_utils.get_expected_tag()
+        ]
         args = (
             ["run"]
             + security_args
             + prevent_leakage_args
             + enable_stdin
             + set_name
-            + extra_args
-            + [self.CONTAINER_NAME]
+            + image_name
             + command
         )
         args = [container_runtime] + args
@@ -325,7 +196,7 @@ class Container(IsolationProvider):
         connected to the Docker daemon, and killing it will just close the associated
         standard streams.
         """
-        container_runtime = self.get_runtime()
+        container_runtime = container_utils.get_runtime()
         cmd = [container_runtime, "kill", name]
         try:
             # We do not check the exit code of the process here, since the container may
@@ -358,15 +229,8 @@ class Container(IsolationProvider):
             "-m",
             "dangerzone.conversion.doc_to_pixels",
         ]
-        # NOTE: Using `--userns nomap` is available only on Podman >= 4.1.0.
-        # XXX: Move this under `get_runtime_security_args()` once #748 is merged.
-        extra_args = []
-        if Container.get_runtime_name() == "podman":
-            if Container.get_runtime_version() >= (4, 1):
-                extra_args += ["--userns", "nomap"]
-
         name = self.doc_to_pixels_container_name(document)
-        return self.exec_container(command, name=name, extra_args=extra_args)
+        return self.exec_container(command, name=name)
 
     def terminate_doc_to_pixels_proc(
         self, document: Document, p: subprocess.Popen
@@ -389,7 +253,7 @@ class Container(IsolationProvider):
         # after a podman kill / docker kill invocation, this will likely be the case,
         # else the container runtime (Docker/Podman) has experienced a problem, and we
         # should report it.
-        container_runtime = self.get_runtime()
+        container_runtime = container_utils.get_runtime()
         name = self.doc_to_pixels_container_name(document)
         all_containers = subprocess.run(
             [container_runtime, "ps", "-a"],
@@ -411,11 +275,11 @@ class Container(IsolationProvider):
             if cpu_count is not None:
                 n_cpu = cpu_count
 
-        elif self.get_runtime_name() == "docker":
+        elif container_utils.get_runtime_name() == "docker":
             # For Windows and MacOS containers run in VM
             # So we obtain the CPU count for the VM
             n_cpu_str = subprocess.check_output(
-                [self.get_runtime(), "info", "--format", "{{.NCPU}}"],
+                [container_utils.get_runtime(), "info", "--format", "{{.NCPU}}"],
                 text=True,
                 startupinfo=get_subprocess_startupinfo(),
             )

dangerzone/isolation_provider/dummy.py

@@ -39,6 +39,14 @@ class Dummy(IsolationProvider):
     def install(self) -> bool:
         return True
 
+    @staticmethod
+    def is_available() -> bool:
+        return True
+
+    @staticmethod
+    def should_wait_install() -> bool:
+        return False
+
     def start_doc_to_pixels_proc(self, document: Document) -> subprocess.Popen:
         cmd = [
             sys.executable,

dangerzone/isolation_provider/qubes.py

@@ -21,6 +21,14 @@ class Qubes(IsolationProvider):
     def install(self) -> bool:
         return True
 
+    @staticmethod
+    def is_available() -> bool:
+        return True
+
+    @staticmethod
+    def should_wait_install() -> bool:
+        return False
+
     def get_max_parallel_conversions(self) -> int:
         return 1
 

dev_scripts/qa.py

@@ -127,9 +127,9 @@ Close the Dangerzone application and get the container image for that
 version. For example:
 
 ```
-$ docker images dangerzone.rocks/dangerzone:latest
+$ docker images dangerzone.rocks/dangerzone
 REPOSITORY                   TAG         IMAGE ID      CREATED       SIZE
-dangerzone.rocks/dangerzone  latest      <image ID>    <date>        <size>
+dangerzone.rocks/dangerzone  <tag>       <image ID>    <date>        <size>
 ```
 
 Then run the version under QA and ensure that the settings remain changed.
@@ -138,9 +138,9 @@ Afterwards check that new docker image was installed by running the same command
 and seeing the following differences:
 
 ```
-$ docker images dangerzone.rocks/dangerzone:latest
+$ docker images dangerzone.rocks/dangerzone
 REPOSITORY                   TAG         IMAGE ID        CREATED       SIZE
-dangerzone.rocks/dangerzone  latest      <different ID>  <newer date>  <different size>
+dangerzone.rocks/dangerzone  <other tag> <different ID>  <newer date>  <different size>
 ```
 
 #### 4. Dangerzone successfully installs the container image

docs/developer/doit.md

@@ -0,0 +1,67 @@
+# Using the Doit Automation Tool
+
+Developers can use the [Doit](https://pydoit.org/) automation tool to create
+release artifacts. The purpose of the tool is to automate the manual release
+instructions in `RELEASE.md` file. Not everything is automated yet, since we're
+still experimenting with this tool. You can find our task definitions in this
+repo's `dodo.py` file.
+
+## Why Doit?
+
+We picked Doit out of the various tools out there for the following reasons:
+
+* **Pythonic:** The configuration file and tasks can be written in Python. Where
+  applicable, it's easy to issue shell commands as well.
+* **File targets:** Doit borrows the file target concept from Makefiles. Tasks
+  can have file dependencies, and targets they build. This makes it easy to
+  define a dependency graph for tasks.
+* **Hash-based caching:** Unlike Makefiles, doit does not look at the
+  modification timestamp of source/target files, to figure out if it needs to
+  run them.  Instead, it hashes those files, and will run a task only if the
+  hash of a file dependency has changed.
+* **Parallelization:** Tasks can be run in parallel with the `-n` argument,
+  which is similar to `make`'s `-j` argument.
+
+## How to Doit?
+
+First, enter your Poetry shell. Then, make sure that your environment is clean,
+and you have ample disk space. You can run:
+
+```bash
+doit clean --dry-run  # if you want to see what would happen
+doit clean  # you'll be asked to cofirm that you want to clean everything
+```
+
+Finally, you can build all the release artifacts with `doit`, or a specific task
+with:
+
+```
+doit <task>
+```
+
+## Tips and tricks
+
+* You can run `doit list --all -s` to see the full list of tasks, their
+  dependencies, and whether they are up to date.
+* You can run `doit info <task>` to see which dependencies are missing.
+* You can change this line in `pyproject.toml` to `true`, to allow using the
+  Docker/Podman build cache:
+
+  ```
+  use_cache = true
+  ```
+
+  > [!WARNING]
+  > Using caching may speed up image builds, but is not suitable for release
+  > artifacts. The ID of our base container image (Alpine Linux) does not change
+  > that often, but its APK package index does. So, if we use caching, we risk
+  > skipping the `apk upgrade` layer and end up with packages that are days
+  > behind.
+
+* You can pass the following environment variables to the script, in order to
+  affect some global parameters:
+  - `CONTAINER_RUNTIME`: The container runtime to use. Either `podman` (default)
+    or `docker`.
+  - `RELEASE_DIR`: Where to store the release artifacts. Default path is
+    `~/release-assets/<version>`
+  - `APPLE_ID`: The Apple ID to use when signing/notarizing the macOS DMG.

dodo.py

@@ -0,0 +1,393 @@
+import json
+import os
+import platform
+import shutil
+from pathlib import Path
+
+from doit.action import CmdAction
+
+ARCH = "arm64" if platform.machine() == "arm64" else "i686"
+VERSION = open("share/version.txt").read().strip()
+FEDORA_VERSIONS = ["40", "41"]
+DEBIAN_VERSIONS = ["bullseye", "focal", "jammy", "mantic", "noble", "trixie"]
+
+### Global parameters
+
+CONTAINER_RUNTIME = os.environ.get("CONTAINER_RUNTIME", "podman")
+DEFAULT_RELEASE_DIR = Path.home() / "release-assets" / VERSION
+RELEASE_DIR = Path(os.environ.get("RELEASE_DIR", DEFAULT_RELEASE_DIR))
+APPLE_ID = os.environ.get("APPLE_ID", None)
+
+### Task Parameters
+
+PARAM_APPLE_ID = {
+    "name": "apple_id",
+    "long": "apple-id",
+    "default": APPLE_ID,
+    "help": "The Apple developer ID that will be used to sign the .dmg",
+}
+
+PARAM_USE_CACHE = {
+    "name": "use_cache",
+    "long": "use-cache",
+    "help": (
+        "Whether to use cached results or not. For reproducibility reasons,"
+        " it's best to leave it to false"
+    ),
+    "default": False,
+}
+
+### File dependencies
+#
+# Define all the file dependencies for our tasks in a single place, since some file
+# dependencies are shared between tasks.
+
+
+def list_files(path, recursive=False):
+    """List files in a directory, and optionally traverse into subdirectories."""
+    glob_fn = Path(path).rglob if recursive else Path(path).glob
+    return [f for f in glob_fn("*") if f.is_file() and not f.suffix == ".pyc"]
+
+
+def list_language_data():
+    """List the expected language data that Dangerzone downloads and stores locally."""
+    tessdata_dir = Path("share") / "tessdata"
+    langs = json.loads(open(tessdata_dir.parent / "ocr-languages.json").read()).values()
+    targets = [tessdata_dir / f"{lang}.traineddata" for lang in langs]
+    targets.append(tessdata_dir)
+    return targets
+
+
+TESSDATA_DEPS = ["install/common/download-tessdata.py", "share/ocr-languages.json"]
+TESSDATA_TARGETS = list_language_data()
+
+IMAGE_DEPS = [
+    "Dockerfile",
+    "poetry.lock",
+    *list_files("dangerzone/conversion"),
+    "dangerzone/gvisor_wrapper/entrypoint.py",
+    "install/common/build-image.py",
+]
+IMAGE_TARGETS = ["share/container.tar.gz", "share/image-id.txt"]
+
+SOURCE_DEPS = [
+    *list_files("assets"),
+    *list_files("share"),
+    *list_files("dangerzone", recursive=True),
+]
+
+PYTHON_DEPS = ["poetry.lock", "pyproject.toml"]
+
+DMG_DEPS = [
+    *list_files("install/macos"),
+    *TESSDATA_TARGETS,
+    *IMAGE_TARGETS,
+    *PYTHON_DEPS,
+    *SOURCE_DEPS,
+]
+
+LINUX_DEPS = [
+    *list_files("install/linux"),
+    *IMAGE_TARGETS,
+    *PYTHON_DEPS,
+    *SOURCE_DEPS,
+]
+
+DEB_DEPS = [*LINUX_DEPS, *list_files("debian")]
+RPM_DEPS = [*LINUX_DEPS, *list_files("qubes")]
+
+
+def copy_dir(src, dst):
+    """Copy a directory to a destination dir, and overwrite it if it exists."""
+    shutil.rmtree(dst, ignore_errors=True)
+    shutil.copytree(src, dst)
+
+
+def create_release_dir():
+    RELEASE_DIR.mkdir(parents=True, exist_ok=True)
+    (RELEASE_DIR / "tmp").mkdir(exist_ok=True)
+
+
+def build_linux_pkg(distro, version, cwd, qubes=False):
+    """Generic command for building a .deb/.rpm in a Dangerzone dev environment."""
+    pkg = "rpm" if distro == "fedora" else "deb"
+    cmd = [
+        "python3",
+        "./dev_scripts/env.py",
+        "--distro",
+        distro,
+        "--version",
+        version,
+        "run",
+        "--no-gui",
+        "--dev",
+        f"./dangerzone/install/linux/build-{pkg}.py",
+    ]
+    if qubes:
+        cmd += ["--qubes"]
+    return CmdAction(" ".join(cmd), cwd=cwd)
+
+
+def build_deb(cwd):
+    """Build a .deb package on Debian Bookworm."""
+    return build_linux_pkg(distro="debian", version="bookworm", cwd=cwd)
+
+
+def build_rpm(version, cwd, qubes=False):
+    """Build an .rpm package on the requested Fedora distro."""
+    return build_linux_pkg(distro="Fedora", version=version, cwd=cwd, qubes=qubes)
+
+
+### Tasks
+
+
+def task_clean_container_runtime():
+    """Clean the storage space of the container runtime."""
+    return {
+        "actions": None,
+        "clean": [
+            [CONTAINER_RUNTIME, "system", "prune", "-a", "-f"],
+        ],
+    }
+
+
+def task_check_container_runtime():
+    """Test that the container runtime is ready."""
+    return {
+        "actions": [
+            ["which", CONTAINER_RUNTIME],
+            [CONTAINER_RUNTIME, "ps"],
+        ],
+    }
+
+
+def task_macos_check_cert():
+    """Test that the Apple developer certificate can be used."""
+    return {
+        "actions": [
+            "xcrun notarytool history --apple-id %(apple_id)s --keychain-profile dz-notarytool-release-key"
+        ],
+        "params": [PARAM_APPLE_ID],
+    }
+
+
+def task_macos_check_system():
+    """Run macOS specific system checks, as well as the generic ones."""
+    return {
+        "actions": None,
+        "task_dep": ["check_container_runtime", "macos_check_cert"],
+    }
+
+
+def task_init_release_dir():
+    """Create a directory for release artifacts."""
+    return {
+        "actions": [create_release_dir],
+        "clean": [f"rm -rf {RELEASE_DIR}"],
+    }
+
+
+def task_download_tessdata():
+    """Download the Tesseract data using ./install/common/download-tessdata.py"""
+    return {
+        "actions": ["python install/common/download-tessdata.py"],
+        "file_dep": TESSDATA_DEPS,
+        "targets": TESSDATA_TARGETS,
+        "clean": True,
+    }
+
+
+def task_build_image():
+    """Build the container image using ./install/common/build-image.py"""
+    img_src = "share/container.tar.gz"
+    img_dst = RELEASE_DIR / f"container-{VERSION}-{ARCH}.tar.gz"  # FIXME: Add arch
+    img_id_src = "share/image-id.txt"
+    img_id_dst = RELEASE_DIR / "image-id.txt"  # FIXME: Add arch
+
+    return {
+        "actions": [
+            f"python install/common/build-image.py --use-cache=%(use_cache)s --runtime={CONTAINER_RUNTIME}",
+            ["cp", img_src, img_dst],
+            ["cp", img_id_src, img_id_dst],
+        ],
+        "params": [PARAM_USE_CACHE],
+        "file_dep": IMAGE_DEPS,
+        "targets": [img_src, img_dst, img_id_src, img_id_dst],
+        "task_dep": ["init_release_dir", "check_container_runtime"],
+        "clean": True,
+    }
+
+
+def task_poetry_install():
+    """Setup the Poetry environment"""
+    return {"actions": ["poetry install --sync"], "clean": ["poetry env remove --all"]}
+
+
+def task_macos_build_dmg():
+    """Build the macOS .dmg file for Dangerzone."""
+    dz_dir = RELEASE_DIR / "tmp" / "macos"
+    dmg_src = dz_dir / "dist" / "Dangerzone.dmg"
+    dmg_dst = RELEASE_DIR / f"Dangerzone-{VERSION}-{ARCH}.dmg"  # FIXME: Add -arch
+
+    return {
+        "actions": [
+            (copy_dir, [".", dz_dir]),
+            f"cd {dz_dir} && poetry run install/macos/build-app.py --with-codesign",
+            (
+                "xcrun notarytool submit --wait --apple-id %(apple_id)s"
+                f" --keychain-profile dz-notarytool-release-key {dmg_src}"
+            ),
+            f"xcrun stapler staple {dmg_src}",
+            ["cp", dmg_src, dmg_dst],
+            ["rm", "-rf", dz_dir],
+        ],
+        "params": [PARAM_APPLE_ID],
+        "file_dep": DMG_DEPS,
+        "task_dep": [
+            "macos_check_system",
+            "init_release_dir",
+            "poetry_install",
+            "download_tessdata",
+        ],
+        "targets": [dmg_src, dmg_dst],
+        "clean": True,
+    }
+
+
+def task_debian_env():
+    """Build a Debian Bookworm dev environment."""
+    return {
+        "actions": [
+            [
+                "python3",
+                "./dev_scripts/env.py",
+                "--distro",
+                "debian",
+                "--version",
+                "bookworm",
+                "build-dev",
+            ]
+        ],
+        "task_dep": ["check_container_runtime"],
+    }
+
+
+def task_debian_deb():
+    """Build a Debian package for Debian Bookworm."""
+    dz_dir = RELEASE_DIR / "tmp" / "debian"
+    deb_name = f"dangerzone_{VERSION}-1_amd64.deb"
+    deb_src = dz_dir / "deb_dist" / deb_name
+    deb_dst = RELEASE_DIR / deb_name
+
+    return {
+        "actions": [
+            (copy_dir, [".", dz_dir]),
+            build_deb(cwd=dz_dir),
+            ["cp", deb_src, deb_dst],
+            ["rm", "-rf", dz_dir],
+        ],
+        "file_dep": DEB_DEPS,
+        "task_dep": ["init_release_dir", "debian_env"],
+        "targets": [deb_dst],
+        "clean": True,
+    }
+
+
+def task_fedora_env():
+    """Build Fedora dev environments."""
+    for version in FEDORA_VERSIONS:
+        yield {
+            "name": version,
+            "doc": f"Build Fedora {version} dev environments",
+            "actions": [
+                [
+                    "python3",
+                    "./dev_scripts/env.py",
+                    "--distro",
+                    "fedora",
+                    "--version",
+                    version,
+                    "build-dev",
+                ],
+            ],
+            "task_dep": ["check_container_runtime"],
+        }
+
+
+def task_fedora_rpm():
+    """Build Fedora packages for every supported version."""
+    for version in FEDORA_VERSIONS:
+        for qubes in (True, False):
+            qubes_ident = "-qubes" if qubes else ""
+            qubes_desc = " for Qubes" if qubes else ""
+            dz_dir = RELEASE_DIR / "tmp" / f"f{version}{qubes_ident}"
+            rpm_names = [
+                f"dangerzone{qubes_ident}-{VERSION}-1.fc{version}.x86_64.rpm",
+                f"dangerzone{qubes_ident}-{VERSION}-1.fc{version}.src.rpm",
+            ]
+            rpm_src = [dz_dir / "dist" / rpm_name for rpm_name in rpm_names]
+            rpm_dst = [RELEASE_DIR / rpm_name for rpm_name in rpm_names]
+
+            yield {
+                "name": version + qubes_ident,
+                "doc": f"Build a Fedora {version} package{qubes_desc}",
+                "actions": [
+                    (copy_dir, [".", dz_dir]),
+                    build_rpm(version, cwd=dz_dir, qubes=qubes),
+                    ["cp", *rpm_src, RELEASE_DIR],
+                    ["rm", "-rf", dz_dir],
+                ],
+                "file_dep": RPM_DEPS,
+                "task_dep": ["init_release_dir", f"fedora_env:{version}"],
+                "targets": rpm_dst,
+                "clean": True,
+            }
+
+
+def task_git_archive():
+    """Build a Git archive of the repo."""
+    target = f"{RELEASE_DIR}/dangerzone-{VERSION}.tar.gz"
+    return {
+        "actions": [
+            f"git archive --format=tar.gz -o {target} --prefix=dangerzone/ v{VERSION}"
+        ],
+        "targets": [target],
+        "task_dep": ["init_release_dir"],
+    }
+
+
+#######################################################################################
+#
+#                              END OF TASKS
+#
+# The following task should be the LAST one in the dodo file, so that it runs first when
+# running `do clean`.
+
+
+def clean_prompt():
+    ans = input(
+        f"""
+You have not specified a target to clean.
+This means that doit will clean the following targets:
+
+* ALL the containers, images, and build cache in {CONTAINER_RUNTIME.capitalize()}
+* ALL the built targets and directories
+
+For a full list of the targets that doit will clean, run: doit clean --dry-run
+
+Are you sure you want to clean everything (y/N): \
+"""
+    )
+    if ans.lower() in ["yes", "y"]:
+        return
+    else:
+        print("Exiting...")
+        exit(1)
+
+
+def task_clean_prompt():
+    """Make sure that the user really wants to run the clean tasks."""
+    return {
+        "actions": None,
+        "clean": [clean_prompt],
+    }

install/common/build-image.py

@@ -2,12 +2,13 @@ import argparse
 import gzip
 import os
 import platform
+import secrets
 import subprocess
 import sys
 from pathlib import Path
 
 BUILD_CONTEXT = "dangerzone/"
-TAG = "dangerzone.rocks/dangerzone:latest"
+IMAGE_NAME = "dangerzone.rocks/dangerzone"
 REQUIREMENTS_TXT = "container-pip-requirements.txt"
 if platform.system() in ["Darwin", "Windows"]:
     CONTAINER_RUNTIME = "docker"
@@ -17,6 +18,17 @@ elif platform.system() == "Linux":
 ARCH = platform.machine()
 
 
+def str2bool(v):
+    if isinstance(v, bool):
+        return v
+    if v.lower() in ("yes", "true", "t", "y", "1"):
+        return True
+    elif v.lower() in ("no", "false", "f", "n", "0"):
+        return False
+    else:
+        raise argparse.ArgumentTypeError("Boolean value expected.")
+
+
 def main():
     parser = argparse.ArgumentParser()
     parser.add_argument(
@@ -39,13 +51,39 @@ def main():
     )
     parser.add_argument(
         "--use-cache",
-        action="store_true",
+        type=str2bool,
+        nargs="?",
+        default=False,
+        const=True,
         help="Use the builder's cache to speed up the builds (not suitable for release builds)",
     )
     args = parser.parse_args()
 
+    tarball_path = Path("share") / "container.tar.gz"
+    image_id_path = Path("share") / "image-id.txt"
+
     print(f"Building for architecture '{ARCH}'")
 
+    # Designate a unique tag for this image, depending on the Git commit it was created
+    # from:
+    # 1. If created from a Git tag (e.g., 0.8.0), the image tag will be `0.8.0`.
+    # 2. If created from a commit, it will be something like `0.8.0-31-g6bdaa7a`.
+    # 3. If the contents of the Git repo are dirty, we will append a unique identifier
+    #    for this run, something like `0.8.0-31-g6bdaa7a-fdcb` or `0.8.0-fdcb`.
+    dirty_ident = secrets.token_hex(2)
+    tag = (
+        subprocess.check_output(
+            ["git", "describe", "--long", "--first-parent", f"--dirty=-{dirty_ident}"],
+        )
+        .decode()
+        .strip()[1:]  # remove the "v" prefix of the tag.
+    )
+    image_name_tagged = IMAGE_NAME + ":" + tag
+
+    print(f"Will tag the container image as '{image_name_tagged}'")
+    with open(image_id_path, "w") as f:
+        f.write(tag)
+
     print("Exporting container pip dependencies")
     with ContainerPipDependencies():
         if not args.use_cache:
@@ -59,6 +97,7 @@ def main():
                 check=True,
             )
 
+        # Build the container image, and tag it with the calculated tag
         print("Building container image")
         cache_args = [] if args.use_cache else ["--no-cache"]
         subprocess.run(
@@ -74,7 +113,7 @@ def main():
                 "-f",
                 "Dockerfile",
                 "--tag",
-                TAG,
+                image_name_tagged,
             ],
             check=True,
         )
@@ -85,7 +124,7 @@ def main():
                 [
                     CONTAINER_RUNTIME,
                     "save",
-                    TAG,
+                    image_name_tagged,
                 ],
                 stdout=subprocess.PIPE,
             )
@@ -93,7 +132,7 @@ def main():
             print("Compressing container image")
             chunk_size = 4 << 20
             with gzip.open(
-                "share/container.tar.gz",
+                tarball_path,
                 "wb",
                 compresslevel=args.compress_level,
             ) as gzip_f:
@@ -105,21 +144,6 @@ def main():
                         break
             cmd.wait(5)
 
-    print("Looking up the image id")
-    image_id = subprocess.check_output(
-        [
-            args.runtime,
-            "image",
-            "list",
-            "--format",
-            "{{.ID}}",
-            TAG,
-        ],
-        text=True,
-    )
-    with open("share/image-id.txt", "w") as f:
-        f.write(image_id)
-
 
 class ContainerPipDependencies:
     """Generates PIP dependencies within container"""

poetry.lock

@@ -229,6 +229,17 @@ files = [
 [package.dependencies]
 colorama = {version = "*", markers = "platform_system == \"Windows\""}
 
+[[package]]
+name = "cloudpickle"
+version = "3.1.0"
+description = "Pickler class to extend the standard pickle.Pickler functionality"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "cloudpickle-3.1.0-py3-none-any.whl", hash = "sha256:fe11acda67f61aaaec473e3afe030feb131d78a43461b718185363384f1ba12e"},
+    {file = "cloudpickle-3.1.0.tar.gz", hash = "sha256:81a929b6e3c7335c863c771d673d105f02efdb89dfaba0c90495d1c64796601b"},
+]
+
 [[package]]
 name = "colorama"
 version = "0.4.6"
@@ -412,6 +423,24 @@ files = [
     {file = "cx_logging-3.2.1.tar.gz", hash = "sha256:812665ae5012680a6fe47095c3772bce638e47cf05b2c3483db3bdbe6b06da44"},
 ]
 
+[[package]]
+name = "doit"
+version = "0.36.0"
+description = "doit - Automation Tool"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "doit-0.36.0-py3-none-any.whl", hash = "sha256:ebc285f6666871b5300091c26eafdff3de968a6bd60ea35dd1e3fc6f2e32479a"},
+    {file = "doit-0.36.0.tar.gz", hash = "sha256:71d07ccc9514cb22fe59d98999577665eaab57e16f644d04336ae0b4bae234bc"},
+]
+
+[package.dependencies]
+cloudpickle = "*"
+importlib-metadata = ">=4.4"
+
+[package.extras]
+toml = ["tomli"]
+
 [[package]]
 name = "exceptiongroup"
 version = "1.2.2"
@@ -554,7 +583,6 @@ python-versions = ">=3.8"
 files = [
     {file = "lief-0.15.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a80246b96501b2b1d4927ceb3cb817eda9333ffa9e07101358929a6cffca5dae"},
     {file = "lief-0.15.1-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:84bf310710369544e2bb82f83d7fdab5b5ac422651184fde8bf9e35f14439691"},
-    {file = "lief-0.15.1-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:517dc5dad31c754720a80a87ad9e6cb1e48223d4505980c2fd86072bd4f69001"},
     {file = "lief-0.15.1-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:8fb58efb77358291109d2675d5459399c0794475b497992d0ecee18a4a46a207"},
     {file = "lief-0.15.1-cp310-cp310-manylinux_2_33_aarch64.whl", hash = "sha256:d5852a246361bbefa4c1d5930741765a2337638d65cfe30de1b7d61f9a54b865"},
     {file = "lief-0.15.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:12e53dc0253c303df386ae45487a2f0078026602b36d0e09e838ae1d4dbef958"},
@@ -562,7 +590,6 @@ files = [
     {file = "lief-0.15.1-cp310-cp310-win_amd64.whl", hash = "sha256:ddf2ebd73766169594d631b35f84c49ef42871de552ad49f36002c60164d0aca"},
     {file = "lief-0.15.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:20508c52de0dffcee3242253541609590167a3e56150cbacb506fdbb822206ef"},
     {file = "lief-0.15.1-cp311-cp311-macosx_11_0_x86_64.whl", hash = "sha256:0750c892fd3b7161a3c2279f25fe1844427610c3a5a4ae23f65674ced6f93ea5"},
-    {file = "lief-0.15.1-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:3e49bd595a8548683bead982bc15b064257fea3110fd15e22fb3feb17d97ad1c"},
     {file = "lief-0.15.1-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:a8634ea79d6d9862297fadce025519ab25ff01fcadb333cf42967c6295f0d057"},
     {file = "lief-0.15.1-cp311-cp311-manylinux_2_33_aarch64.whl", hash = "sha256:1e11e046ad71fe8c81e1a8d1d207fe2b99c967d33ce79c3d3915cb8f5ecacf52"},
     {file = "lief-0.15.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:674b620cdf1d686f52450fd97c1056d4c92e55af8217ce85a1b2efaf5b32140b"},
@@ -570,15 +597,11 @@ files = [
     {file = "lief-0.15.1-cp311-cp311-win_amd64.whl", hash = "sha256:e9b96a37bf11ca777ff305d85d957eabad2a92a6e577b6e2fb3ab79514e5a12e"},
     {file = "lief-0.15.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1a96f17c2085ef38d12ad81427ae8a5d6ad76f0bc62a1e1f5fe384255cd2cc94"},
     {file = "lief-0.15.1-cp312-cp312-macosx_11_0_x86_64.whl", hash = "sha256:d780af1762022b8e01b613253af490afea3864fbd6b5a49c6de7cea8fde0443d"},
-    {file = "lief-0.15.1-cp312-cp312-manylinux2014_aarch64.whl", hash = "sha256:536a4ecd46b295b3acac0d60a68d1646480b7761ade862c6c87ccbb41229fae3"},
     {file = "lief-0.15.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:d0f10d80202de9634a16786b53ba3a8f54ae8b9a9e124a964d83212444486087"},
     {file = "lief-0.15.1-cp312-cp312-manylinux_2_33_aarch64.whl", hash = "sha256:864f17ecf1736296e6d5fc38b11983f9d19a5e799f094e21e20d58bfb1b95b80"},
     {file = "lief-0.15.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c2ec738bcafee8a569741f4a749f0596823b12f10713306c7d0cbbf85759f51c"},
     {file = "lief-0.15.1-cp312-cp312-win32.whl", hash = "sha256:db38619edf70e27fb3686b8c0f0bec63ad494ac88ab51660c5ecd2720b506e41"},
     {file = "lief-0.15.1-cp312-cp312-win_amd64.whl", hash = "sha256:28bf0922de5fb74502a29cc47930d3a052df58dc23ab6519fa590e564f194a60"},
-    {file = "lief-0.15.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0805301e8fef9b13da00c33c831fb0c05ea892309230f3a35551c2dfaf69b11d"},
-    {file = "lief-0.15.1-cp313-cp313-macosx_11_0_x86_64.whl", hash = "sha256:7580defe140e921bc4f210e8a6cb115fcf2923f00d37800b1626168cbca95108"},
-    {file = "lief-0.15.1-cp313-cp313-manylinux2014_aarch64.whl", hash = "sha256:c0119306b6a38759483136de7242b7c2e0a23f1de1d4ae53f12792c279607410"},
     {file = "lief-0.15.1-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:0616e6048f269d262ff93d67c497ebff3c1d3965ffb9427b0f2b474764fd2e8c"},
     {file = "lief-0.15.1-cp313-cp313-manylinux_2_33_aarch64.whl", hash = "sha256:6a08b2e512a80040429febddc777768c949bcd53f6f580e902e41ec0d9d936b8"},
     {file = "lief-0.15.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:fcd489ff80860bcc2b2689faa330a46b6d66f0ee3e0f6ef9e643e2b996128a06"},
@@ -586,7 +609,6 @@ files = [
     {file = "lief-0.15.1-cp313-cp313-win_amd64.whl", hash = "sha256:5af7dcb9c3f44baaf60875df6ba9af6777db94776cc577ee86143bcce105ba2f"},
     {file = "lief-0.15.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:f9757ff0c7c3d6f66e5fdcc6a9df69680fad0dc2707d64a3428f0825dfce1a85"},
     {file = "lief-0.15.1-cp38-cp38-macosx_11_0_x86_64.whl", hash = "sha256:8ac3cd099be2580d0e15150b1d2f5095c38f150af89993ddf390d7897ee8135f"},
-    {file = "lief-0.15.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:e732619acc34943b504c867258fc0196f1931f72c2a627219d4f116a7acc726d"},
     {file = "lief-0.15.1-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:4dedeab498c312a29b58f16b739895f65fa54b2a21b8d98b111e99ad3f7e30a8"},
     {file = "lief-0.15.1-cp38-cp38-manylinux_2_33_aarch64.whl", hash = "sha256:b9217578f7a45f667503b271da8481207fb4edda8d4a53e869fb922df6030484"},
     {file = "lief-0.15.1-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:82e6308ad8bd4bc7eadee3502ede13a5bb398725f25513a0396c8dba850f58a1"},
@@ -594,7 +616,6 @@ files = [
     {file = "lief-0.15.1-cp38-cp38-win_amd64.whl", hash = "sha256:a079a76bca23aa73c850ab5beb7598871a1bf44662658b952cead2b5ddd31bee"},
     {file = "lief-0.15.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:785a3aa14575f046ed9c8d44ea222ea14c697cd03b5331d1717b5b0cf4f72466"},
     {file = "lief-0.15.1-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:d7044553cf07c8a2ab6e21874f07585610d996ff911b9af71dc6085a89f59daa"},
-    {file = "lief-0.15.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:fa020f3ed6e95bb110a4316af544021b74027d18bf4671339d4cffec27aa5884"},
     {file = "lief-0.15.1-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:13285c3ff5ef6de2421d85684c954905af909db0ad3472e33c475e5f0f657dcf"},
     {file = "lief-0.15.1-cp39-cp39-manylinux_2_33_aarch64.whl", hash = "sha256:932f880ee8a130d663a97a9099516d8570b1b303af7816e70a02f9931d5ef4c2"},
     {file = "lief-0.15.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:de9453f94866e0f2c36b6bd878625880080e7e5800788f5cbc06a76debf283b9"},
@@ -1189,4 +1210,4 @@ type = ["pytest-mypy"]
 [metadata]
 lock-version = "2.0"
 python-versions = ">=3.9,<3.13"
-content-hash = "5d1ff28aa04c3a814280e55c0b2a307efe5ca953cd4cb281056c35fd2e53fdf0"
+content-hash = "a2937fd8ead7b45da571cb943ab43918a9c6d3dcbc6935dc8d0af3d1d4190371"

pyproject.toml

@@ -34,6 +34,7 @@ setuptools = "*"
 cx_freeze = {version = "^7.2.5", platform = "win32"}
 pywin32 = {version = "*", platform = "win32"}
 pyinstaller = {version = "*", platform = "darwin"}
+doit = "^0.36.0"
 
 # Dependencies required for linting the code.
 [tool.poetry.group.lint.dependencies]
@@ -66,6 +67,14 @@ skip_gitignore = true
 # This is necessary due to https://github.com/PyCQA/isort/issues/1835
 follow_links = false
 
+[tool.doit]
+verbosity = 3
+
+[tool.doit.tasks.build_image]
+# DO NOT change this to 'true' for release artifacts, else we risk building
+# images that are a few days behind. See also: docs/developer/doit.md
+use_cache = false
+
 [build-system]
 requires = ["poetry-core>=1.2.0"]
 build-backend = "poetry.core.masonry.api"

tests/gui/test_main_window.py

@@ -10,6 +10,7 @@ from pytest_mock import MockerFixture
 from pytest_subprocess import FakeProcess
 from pytestqt.qtbot import QtBot
 
+from dangerzone import errors
 from dangerzone.document import Document
 from dangerzone.gui import MainWindow
 from dangerzone.gui import main_window as main_window_module
@@ -25,11 +26,8 @@ from dangerzone.gui.main_window import (
     WaitingWidgetContainer,
 )
 from dangerzone.gui.updater import UpdateReport, UpdaterThread
-from dangerzone.isolation_provider.container import (
-    Container,
-    NoContainerTechException,
-    NotAvailableContainerTechException,
-)
+from dangerzone.isolation_provider.container import Container
+from dangerzone.isolation_provider.dummy import Dummy
 
 from .test_updater import assert_report_equal, default_updater_settings
 
@@ -510,9 +508,9 @@ def test_not_available_container_tech_exception(
 ) -> None:
     # Setup
     mock_app = mocker.MagicMock()
-    dummy = mocker.MagicMock()
-
-    dummy.is_runtime_available.side_effect = NotAvailableContainerTechException(
+    dummy = Dummy()
+    fn = mocker.patch.object(dummy, "is_available")
+    fn.side_effect = errors.NotAvailableContainerTechException(
         "podman", "podman image ls logs"
     )
 
@@ -535,7 +533,7 @@ def test_no_container_tech_exception(qtbot: QtBot, mocker: MockerFixture) -> Non
     dummy = mocker.MagicMock()
 
     # Raise
-    dummy.is_runtime_available.side_effect = NoContainerTechException("podman")
+    dummy.is_available.side_effect = errors.NoContainerTechException("podman")
 
     dz = DangerzoneGui(mock_app, dummy)
     widget = WaitingWidgetContainer(dz)

tests/isolation_provider/test_container.py

@@ -4,12 +4,8 @@ import pytest
 from pytest_mock import MockerFixture
 from pytest_subprocess import FakeProcess
 
-from dangerzone.isolation_provider.container import (
-    Container,
-    ImageInstallationException,
-    ImageNotPresentException,
-    NotAvailableContainerTechException,
-)
+from dangerzone import container_utils, errors
+from dangerzone.isolation_provider.container import Container
 from dangerzone.isolation_provider.qubes import is_qubes_native_conversion
 
 from .base import IsolationProviderTermination, IsolationProviderTest
@@ -27,31 +23,27 @@ def provider() -> Container:
 
 
 class TestContainer(IsolationProviderTest):
-    def test_is_runtime_available_raises(
-        self, provider: Container, fp: FakeProcess
-    ) -> None:
+    def test_is_available_raises(self, provider: Container, fp: FakeProcess) -> None:
         """
         NotAvailableContainerTechException should be raised when
         the "podman image ls" command fails.
         """
         fp.register_subprocess(
-            [provider.get_runtime(), "image", "ls"],
+            [container_utils.get_runtime(), "image", "ls"],
             returncode=-1,
             stderr="podman image ls logs",
         )
-        with pytest.raises(NotAvailableContainerTechException):
-            provider.is_runtime_available()
+        with pytest.raises(errors.NotAvailableContainerTechException):
+            provider.is_available()
 
-    def test_is_runtime_available_works(
-        self, provider: Container, fp: FakeProcess
-    ) -> None:
+    def test_is_available_works(self, provider: Container, fp: FakeProcess) -> None:
         """
         No exception should be raised when the "podman image ls" can return properly.
         """
         fp.register_subprocess(
-            [provider.get_runtime(), "image", "ls"],
+            [container_utils.get_runtime(), "image", "ls"],
         )
-        provider.is_runtime_available()
+        provider.is_available()
 
     def test_install_raise_if_image_cant_be_installed(
         self, mocker: MockerFixture, provider: Container, fp: FakeProcess
@@ -59,17 +51,17 @@ class TestContainer(IsolationProviderTest):
         """When an image installation fails, an exception should be raised"""
 
         fp.register_subprocess(
-            [provider.get_runtime(), "image", "ls"],
+            [container_utils.get_runtime(), "image", "ls"],
         )
 
         # First check should return nothing.
         fp.register_subprocess(
             [
-                provider.get_runtime(),
+                container_utils.get_runtime(),
                 "image",
                 "list",
                 "--format",
-                "{{.ID}}",
+                "{{ .Tag }}",
                 "dangerzone.rocks/dangerzone",
             ],
             occurrences=2,
@@ -79,11 +71,11 @@ class TestContainer(IsolationProviderTest):
         mocker.patch("gzip.open", mocker.mock_open(read_data=""))
 
         fp.register_subprocess(
-            [provider.get_runtime(), "load"],
+            [container_utils.get_runtime(), "load"],
             returncode=-1,
         )
 
-        with pytest.raises(ImageInstallationException):
+        with pytest.raises(errors.ImageInstallationException):
             provider.install()
 
     def test_install_raises_if_still_not_installed(
@@ -92,17 +84,17 @@ class TestContainer(IsolationProviderTest):
         """When an image keep being not installed, it should return False"""
 
         fp.register_subprocess(
-            [provider.get_runtime(), "image", "ls"],
+            [container_utils.get_runtime(), "image", "ls"],
         )
 
         # First check should return nothing.
         fp.register_subprocess(
             [
-                provider.get_runtime(),
+                container_utils.get_runtime(),
                 "image",
                 "list",
                 "--format",
-                "{{.ID}}",
+                "{{ .Tag }}",
                 "dangerzone.rocks/dangerzone",
             ],
             occurrences=2,
@@ -111,9 +103,9 @@ class TestContainer(IsolationProviderTest):
         # Patch gzip.open and podman load so that it works
         mocker.patch("gzip.open", mocker.mock_open(read_data=""))
         fp.register_subprocess(
-            [provider.get_runtime(), "load"],
+            [container_utils.get_runtime(), "load"],
         )
-        with pytest.raises(ImageNotPresentException):
+        with pytest.raises(errors.ImageNotPresentException):
             provider.install()