mirror of
https://github.com/freedomofpress/dangerzone.git
synced 2025-05-03 04:01:49 +02:00
814d533c3b
The files in `container/` no longer make sense to have that name since
the "document to pixels" part will run in Qubes OS in its own virtual
machine.
To adapt to this, this PR does the following:
- Moves all the files in `container` to `dangerzone/conversion`
- Splits the old `container/dangerzone.py` into its two components
`dangerzone/conversion/{doc_to_pixels,pixels_to_pdf}.py` with a
`common.py` file for shared functions
- Moves the Dockerfile to the project root and adapts it to the new
container code location
- Updates the CircleCI config to properly cache Docker images.
- Updates our install scripts to properly build Docker images.
- Adds the new conversion module to the container image, so that it can
be imported as a package.
- Adapts the container isolation provider to use the new way of calling
the code.
NOTE: We have made zero changes to the conversion code in this commit,
except for necessary imports in order to factor out some common parts.
Any changes necessary for Qubes integration follow in the subsequent
commits.
515 lines
15 KiB
YAML
515 lines
15 KiB
YAML
version: 2.1
|
|
|
|
aliases:
|
|
- &install-podman
|
|
name: Install Podman in Ubuntu Focal
|
|
command: ./install/linux/install-podman-ubuntu-focal.sh
|
|
|
|
# FIXME: Remove the following step once we drop Ubuntu Focal support. The
|
|
# python-all dependency is an artificial requirement due to an stdeb bug
|
|
# prior to v0.9.1. See:
|
|
#
|
|
# * https://github.com/astraw/stdeb/issues/153
|
|
# * https://github.com/freedomofpress/dangerzone/issues/292#issuecomment-1349967888
|
|
- &install-python-all
|
|
name: Install python-all package
|
|
command: |
|
|
export DEBIAN_FRONTEND=noninteractive DEBCONF_NONINTERACTIVE_SEEN=true
|
|
apt-get update
|
|
apt-get install -y python-all
|
|
|
|
- &install-dependencies-deb
|
|
name: Install dependencies (deb)
|
|
command: |
|
|
export DEBIAN_FRONTEND=noninteractive DEBCONF_NONINTERACTIVE_SEEN=true
|
|
apt-get update
|
|
apt-get install -y dh-python python3 python3-stdeb
|
|
|
|
- &install-dependencies-rpm
|
|
name: Install dependencies (rpm)
|
|
command: |
|
|
dnf install -y rpm-build python3 python3-setuptools
|
|
|
|
- &build-deb
|
|
name: Build the .deb package
|
|
command: |
|
|
./install/linux/build-deb.py
|
|
ls -lh deb_dist/
|
|
|
|
- &build-rpm
|
|
name: Build the .rpm package
|
|
command: |
|
|
./install/linux/build-rpm.py
|
|
ls -lh dist/
|
|
|
|
- &calculate-cache-key
|
|
name: Caculating container cache key
|
|
command: |
|
|
mkdir -p /caches/
|
|
cd dangerzone/conversion/
|
|
cat common.py doc_to_pixels.py pixels_to_pdf.py | sha1sum | cut -d' ' -f1 > /caches/cache-id.txt
|
|
cd ../../
|
|
|
|
- &restore-cache
|
|
key: v1-{{ checksum "Dockerfile" }}-{{ checksum "/caches/cache-id.txt" }}
|
|
paths:
|
|
- /caches/container.tar.gz
|
|
- /caches/image-id.txt
|
|
|
|
- ©-image
|
|
name: Copy container image into package
|
|
command: |
|
|
cp /caches/container.tar.gz share/
|
|
cp /caches/image-id.txt share/
|
|
|
|
jobs:
|
|
run-lint:
|
|
docker:
|
|
- image: debian:bookworm
|
|
resource_class: small
|
|
steps:
|
|
- checkout
|
|
- run:
|
|
name: Install dev. dependencies
|
|
# Install only the necessary packages to run our linters.
|
|
#
|
|
# We run poetry with --no-ansi, to sidestep a Poetry bug that
|
|
# currently exists in 1.3. See:
|
|
# https://github.com/freedomofpress/dangerzone/issues/292#issuecomment-1351368122
|
|
command: |
|
|
apt-get update
|
|
apt-get install -y make python3 python3-poetry --no-install-recommends
|
|
poetry install --no-ansi --only lint
|
|
- run:
|
|
name: Run linters to enforce code style
|
|
command: poetry run make lint
|
|
- run:
|
|
name: Check that the QA script is up to date with the docs
|
|
command: ./dev_scripts/qa.py --check-refs
|
|
|
|
build-container-image:
|
|
working_directory: /app
|
|
docker:
|
|
- image: docker:dind
|
|
steps:
|
|
- checkout
|
|
- run: *calculate-cache-key
|
|
- restore_cache: *restore-cache
|
|
- setup_remote_docker
|
|
- run:
|
|
name: Build Dangerzone image
|
|
command: |
|
|
if [ -f "/caches/container.tar.gz" ]; then
|
|
echo "Already cached, skipping"
|
|
else
|
|
docker build dangerzone/ -f Dockerfile \
|
|
--cache-from=dangerzone.rocks/dangerzone \
|
|
--tag dangerzone.rocks/dangerzone
|
|
fi
|
|
- run:
|
|
name: Save Dangerzone image and image-id.txt to cache
|
|
command: |
|
|
if [ -f "/caches/container.tar.gz" ]; then
|
|
echo "Already cached, skipping"
|
|
else
|
|
mkdir -p /caches
|
|
docker save -o /caches/container.tar dangerzone.rocks/dangerzone
|
|
gzip -f /caches/container.tar
|
|
docker image ls dangerzone.rocks/dangerzone | grep "dangerzone.rocks/dangerzone" | tr -s ' ' | cut -d' ' -f3 > /caches/image-id.txt
|
|
fi
|
|
- run: *calculate-cache-key
|
|
- save_cache:
|
|
key: v1-{{ checksum "Dockerfile" }}-{{ checksum "/caches/cache-id.txt" }}
|
|
paths:
|
|
- /caches/container.tar.gz
|
|
- /caches/image-id.txt
|
|
|
|
convert-test-docs:
|
|
machine:
|
|
image: ubuntu-2004:202111-01
|
|
steps:
|
|
- checkout
|
|
- run: *install-podman
|
|
- run:
|
|
name: Install poetry dependencies
|
|
command: |
|
|
sudo pip3 install poetry
|
|
# This flag is important, due to an open upstream Poetry issue:
|
|
# https://github.com/python-poetry/poetry/issues/7184
|
|
poetry install --no-ansi
|
|
- run:
|
|
name: Install test dependencies
|
|
command: |
|
|
sudo apt-get install -y libqt5gui5 --no-install-recommends
|
|
- run:
|
|
name: Prepare cache directory
|
|
command: |
|
|
sudo mkdir -p /caches
|
|
sudo chown -R $USER:$USER /caches
|
|
- run: *calculate-cache-key
|
|
- restore_cache: *restore-cache
|
|
- run: *copy-image
|
|
- run:
|
|
name: run automated tests
|
|
command: |
|
|
poetry run make test
|
|
|
|
ci-ubuntu-kinetic:
|
|
machine:
|
|
image: ubuntu-2004:202111-01
|
|
steps:
|
|
- checkout
|
|
- run: *install-podman
|
|
|
|
- run:
|
|
name: Prepare cache directory
|
|
command: |
|
|
sudo mkdir -p /caches
|
|
sudo chown -R $USER:$USER /caches
|
|
- run: *calculate-cache-key
|
|
- restore_cache: *restore-cache
|
|
- run: *copy-image
|
|
|
|
- run:
|
|
name: Prepare Dangerzone environment
|
|
command: |
|
|
./dev_scripts/env.py --distro ubuntu --version 22.10 build-dev
|
|
|
|
- run:
|
|
name: Run CI tests
|
|
command: |
|
|
./dev_scripts/env.py --distro ubuntu --version 22.10 run --dev \
|
|
bash -c 'cd dangerzone; poetry run make test'
|
|
|
|
ci-ubuntu-jammy:
|
|
machine:
|
|
image: ubuntu-2004:202111-01
|
|
steps:
|
|
- checkout
|
|
- run: *install-podman
|
|
|
|
- run:
|
|
name: Prepare cache directory
|
|
command: |
|
|
sudo mkdir -p /caches
|
|
sudo chown -R $USER:$USER /caches
|
|
- run: *calculate-cache-key
|
|
- restore_cache: *restore-cache
|
|
- run: *copy-image
|
|
|
|
- run:
|
|
name: Prepare Dangerzone environment
|
|
command: |
|
|
./dev_scripts/env.py --distro ubuntu --version 22.04 build-dev
|
|
|
|
- run:
|
|
name: Run CI tests
|
|
command: |
|
|
./dev_scripts/env.py --distro ubuntu --version 22.04 run --dev \
|
|
bash -c 'cd dangerzone; poetry run make test'
|
|
|
|
ci-ubuntu-focal:
|
|
machine:
|
|
image: ubuntu-2004:202111-01
|
|
steps:
|
|
- checkout
|
|
- run: *install-podman
|
|
|
|
- run:
|
|
name: Prepare cache directory
|
|
command: |
|
|
sudo mkdir -p /caches
|
|
sudo chown -R $USER:$USER /caches
|
|
- run: *calculate-cache-key
|
|
- restore_cache: *restore-cache
|
|
- run: *copy-image
|
|
|
|
- run:
|
|
name: Prepare Dangerzone environment
|
|
command: |
|
|
./dev_scripts/env.py --distro ubuntu --version 20.04 build-dev
|
|
|
|
- run:
|
|
name: Run CI tests
|
|
command: |
|
|
./dev_scripts/env.py --distro ubuntu --version 20.04 run --dev \
|
|
bash -c 'cd dangerzone; poetry run make test'
|
|
|
|
ci-fedora-38:
|
|
machine:
|
|
image: ubuntu-2004:202111-01
|
|
steps:
|
|
- checkout
|
|
- run: *install-podman
|
|
|
|
- run:
|
|
name: Prepare cache directory
|
|
command: |
|
|
sudo mkdir -p /caches
|
|
sudo chown -R $USER:$USER /caches
|
|
- run: *calculate-cache-key
|
|
- restore_cache: *restore-cache
|
|
- run: *copy-image
|
|
|
|
- run:
|
|
name: Prepare Dangerzone environment
|
|
command: |
|
|
./dev_scripts/env.py --distro fedora --version 38 build-dev
|
|
|
|
- run:
|
|
name: Run CI tests
|
|
command: |
|
|
./dev_scripts/env.py --distro fedora --version 38 run --dev \
|
|
bash -c 'cd dangerzone; poetry run make test'
|
|
|
|
ci-fedora-37:
|
|
machine:
|
|
image: ubuntu-2004:202111-01
|
|
steps:
|
|
- checkout
|
|
- run: *install-podman
|
|
|
|
- run:
|
|
name: Prepare cache directory
|
|
command: |
|
|
sudo mkdir -p /caches
|
|
sudo chown -R $USER:$USER /caches
|
|
- run: *calculate-cache-key
|
|
- restore_cache: *restore-cache
|
|
- run: *copy-image
|
|
|
|
- run:
|
|
name: Prepare Dangerzone environment
|
|
command: |
|
|
./dev_scripts/env.py --distro fedora --version 37 build-dev
|
|
|
|
- run:
|
|
name: Run CI tests
|
|
command: |
|
|
./dev_scripts/env.py --distro fedora --version 37 run --dev \
|
|
bash -c 'cd dangerzone; poetry run make test'
|
|
|
|
ci-debian-bookworm:
|
|
machine:
|
|
image: ubuntu-2004:202111-01
|
|
steps:
|
|
- checkout
|
|
- run: *install-podman
|
|
|
|
- run:
|
|
name: Prepare cache directory
|
|
command: |
|
|
sudo mkdir -p /caches
|
|
sudo chown -R $USER:$USER /caches
|
|
- run: *calculate-cache-key
|
|
- restore_cache: *restore-cache
|
|
- run: *copy-image
|
|
|
|
- run:
|
|
name: Prepare Dangerzone environment
|
|
command: |
|
|
./dev_scripts/env.py --distro debian --version bookworm build-dev
|
|
|
|
- run:
|
|
name: Run CI tests
|
|
command: |
|
|
./dev_scripts/env.py --distro debian --version bookworm run --dev \
|
|
bash -c 'cd dangerzone; poetry run make test'
|
|
|
|
# NOTE: Making CI tests work in Debian Bullseye requires some tip-toeing
|
|
# around certain Podman issues, as you'll see below. Read the following for
|
|
# more details:
|
|
#
|
|
# https://github.com/freedomofpress/dangerzone/issues/388
|
|
ci-debian-bullseye:
|
|
machine:
|
|
image: ubuntu-2204:2023.04.2
|
|
steps:
|
|
- checkout
|
|
- run: *install-podman
|
|
- run:
|
|
name: Configure Podman for Ubuntu 22.04
|
|
command: |
|
|
# This config circumvents the following issues:
|
|
# * https://github.com/containers/podman/issues/6368
|
|
# * https://github.com/containers/podman/issues/10987
|
|
mkdir -p ~/.config/containers
|
|
cat > ~/.config/containers/containers.conf \<<EOF
|
|
[engine]
|
|
cgroup_manager="cgroupfs"
|
|
events_logger="file"
|
|
EOF
|
|
|
|
- run:
|
|
name: Prepare cache directory
|
|
command: |
|
|
sudo mkdir -p /caches
|
|
sudo chown -R $USER:$USER /caches
|
|
- run: *calculate-cache-key
|
|
- restore_cache: *restore-cache
|
|
- run: *copy-image
|
|
|
|
- run:
|
|
name: Prepare Dangerzone environment
|
|
command: |
|
|
./dev_scripts/env.py --distro debian --version bullseye build-dev
|
|
|
|
- run:
|
|
name: Configure Podman for Debian Bullseye
|
|
command: |
|
|
# Copy the Podman config into the container image we created for the
|
|
# Dangerzone environment.
|
|
cp ~/.config/containers/containers.conf containers.conf
|
|
cat > Dockerfile.bullseye \<<EOF
|
|
FROM dangerzone.rocks/build/debian:bullseye-backports
|
|
RUN mkdir -p /home/user/.config/containers
|
|
COPY containers.conf /home/user/.config/containers/
|
|
EOF
|
|
|
|
# Create a new image from the Dangerzone environment and re-tag it.
|
|
podman build -t dangerzone.rocks/build/debian:bullseye-backports \
|
|
-f Dockerfile.bullseye .
|
|
|
|
- run:
|
|
name: Run CI tests
|
|
command: |
|
|
./dev_scripts/env.py --distro debian --version bullseye run --dev \
|
|
bash -c 'cd dangerzone; poetry run make test'
|
|
|
|
build-ubuntu-kinetic:
|
|
docker:
|
|
- image: ubuntu:22.10
|
|
resource_class: medium+
|
|
steps:
|
|
- run: *install-dependencies-deb
|
|
- checkout
|
|
- run: *calculate-cache-key
|
|
- restore_cache: *restore-cache
|
|
- run: *copy-image
|
|
- run: *build-deb
|
|
|
|
build-ubuntu-jammy:
|
|
docker:
|
|
- image: ubuntu:22.04
|
|
resource_class: medium+
|
|
steps:
|
|
- run: *install-dependencies-deb
|
|
- checkout
|
|
- run: *calculate-cache-key
|
|
- restore_cache: *restore-cache
|
|
- run: *copy-image
|
|
- run: *build-deb
|
|
|
|
build-ubuntu-focal:
|
|
docker:
|
|
- image: ubuntu:20.04
|
|
resource_class: medium+
|
|
steps:
|
|
- run: *install-dependencies-deb
|
|
- run: *install-python-all
|
|
- checkout
|
|
- run: *calculate-cache-key
|
|
- restore_cache: *restore-cache
|
|
- run: *copy-image
|
|
- run: *build-deb
|
|
|
|
build-debian-bookworm:
|
|
docker:
|
|
- image: debian:bookworm
|
|
resource_class: medium+
|
|
steps:
|
|
- run: *install-dependencies-deb
|
|
- checkout
|
|
- run: *calculate-cache-key
|
|
- restore_cache: *restore-cache
|
|
- run: *copy-image
|
|
- run: *build-deb
|
|
|
|
build-debian-bullseye:
|
|
docker:
|
|
- image: debian:bullseye
|
|
resource_class: medium+
|
|
steps:
|
|
- run: *install-dependencies-deb
|
|
- checkout
|
|
- run: *calculate-cache-key
|
|
- restore_cache: *restore-cache
|
|
- run: *copy-image
|
|
- run: *build-deb
|
|
|
|
build-fedora-38:
|
|
docker:
|
|
- image: fedora:38
|
|
resource_class: medium+
|
|
steps:
|
|
- run: *install-dependencies-rpm
|
|
- checkout
|
|
- run: *calculate-cache-key
|
|
- restore_cache: *restore-cache
|
|
- run: *copy-image
|
|
- run: *build-rpm
|
|
|
|
build-fedora-37:
|
|
docker:
|
|
- image: fedora:37
|
|
resource_class: medium+
|
|
steps:
|
|
- run: *install-dependencies-rpm
|
|
- checkout
|
|
- run: *calculate-cache-key
|
|
- restore_cache: *restore-cache
|
|
- run: *copy-image
|
|
- run: *build-rpm
|
|
|
|
workflows:
|
|
version: 2
|
|
|
|
build:
|
|
jobs:
|
|
- run-lint
|
|
- build-container-image
|
|
- convert-test-docs:
|
|
requires:
|
|
- build-container-image
|
|
- ci-ubuntu-kinetic:
|
|
requires:
|
|
- build-container-image
|
|
- ci-ubuntu-jammy:
|
|
requires:
|
|
- build-container-image
|
|
- ci-ubuntu-focal:
|
|
requires:
|
|
- build-container-image
|
|
- ci-debian-bookworm:
|
|
requires:
|
|
- build-container-image
|
|
- ci-debian-bullseye:
|
|
requires:
|
|
- build-container-image
|
|
- ci-fedora-38:
|
|
requires:
|
|
- build-container-image
|
|
- ci-fedora-37:
|
|
requires:
|
|
- build-container-image
|
|
- build-ubuntu-kinetic:
|
|
requires:
|
|
- build-container-image
|
|
- build-ubuntu-jammy:
|
|
requires:
|
|
- build-container-image
|
|
- build-ubuntu-focal:
|
|
requires:
|
|
- build-container-image
|
|
- build-debian-bullseye:
|
|
requires:
|
|
- build-container-image
|
|
- build-debian-bookworm:
|
|
requires:
|
|
- build-container-image
|
|
- build-fedora-38:
|
|
requires:
|
|
- build-container-image
|
|
- build-fedora-37:
|
|
requires:
|
|
- build-container-image
|