mirror of
https://github.com/freedomofpress/dangerzone.git
synced 2025-04-28 18:02:38 +02:00
Compare commits
No commits in common. "main" and "v0.6.1" have entirely different histories.
147 changed files with 4080 additions and 10796 deletions
669
.circleci/config.yml
Normal file
669
.circleci/config.yml
Normal file
|
|
@ -0,0 +1,669 @@
|
|||
version: 2.1
|
||||
|
||||
aliases:
|
||||
- &install-podman
|
||||
name: Install Podman in Ubuntu Focal
|
||||
command: ./install/linux/install-podman-ubuntu-focal.sh
|
||||
|
||||
# FIXME: Remove the following step once we drop Ubuntu Focal support. The
|
||||
# python-all dependency is an artificial requirement due to an stdeb bug
|
||||
# prior to v0.9.1. See:
|
||||
#
|
||||
# * https://github.com/astraw/stdeb/issues/153
|
||||
# * https://github.com/freedomofpress/dangerzone/issues/292#issuecomment-1349967888
|
||||
- &install-python-all
|
||||
name: Install python-all package
|
||||
command: |
|
||||
export DEBIAN_FRONTEND=noninteractive DEBCONF_NONINTERACTIVE_SEEN=true
|
||||
apt-get update
|
||||
apt-get install -y python-all
|
||||
|
||||
- &install-dependencies-deb
|
||||
name: Install dependencies (deb)
|
||||
command: |
|
||||
export DEBIAN_FRONTEND=noninteractive DEBCONF_NONINTERACTIVE_SEEN=true
|
||||
apt-get update
|
||||
apt-get install -y dh-python python3 python3-stdeb
|
||||
|
||||
- &install-dependencies-rpm
|
||||
name: Install dependencies (rpm)
|
||||
command: |
|
||||
dnf install -y rpm-build python3 python3-devel python3-poetry-core pipx
|
||||
pipx install poetry
|
||||
|
||||
- &build-deb
|
||||
name: Build the .deb package
|
||||
command: |
|
||||
./install/linux/build-deb.py
|
||||
ls -lh deb_dist/
|
||||
|
||||
- &build-rpm
|
||||
name: Build the .rpm package
|
||||
command: |
|
||||
PATH=/root/.local/bin:$PATH ./install/linux/build-rpm.py
|
||||
ls -lh dist/
|
||||
|
||||
- &build-rpm-qubes
|
||||
name: Build the Qubes .rpm package
|
||||
command: |
|
||||
PATH=/root/.local/bin:$PATH ./install/linux/build-rpm.py --qubes
|
||||
ls -lh dist/
|
||||
|
||||
- &calculate-cache-key
|
||||
name: Caculating container cache key
|
||||
command: |
|
||||
mkdir -p /caches/
|
||||
cd dangerzone/conversion/
|
||||
cat common.py doc_to_pixels.py pixels_to_pdf.py | sha1sum | cut -d' ' -f1 > /caches/cache-id.txt
|
||||
cd ../../
|
||||
|
||||
- &restore-cache
|
||||
key: v1-{{ checksum "Dockerfile" }}-{{ checksum "/caches/cache-id.txt" }}
|
||||
paths:
|
||||
- /caches/container.tar.gz
|
||||
- /caches/image-id.txt
|
||||
|
||||
- ©-image
|
||||
name: Copy container image into package
|
||||
command: |
|
||||
cp /caches/container.tar.gz share/
|
||||
cp /caches/image-id.txt share/
|
||||
|
||||
jobs:
|
||||
run-lint:
|
||||
docker:
|
||||
- image: debian:bookworm
|
||||
resource_class: small
|
||||
steps:
|
||||
- checkout
|
||||
- run:
|
||||
name: Install dev. dependencies
|
||||
# Install only the necessary packages to run our linters.
|
||||
#
|
||||
# We run poetry with --no-ansi, to sidestep a Poetry bug that
|
||||
# currently exists in 1.3. See:
|
||||
# https://github.com/freedomofpress/dangerzone/issues/292#issuecomment-1351368122
|
||||
command: |
|
||||
apt-get update
|
||||
apt-get install -y git make python3 python3-poetry --no-install-recommends
|
||||
poetry install --no-ansi --only lint,test
|
||||
- run:
|
||||
name: Run linters to enforce code style
|
||||
command: poetry run make lint
|
||||
- run:
|
||||
name: Check that the QA script is up to date with the docs
|
||||
command: ./dev_scripts/qa.py --check-refs
|
||||
|
||||
build-container-image:
|
||||
machine:
|
||||
image: ubuntu-2004:202111-01
|
||||
steps:
|
||||
- checkout
|
||||
- run: *install-podman
|
||||
- run:
|
||||
name: Prepare cache directory
|
||||
command: |
|
||||
sudo mkdir -p /caches
|
||||
sudo chown -R $USER:$USER /caches
|
||||
- run: *calculate-cache-key
|
||||
- restore_cache: *restore-cache
|
||||
# setup_remote_docker
|
||||
- run:
|
||||
name: Build Dangerzone image
|
||||
command: |
|
||||
if [ -f "/caches/container.tar.gz" ]; then
|
||||
echo "Already cached, skipping"
|
||||
else
|
||||
sudo pip3 install poetry
|
||||
python3 ./install/common/build-image.py
|
||||
fi
|
||||
- run:
|
||||
name: Save Dangerzone image and image-id.txt to cache
|
||||
command: |
|
||||
if [ -f "/caches/container.tar.gz" ]; then
|
||||
echo "Already cached, skipping"
|
||||
else
|
||||
mkdir -p /caches
|
||||
podman save -o /caches/container.tar dangerzone.rocks/dangerzone
|
||||
gzip -f /caches/container.tar
|
||||
podman image ls dangerzone.rocks/dangerzone | grep "dangerzone.rocks/dangerzone" | tr -s ' ' | cut -d' ' -f3 > /caches/image-id.txt
|
||||
fi
|
||||
- run: *calculate-cache-key
|
||||
- save_cache:
|
||||
key: v1-{{ checksum "Dockerfile" }}-{{ checksum "/caches/cache-id.txt" }}
|
||||
paths:
|
||||
- /caches/container.tar.gz
|
||||
- /caches/image-id.txt
|
||||
|
||||
convert-test-docs:
|
||||
machine:
|
||||
image: ubuntu-2004:202111-01
|
||||
steps:
|
||||
- checkout
|
||||
- run: *install-podman
|
||||
- run:
|
||||
name: Install poetry dependencies
|
||||
command: |
|
||||
sudo pip3 install poetry
|
||||
# This flag is important, due to an open upstream Poetry issue:
|
||||
# https://github.com/python-poetry/poetry/issues/7184
|
||||
poetry install --no-ansi
|
||||
- run:
|
||||
name: Install test dependencies
|
||||
command: |
|
||||
sudo apt-get install -y libqt5gui5 libxcb-cursor0 --no-install-recommends
|
||||
- run:
|
||||
name: Prepare cache directory
|
||||
command: |
|
||||
sudo mkdir -p /caches
|
||||
sudo chown -R $USER:$USER /caches
|
||||
- run: *calculate-cache-key
|
||||
- restore_cache: *restore-cache
|
||||
- run: *copy-image
|
||||
- run:
|
||||
name: run automated tests
|
||||
command: |
|
||||
poetry run make test
|
||||
|
||||
ci-ubuntu-noble:
|
||||
machine:
|
||||
image: ubuntu-2004:202111-01
|
||||
steps:
|
||||
- checkout
|
||||
- run: *install-podman
|
||||
|
||||
- run:
|
||||
name: Prepare cache directory
|
||||
command: |
|
||||
sudo mkdir -p /caches
|
||||
sudo chown -R $USER:$USER /caches
|
||||
- run: *calculate-cache-key
|
||||
- restore_cache: *restore-cache
|
||||
- run: *copy-image
|
||||
|
||||
- run:
|
||||
name: Prepare Dangerzone environment
|
||||
command: |
|
||||
./dev_scripts/env.py --distro ubuntu --version 24.04 build-dev
|
||||
|
||||
- run:
|
||||
name: Run CI tests
|
||||
command: |
|
||||
./dev_scripts/env.py --distro ubuntu --version 24.04 run --dev \
|
||||
bash -c 'cd dangerzone; poetry run make test'
|
||||
|
||||
ci-ubuntu-mantic:
|
||||
machine:
|
||||
image: ubuntu-2004:202111-01
|
||||
steps:
|
||||
- checkout
|
||||
- run: *install-podman
|
||||
|
||||
- run:
|
||||
name: Prepare cache directory
|
||||
command: |
|
||||
sudo mkdir -p /caches
|
||||
sudo chown -R $USER:$USER /caches
|
||||
- run: *calculate-cache-key
|
||||
- restore_cache: *restore-cache
|
||||
- run: *copy-image
|
||||
|
||||
- run:
|
||||
name: Prepare Dangerzone environment
|
||||
command: |
|
||||
./dev_scripts/env.py --distro ubuntu --version 23.10 build-dev
|
||||
|
||||
- run:
|
||||
name: Run CI tests
|
||||
command: |
|
||||
./dev_scripts/env.py --distro ubuntu --version 23.10 run --dev \
|
||||
bash -c 'cd dangerzone; poetry run make test'
|
||||
|
||||
ci-ubuntu-jammy:
|
||||
machine:
|
||||
image: ubuntu-2004:202111-01
|
||||
steps:
|
||||
- checkout
|
||||
- run: *install-podman
|
||||
|
||||
- run:
|
||||
name: Prepare cache directory
|
||||
command: |
|
||||
sudo mkdir -p /caches
|
||||
sudo chown -R $USER:$USER /caches
|
||||
- run: *calculate-cache-key
|
||||
- restore_cache: *restore-cache
|
||||
- run: *copy-image
|
||||
|
||||
- run:
|
||||
name: Prepare Dangerzone environment
|
||||
command: |
|
||||
./dev_scripts/env.py --distro ubuntu --version 22.04 build-dev
|
||||
|
||||
- run:
|
||||
name: Run CI tests
|
||||
command: |
|
||||
./dev_scripts/env.py --distro ubuntu --version 22.04 run --dev \
|
||||
bash -c 'cd dangerzone; poetry run make test'
|
||||
|
||||
ci-ubuntu-focal:
|
||||
machine:
|
||||
image: ubuntu-2004:202111-01
|
||||
steps:
|
||||
- checkout
|
||||
- run: *install-podman
|
||||
|
||||
- run:
|
||||
name: Prepare cache directory
|
||||
command: |
|
||||
sudo mkdir -p /caches
|
||||
sudo chown -R $USER:$USER /caches
|
||||
- run: *calculate-cache-key
|
||||
- restore_cache: *restore-cache
|
||||
- run: *copy-image
|
||||
|
||||
- run:
|
||||
name: Prepare Dangerzone environment
|
||||
command: |
|
||||
./dev_scripts/env.py --distro ubuntu --version 20.04 build-dev
|
||||
|
||||
- run:
|
||||
name: Run CI tests
|
||||
command: |
|
||||
./dev_scripts/env.py --distro ubuntu --version 20.04 run --dev \
|
||||
bash -c 'cd dangerzone; poetry run make test'
|
||||
|
||||
ci-fedora-40:
|
||||
machine:
|
||||
image: ubuntu-2004:202111-01
|
||||
steps:
|
||||
- checkout
|
||||
- run: *install-podman
|
||||
|
||||
- run:
|
||||
name: Prepare cache directory
|
||||
command: |
|
||||
sudo mkdir -p /caches
|
||||
sudo chown -R $USER:$USER /caches
|
||||
- run: *calculate-cache-key
|
||||
- restore_cache: *restore-cache
|
||||
- run: *copy-image
|
||||
|
||||
- run:
|
||||
name: Prepare Dangerzone environment
|
||||
command: |
|
||||
./dev_scripts/env.py --distro fedora --version 40 build-dev
|
||||
|
||||
- run:
|
||||
name: Run CI tests
|
||||
command: |
|
||||
./dev_scripts/env.py --distro fedora --version 40 run --dev \
|
||||
bash -c 'cd dangerzone; poetry run make test'
|
||||
|
||||
ci-fedora-39:
|
||||
machine:
|
||||
image: ubuntu-2004:202111-01
|
||||
steps:
|
||||
- checkout
|
||||
- run: *install-podman
|
||||
|
||||
- run:
|
||||
name: Prepare cache directory
|
||||
command: |
|
||||
sudo mkdir -p /caches
|
||||
sudo chown -R $USER:$USER /caches
|
||||
- run: *calculate-cache-key
|
||||
- restore_cache: *restore-cache
|
||||
- run: *copy-image
|
||||
|
||||
- run:
|
||||
name: Prepare Dangerzone environment
|
||||
command: |
|
||||
./dev_scripts/env.py --distro fedora --version 39 build-dev
|
||||
|
||||
- run:
|
||||
name: Run CI tests
|
||||
command: |
|
||||
./dev_scripts/env.py --distro fedora --version 39 run --dev \
|
||||
bash -c 'cd dangerzone; poetry run make test'
|
||||
|
||||
ci-fedora-38:
|
||||
machine:
|
||||
image: ubuntu-2004:202111-01
|
||||
steps:
|
||||
- checkout
|
||||
- run: *install-podman
|
||||
|
||||
- run:
|
||||
name: Prepare cache directory
|
||||
command: |
|
||||
sudo mkdir -p /caches
|
||||
sudo chown -R $USER:$USER /caches
|
||||
- run: *calculate-cache-key
|
||||
- restore_cache: *restore-cache
|
||||
- run: *copy-image
|
||||
|
||||
- run:
|
||||
name: Prepare Dangerzone environment
|
||||
command: |
|
||||
./dev_scripts/env.py --distro fedora --version 38 build-dev
|
||||
|
||||
- run:
|
||||
name: Run CI tests
|
||||
command: |
|
||||
./dev_scripts/env.py --distro fedora --version 38 run --dev \
|
||||
bash -c 'cd dangerzone; poetry run make test'
|
||||
|
||||
ci-debian-trixie:
|
||||
machine:
|
||||
image: ubuntu-2004:202111-01
|
||||
steps:
|
||||
- checkout
|
||||
- run: *install-podman
|
||||
|
||||
- run:
|
||||
name: Prepare cache directory
|
||||
command: |
|
||||
sudo mkdir -p /caches
|
||||
sudo chown -R $USER:$USER /caches
|
||||
- run: *calculate-cache-key
|
||||
- restore_cache: *restore-cache
|
||||
- run: *copy-image
|
||||
|
||||
- run:
|
||||
name: Prepare Dangerzone environment
|
||||
command: |
|
||||
./dev_scripts/env.py --distro debian --version trixie build-dev
|
||||
|
||||
- run:
|
||||
name: Run CI tests
|
||||
command: |
|
||||
./dev_scripts/env.py --distro debian --version trixie run --dev \
|
||||
bash -c 'cd dangerzone; poetry run make test'
|
||||
|
||||
ci-debian-bookworm:
|
||||
machine:
|
||||
image: ubuntu-2004:202111-01
|
||||
steps:
|
||||
- checkout
|
||||
- run: *install-podman
|
||||
|
||||
- run:
|
||||
name: Prepare cache directory
|
||||
command: |
|
||||
sudo mkdir -p /caches
|
||||
sudo chown -R $USER:$USER /caches
|
||||
- run: *calculate-cache-key
|
||||
- restore_cache: *restore-cache
|
||||
- run: *copy-image
|
||||
|
||||
- run:
|
||||
name: Prepare Dangerzone environment
|
||||
command: |
|
||||
./dev_scripts/env.py --distro debian --version bookworm build-dev
|
||||
|
||||
- run:
|
||||
name: Run CI tests
|
||||
command: |
|
||||
./dev_scripts/env.py --distro debian --version bookworm run --dev \
|
||||
bash -c 'cd dangerzone; poetry run make test'
|
||||
|
||||
# NOTE: Making CI tests work in Debian Bullseye requires some tip-toeing
|
||||
# around certain Podman issues, as you'll see below. Read the following for
|
||||
# more details:
|
||||
#
|
||||
# https://github.com/freedomofpress/dangerzone/issues/388
|
||||
ci-debian-bullseye:
|
||||
machine:
|
||||
image: ubuntu-2204:2023.04.2
|
||||
steps:
|
||||
- checkout
|
||||
- run: *install-podman
|
||||
- run:
|
||||
name: Configure Podman for Ubuntu 22.04
|
||||
command: |
|
||||
# This config circumvents the following issues:
|
||||
# * https://github.com/containers/podman/issues/6368
|
||||
# * https://github.com/containers/podman/issues/10987
|
||||
mkdir -p ~/.config/containers
|
||||
cat > ~/.config/containers/containers.conf \<<EOF
|
||||
[engine]
|
||||
cgroup_manager="cgroupfs"
|
||||
events_logger="file"
|
||||
EOF
|
||||
|
||||
- run:
|
||||
name: Prepare cache directory
|
||||
command: |
|
||||
sudo mkdir -p /caches
|
||||
sudo chown -R $USER:$USER /caches
|
||||
- run: *calculate-cache-key
|
||||
- restore_cache: *restore-cache
|
||||
- run: *copy-image
|
||||
|
||||
- run:
|
||||
name: Prepare Dangerzone environment
|
||||
command: |
|
||||
./dev_scripts/env.py --distro debian --version bullseye build-dev
|
||||
|
||||
- run:
|
||||
name: Configure Podman for Debian Bullseye
|
||||
command: |
|
||||
# Copy the Podman config into the container image we created for the
|
||||
# Dangerzone environment.
|
||||
cp ~/.config/containers/containers.conf containers.conf
|
||||
cat > Dockerfile.bullseye \<<EOF
|
||||
FROM dangerzone.rocks/build/debian:bullseye-backports
|
||||
RUN mkdir -p /home/user/.config/containers
|
||||
COPY containers.conf /home/user/.config/containers/
|
||||
EOF
|
||||
|
||||
# Create a new image from the Dangerzone environment and re-tag it.
|
||||
podman build -t dangerzone.rocks/build/debian:bullseye-backports \
|
||||
-f Dockerfile.bullseye .
|
||||
|
||||
- run:
|
||||
name: Run CI tests
|
||||
command: |
|
||||
./dev_scripts/env.py --distro debian --version bullseye run --dev \
|
||||
bash -c 'cd dangerzone; poetry run make test'
|
||||
|
||||
build-ubuntu-noble:
|
||||
docker:
|
||||
- image: ubuntu:24.04
|
||||
resource_class: medium+
|
||||
steps:
|
||||
- run: *install-dependencies-deb
|
||||
- checkout
|
||||
- run: *calculate-cache-key
|
||||
- restore_cache: *restore-cache
|
||||
- run: *copy-image
|
||||
- run: *build-deb
|
||||
|
||||
build-ubuntu-mantic:
|
||||
docker:
|
||||
- image: ubuntu:23.10
|
||||
resource_class: medium+
|
||||
steps:
|
||||
- run: *install-dependencies-deb
|
||||
- checkout
|
||||
- run: *calculate-cache-key
|
||||
- restore_cache: *restore-cache
|
||||
- run: *copy-image
|
||||
- run: *build-deb
|
||||
|
||||
build-ubuntu-jammy:
|
||||
docker:
|
||||
- image: ubuntu:22.04
|
||||
resource_class: medium+
|
||||
steps:
|
||||
- run: *install-dependencies-deb
|
||||
- checkout
|
||||
- run: *calculate-cache-key
|
||||
- restore_cache: *restore-cache
|
||||
- run: *copy-image
|
||||
- run: *build-deb
|
||||
|
||||
build-ubuntu-focal:
|
||||
docker:
|
||||
- image: ubuntu:20.04
|
||||
resource_class: medium+
|
||||
steps:
|
||||
- run: *install-dependencies-deb
|
||||
- run: *install-python-all
|
||||
- checkout
|
||||
- run: *calculate-cache-key
|
||||
- restore_cache: *restore-cache
|
||||
- run: *copy-image
|
||||
- run: *build-deb
|
||||
|
||||
build-debian-trixie:
|
||||
docker:
|
||||
- image: debian:trixie
|
||||
resource_class: medium+
|
||||
steps:
|
||||
- run: *install-dependencies-deb
|
||||
- checkout
|
||||
- run: *calculate-cache-key
|
||||
- restore_cache: *restore-cache
|
||||
- run: *copy-image
|
||||
- run: *build-deb
|
||||
|
||||
build-debian-bookworm:
|
||||
docker:
|
||||
- image: debian:bookworm
|
||||
resource_class: medium+
|
||||
steps:
|
||||
- run: *install-dependencies-deb
|
||||
- checkout
|
||||
- run: *calculate-cache-key
|
||||
- restore_cache: *restore-cache
|
||||
- run: *copy-image
|
||||
- run: *build-deb
|
||||
|
||||
build-debian-bullseye:
|
||||
docker:
|
||||
- image: debian:bullseye
|
||||
resource_class: medium+
|
||||
steps:
|
||||
- run: *install-dependencies-deb
|
||||
- checkout
|
||||
- run: *calculate-cache-key
|
||||
- restore_cache: *restore-cache
|
||||
- run: *copy-image
|
||||
- run: *build-deb
|
||||
|
||||
build-fedora-40:
|
||||
docker:
|
||||
- image: fedora:40
|
||||
resource_class: medium+
|
||||
steps:
|
||||
- run: *install-dependencies-rpm
|
||||
- checkout
|
||||
- run: *calculate-cache-key
|
||||
- restore_cache: *restore-cache
|
||||
- run: *copy-image
|
||||
- run: *build-rpm
|
||||
- run: *build-rpm-qubes
|
||||
|
||||
build-fedora-39:
|
||||
docker:
|
||||
- image: fedora:39
|
||||
resource_class: medium+
|
||||
steps:
|
||||
- run: *install-dependencies-rpm
|
||||
- checkout
|
||||
- run: *calculate-cache-key
|
||||
- restore_cache: *restore-cache
|
||||
- run: *copy-image
|
||||
- run: *build-rpm
|
||||
- run: *build-rpm-qubes
|
||||
|
||||
build-fedora-38:
|
||||
docker:
|
||||
- image: fedora:38
|
||||
resource_class: medium+
|
||||
steps:
|
||||
- run: *install-dependencies-rpm
|
||||
- checkout
|
||||
- run: *calculate-cache-key
|
||||
- restore_cache: *restore-cache
|
||||
- run: *copy-image
|
||||
- run: *build-rpm
|
||||
- run: *build-rpm-qubes
|
||||
|
||||
|
||||
workflows:
|
||||
version: 2
|
||||
|
||||
build:
|
||||
jobs:
|
||||
- run-lint
|
||||
- build-container-image
|
||||
- convert-test-docs:
|
||||
requires:
|
||||
- build-container-image
|
||||
- ci-ubuntu-noble:
|
||||
requires:
|
||||
- build-container-image
|
||||
- ci-ubuntu-mantic:
|
||||
requires:
|
||||
- build-container-image
|
||||
- ci-ubuntu-jammy:
|
||||
requires:
|
||||
- build-container-image
|
||||
- ci-ubuntu-focal:
|
||||
requires:
|
||||
- build-container-image
|
||||
- ci-debian-trixie:
|
||||
requires:
|
||||
- build-container-image
|
||||
- ci-debian-bookworm:
|
||||
requires:
|
||||
- build-container-image
|
||||
- ci-debian-bullseye:
|
||||
requires:
|
||||
- build-container-image
|
||||
- ci-fedora-40:
|
||||
requires:
|
||||
- build-container-image
|
||||
- ci-fedora-39:
|
||||
requires:
|
||||
- build-container-image
|
||||
- ci-fedora-38:
|
||||
requires:
|
||||
- build-container-image
|
||||
# FIXME: Currently disabled because `stdeb` does not work with Python
|
||||
# 3.12, which is the default in Ubuntu Noble. See also:
|
||||
# https://github.com/freedomofpress/dangerzone/issues/773
|
||||
#
|
||||
#- build-ubuntu-noble:
|
||||
# requires:
|
||||
# - build-container-image
|
||||
- build-ubuntu-mantic:
|
||||
requires:
|
||||
- build-container-image
|
||||
- build-ubuntu-jammy:
|
||||
requires:
|
||||
- build-container-image
|
||||
- build-ubuntu-focal:
|
||||
requires:
|
||||
- build-container-image
|
||||
- build-debian-bullseye:
|
||||
requires:
|
||||
- build-container-image
|
||||
- build-debian-trixie:
|
||||
requires:
|
||||
- build-container-image
|
||||
- build-debian-bookworm:
|
||||
requires:
|
||||
- build-container-image
|
||||
- build-fedora-40:
|
||||
requires:
|
||||
- build-container-image
|
||||
- build-fedora-39:
|
||||
requires:
|
||||
- build-container-image
|
||||
- build-fedora-38:
|
||||
requires:
|
||||
- build-container-image
|
||||
5
.gitattributes
vendored
5
.gitattributes
vendored
|
|
@ -1,5 +0,0 @@
|
|||
* text=auto
|
||||
*.py text eol=lf
|
||||
*.jpg -text
|
||||
*.gif -text
|
||||
*.png -text
|
||||
67
.github/ISSUE_TEMPLATE/bug_report_linux.yml
vendored
67
.github/ISSUE_TEMPLATE/bug_report_linux.yml
vendored
|
|
@ -1,67 +0,0 @@
|
|||
name: Bug Report (Linux)
|
||||
description: File a bug report for Linux.
|
||||
labels: ["bug", "triage"]
|
||||
projects: ["freedomofpress/dangerzone"]
|
||||
body:
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: |
|
||||
Hi, and thanks for taking the time to open this bug report.
|
||||
- type: textarea
|
||||
id: what-happened
|
||||
attributes:
|
||||
label: What happened?
|
||||
description: What was the expected behaviour, and what was the actual behaviour? Can you specify the steps you followed, so that we can reproduce?
|
||||
placeholder: "A bug happened!"
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: os-version
|
||||
attributes:
|
||||
label: Linux distribution
|
||||
description: |
|
||||
What is the name and version of your Linux distribution? You can find it out with `cat /etc/os-release`
|
||||
placeholder: Ubuntu 22.04.5 LTS
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: dangerzone-version
|
||||
attributes:
|
||||
label: Dangerzone version
|
||||
description: Which version of Dangerzone are you using?
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: podman-info
|
||||
attributes:
|
||||
label: Podman info
|
||||
description: |
|
||||
Please copy and paste the following commands in your terminal, and provide us with the output:
|
||||
|
||||
```shell
|
||||
podman version
|
||||
podman info -f 'json'
|
||||
podman images
|
||||
podman run hello-world
|
||||
```
|
||||
|
||||
This will be automatically formatted into code, so no need for backticks.
|
||||
render: shell
|
||||
- type: textarea
|
||||
id: logs
|
||||
attributes:
|
||||
label: Document conversion logs
|
||||
description: |
|
||||
If the bug occurs during document conversion, we'd like some logs from this process. Please copy and paste the following commands in your terminal, and provide us with the output (replace `/path/to/file` with the path to your document):
|
||||
|
||||
```bash
|
||||
dangerzone-cli /path/to/file
|
||||
```
|
||||
|
||||
render: shell
|
||||
- type: textarea
|
||||
id: additional-info
|
||||
attributes:
|
||||
label: Additional info
|
||||
description: |
|
||||
Please provide us with any additional info, such as logs, extra content, that may help us debug this issue.
|
||||
82
.github/ISSUE_TEMPLATE/bug_report_macos.yml
vendored
82
.github/ISSUE_TEMPLATE/bug_report_macos.yml
vendored
|
|
@ -1,82 +0,0 @@
|
|||
name: Bug Report (MacOS)
|
||||
description: File a bug report for MacOS.
|
||||
labels: ["bug", "triage"]
|
||||
projects: ["freedomofpress/dangerzone"]
|
||||
body:
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: |
|
||||
Hi, and thanks for taking the time to open this bug report.
|
||||
- type: textarea
|
||||
id: what-happened
|
||||
attributes:
|
||||
label: What happened?
|
||||
description: What was the expected behaviour, and what was the actual behaviour? Can you specify the steps you followed, so that we can reproduce?
|
||||
placeholder: "A bug happened!"
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: os-version
|
||||
attributes:
|
||||
label: operating system version
|
||||
description: Which version of MacOS do you use? You can follow [this link](https://support.apple.com/en-us/109033) to find out more.
|
||||
placeholder: macOS Sequoia 15
|
||||
validations:
|
||||
required: true
|
||||
- type: dropdown
|
||||
id: proc-architecture
|
||||
attributes:
|
||||
label: Processor type
|
||||
description: |
|
||||
Which kind of processor do you use?
|
||||
|
||||
You can follow [this link](https://support.apple.com/en-us/109033) to find out more.
|
||||
options:
|
||||
- Intel
|
||||
- Apple Silicon
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: dangerzone-version
|
||||
attributes:
|
||||
label: Dangerzone version
|
||||
description: Which version of Dangerzone are you using?
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: docker-info
|
||||
attributes:
|
||||
label: Docker info
|
||||
description: |
|
||||
Please copy and paste the following commands in your
|
||||
terminal, and provide us with the output:
|
||||
|
||||
```shell
|
||||
docker version
|
||||
docker info -f 'json'
|
||||
docker images
|
||||
docker run hello-world
|
||||
```
|
||||
|
||||
This will be automatically formatted into code, so no need for backticks.
|
||||
render: shell
|
||||
- type: textarea
|
||||
id: logs
|
||||
attributes:
|
||||
label: Document conversion logs
|
||||
description: |
|
||||
|
||||
If the bug occurs during document conversion, we'd like some logs from this process. Please copy and paste the following commands in your terminal, and provide us with the output (replace `/path/to/file` with the path to your document):
|
||||
|
||||
```bash
|
||||
|
||||
/Applications/Dangerzone.app/Contents/MacOS/dangerzone-cli /path/to/file
|
||||
```
|
||||
|
||||
render: shell
|
||||
- type: textarea
|
||||
id: additional-info
|
||||
attributes:
|
||||
label: Additional info
|
||||
description: |
|
||||
Please provide us with any additional info, such as logs, extra content, that may help us debug this issue.
|
||||
67
.github/ISSUE_TEMPLATE/bug_report_windows.yml
vendored
67
.github/ISSUE_TEMPLATE/bug_report_windows.yml
vendored
|
|
@ -1,67 +0,0 @@
|
|||
name: Bug Report (Windows)
|
||||
description: File a bug report for Windows.
|
||||
labels: ["bug", "triage"]
|
||||
projects: ["freedomofpress/dangerzone"]
|
||||
body:
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: |
|
||||
Hi, and thanks for taking the time to open this bug report.
|
||||
- type: textarea
|
||||
id: what-happened
|
||||
attributes:
|
||||
label: What happened?
|
||||
description: What was the expected behaviour, and what was the actual behaviour? Can you specify the steps you followed, so that we can reproduce?
|
||||
placeholder: "A bug happened!"
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: os-version
|
||||
attributes:
|
||||
label: operating system version
|
||||
description: |
|
||||
Which version of Windows do you use? Follow [this link](https://learn.microsoft.com/en-us/windows/client-management/client-tools/windows-version-search) to find out.
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: dangerzone-version
|
||||
attributes:
|
||||
label: Dangerzone version
|
||||
description: Which version of Dangerzone are you using?
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: docker-info
|
||||
attributes:
|
||||
label: Docker info
|
||||
description: |
|
||||
Please copy and paste the following commands in your
|
||||
terminal, and provide us with the output:
|
||||
|
||||
```shell
|
||||
docker version
|
||||
docker info -f 'json'
|
||||
docker images
|
||||
docker run hello-world
|
||||
```
|
||||
|
||||
This will be automatically formatted into code, so no need for backticks.
|
||||
render: shell
|
||||
- type: textarea
|
||||
id: logs
|
||||
attributes:
|
||||
label: Document conversion logs
|
||||
description: |
|
||||
If the bug occurs during document conversion, we'd like some logs from this process. Please copy and paste the following commands in your terminal, and provide us with the output (replace `\path\to\file` with the path to your document):
|
||||
|
||||
```bash
|
||||
'C:\Program Files (x86)\Dangerzone\dangerzone-cli.exe' \path\to\file
|
||||
```
|
||||
|
||||
render: shell
|
||||
- type: textarea
|
||||
id: additional-info
|
||||
attributes:
|
||||
label: Additional info
|
||||
description: |
|
||||
Please provide us with any additional info, such as logs, extra content, that may help us debug this issue.
|
||||
1
.github/ISSUE_TEMPLATE/config.yml
vendored
1
.github/ISSUE_TEMPLATE/config.yml
vendored
|
|
@ -1 +0,0 @@
|
|||
blank_issues_enabled: true
|
||||
21
.github/ISSUE_TEMPLATE/feature_request.md
vendored
21
.github/ISSUE_TEMPLATE/feature_request.md
vendored
|
|
@ -1,21 +0,0 @@
|
|||
---
|
||||
name: Feature request
|
||||
about: Suggest an idea for this project
|
||||
title: ''
|
||||
labels: enhancement
|
||||
assignees: ''
|
||||
---
|
||||
|
||||
**What is the feature you think should be a good addition to Dangerzone?**
|
||||
|
||||
?
|
||||
|
||||
**Is your feature request related to a problem? Please describe.**
|
||||
|
||||
It's always useful for us to know more about your context, and why you think
|
||||
this would be a great addition. Don't hesitate to put some details about your
|
||||
current workflow and how this could be useful to you.
|
||||
|
||||
**Additional context**
|
||||
|
||||
Add any other context or screenshots about the feature request here.
|
||||
248
.github/workflows/build-push-image.yml
vendored
248
.github/workflows/build-push-image.yml
vendored
|
|
@ -1,248 +0,0 @@
|
|||
name: Build and push multi-arch container image
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
registry:
|
||||
required: true
|
||||
type: string
|
||||
registry_user:
|
||||
required: true
|
||||
type: string
|
||||
image_name:
|
||||
required: true
|
||||
type: string
|
||||
reproduce:
|
||||
required: true
|
||||
type: boolean
|
||||
secrets:
|
||||
registry_token:
|
||||
required: true
|
||||
|
||||
|
||||
jobs:
|
||||
lint:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Install dev. dependencies
|
||||
run: |-
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y git python3-poetry --no-install-recommends
|
||||
poetry install --only package
|
||||
|
||||
- name: Verify that the Dockerfile matches the commited template and params
|
||||
run: |-
|
||||
cp Dockerfile Dockerfile.orig
|
||||
make Dockerfile
|
||||
diff Dockerfile.orig Dockerfile
|
||||
|
||||
prepare:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
debian_archive_date: ${{ steps.params.outputs.debian_archive_date }}
|
||||
source_date_epoch: ${{ steps.params.outputs.source_date_epoch }}
|
||||
image: ${{ steps.params.outputs.full_image_name }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Compute image parameters
|
||||
id: params
|
||||
run: |
|
||||
source Dockerfile.env
|
||||
DEBIAN_ARCHIVE_DATE=$(date -u +'%Y%m%d')
|
||||
SOURCE_DATE_EPOCH=$(date -u -d ${DEBIAN_ARCHIVE_DATE} +"%s")
|
||||
TAG=${DEBIAN_ARCHIVE_DATE}-$(git describe --long --first-parent | tail -c +2)
|
||||
FULL_IMAGE_NAME=${{ inputs.registry }}/${{ inputs.image_name }}:${TAG}
|
||||
|
||||
echo "debian_archive_date=${DEBIAN_ARCHIVE_DATE}" >> $GITHUB_OUTPUT
|
||||
echo "source_date_epoch=${SOURCE_DATE_EPOCH}" >> $GITHUB_OUTPUT
|
||||
echo "tag=${DEBIAN_ARCHIVE_DATE}-${TAG}" >> $GITHUB_OUTPUT
|
||||
echo "full_image_name=${FULL_IMAGE_NAME}" >> $GITHUB_OUTPUT
|
||||
echo "buildkit_image=${BUILDKIT_IMAGE}" >> $GITHUB_OUTPUT
|
||||
|
||||
build:
|
||||
name: Build ${{ matrix.platform.name }} image
|
||||
runs-on: ${{ matrix.platform.runs-on }}
|
||||
needs:
|
||||
- prepare
|
||||
outputs:
|
||||
debian_archive_date: ${{ needs.prepare.outputs.debian_archive_date }}
|
||||
source_date_epoch: ${{ needs.prepare.outputs.source_date_epoch }}
|
||||
image: ${{ needs.prepare.outputs.image }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
platform:
|
||||
- runs-on: "ubuntu-24.04"
|
||||
name: "linux/amd64"
|
||||
- runs-on: "ubuntu-24.04-arm"
|
||||
name: "linux/arm64"
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Prepare
|
||||
run: |
|
||||
platform=${{ matrix.platform.name }}
|
||||
echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV
|
||||
|
||||
- name: Login to GHCR
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ inputs.registry_user }}
|
||||
password: ${{ secrets.registry_token }}
|
||||
|
||||
# Instructions for reproducibly building a container image are taken from:
|
||||
# https://github.com/freedomofpress/repro-build?tab=readme-ov-file#build-and-push-a-container-image-on-github-actions
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
with:
|
||||
driver-opts: image=${{ needs.prepare.outputs.buildkit_image }}
|
||||
|
||||
- name: Build and push by digest
|
||||
id: build
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: ./dangerzone/
|
||||
file: Dockerfile
|
||||
build-args: |
|
||||
DEBIAN_ARCHIVE_DATE=${{ needs.prepare.outputs.debian_archive_date }}
|
||||
SOURCE_DATE_EPOCH=${{ needs.prepare.outputs.source_date_epoch }}
|
||||
provenance: false
|
||||
outputs: type=image,"name=${{ inputs.registry }}/${{ inputs.image_name }}",push-by-digest=true,push=true,rewrite-timestamp=true,name-canonical=true
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
|
||||
- name: Export digest
|
||||
run: |
|
||||
mkdir -p ${{ runner.temp }}/digests
|
||||
digest="${{ steps.build.outputs.digest }}"
|
||||
touch "${{ runner.temp }}/digests/${digest#sha256:}"
|
||||
echo "Image digest is: ${digest}"
|
||||
|
||||
- name: Upload digest
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: digests-${{ env.PLATFORM_PAIR }}
|
||||
path: ${{ runner.temp }}/digests/*
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
|
||||
merge:
|
||||
runs-on: ubuntu-latest
|
||||
needs:
|
||||
- build
|
||||
outputs:
|
||||
debian_archive_date: ${{ needs.build.outputs.debian_archive_date }}
|
||||
source_date_epoch: ${{ needs.build.outputs.source_date_epoch }}
|
||||
image: ${{ needs.build.outputs.image }}
|
||||
digest_root: ${{ steps.image.outputs.digest_root }}
|
||||
digest_amd64: ${{ steps.image.outputs.digest_amd64 }}
|
||||
digest_arm64: ${{ steps.image.outputs.digest_arm64 }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Download digests
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
path: ${{ runner.temp }}/digests
|
||||
pattern: digests-*
|
||||
merge-multiple: true
|
||||
|
||||
- name: Login to GHCR
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ inputs.registry_user }}
|
||||
password: ${{ secrets.registry_token }}
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
with:
|
||||
driver-opts: image=${{ env.BUILDKIT_IMAGE }}
|
||||
|
||||
- name: Create manifest list and push
|
||||
working-directory: ${{ runner.temp }}/digests
|
||||
run: |
|
||||
DIGESTS=$(printf '${{ needs.build.outputs.image }}@sha256:%s ' *)
|
||||
docker buildx imagetools create -t ${{ needs.build.outputs.image }} ${DIGESTS}
|
||||
|
||||
- name: Inspect image
|
||||
id: image
|
||||
run: |
|
||||
# Inspect the image
|
||||
docker buildx imagetools inspect ${{ needs.build.outputs.image }}
|
||||
docker buildx imagetools inspect ${{ needs.build.outputs.image }} --format "{{json .Manifest}}" > manifest
|
||||
|
||||
# Calculate and print the digests
|
||||
digest_root=$(jq -r .digest manifest)
|
||||
digest_amd64=$(jq -r '.manifests[] | select(.platform.architecture=="amd64") | .digest' manifest)
|
||||
digest_arm64=$(jq -r '.manifests[] | select(.platform.architecture=="arm64") | .digest' manifest)
|
||||
|
||||
echo "The image digests are:"
|
||||
echo " Root: $digest_root"
|
||||
echo " linux/amd64: $digest_amd64"
|
||||
echo " linux/arm64: $digest_arm64"
|
||||
|
||||
# NOTE: Set the digests as an output because the `env` context is not
|
||||
# available to the inputs of a reusable workflow call.
|
||||
echo "digest_root=$digest_root" >> "$GITHUB_OUTPUT"
|
||||
echo "digest_amd64=$digest_amd64" >> "$GITHUB_OUTPUT"
|
||||
echo "digest_arm64=$digest_arm64" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# This step calls the container workflow to generate provenance and push it to
|
||||
# the container registry.
|
||||
provenance:
|
||||
needs:
|
||||
- merge
|
||||
strategy:
|
||||
matrix:
|
||||
manifest_type:
|
||||
- root
|
||||
- amd64
|
||||
- arm64
|
||||
permissions:
|
||||
actions: read # for detecting the Github Actions environment.
|
||||
id-token: write # for creating OIDC tokens for signing.
|
||||
packages: write # for uploading attestations.
|
||||
uses: slsa-framework/slsa-github-generator/.github/workflows/generator_container_slsa3.yml@v2.1.0
|
||||
with:
|
||||
digest: ${{ needs.merge.outputs[format('digest_{0}', matrix.manifest_type)] }}
|
||||
image: ${{ needs.merge.outputs.image }}
|
||||
registry-username: ${{ inputs.registry_user }}
|
||||
secrets:
|
||||
registry-password: ${{ secrets.registry_token }}
|
||||
|
||||
# This step ensures that the image is reproducible
|
||||
check-reproducibility:
|
||||
if: ${{ inputs.reproduce }}
|
||||
needs:
|
||||
- merge
|
||||
runs-on: ${{ matrix.platform.runs-on }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
platform:
|
||||
- runs-on: "ubuntu-24.04"
|
||||
name: "amd64"
|
||||
- runs-on: "ubuntu-24.04-arm"
|
||||
name: "arm64"
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Reproduce the same container image
|
||||
run: |
|
||||
./dev_scripts/reproduce-image.py \
|
||||
--runtime \
|
||||
docker \
|
||||
--debian-archive-date \
|
||||
${{ needs.merge.outputs.debian_archive_date }} \
|
||||
--platform \
|
||||
linux/${{ matrix.platform.name }} \
|
||||
${{ needs.merge.outputs[format('digest_{0}', matrix.platform.name)] }}
|
||||
98
.github/workflows/build.yml
vendored
98
.github/workflows/build.yml
vendored
|
|
@ -1,98 +0,0 @@
|
|||
name: Build dev environments
|
||||
on:
|
||||
pull_request:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
- "test/**"
|
||||
schedule:
|
||||
- cron: "0 0 * * *" # Run every day at 00:00 UTC.
|
||||
|
||||
permissions:
|
||||
packages: write
|
||||
|
||||
env:
|
||||
IMAGE_REGISTRY: ghcr.io/${{ github.repository_owner }}
|
||||
REGISTRY_USER: ${{ github.actor }}
|
||||
REGISTRY_PASSWORD: ${{ github.token }}
|
||||
|
||||
# Each day, build and publish to ghcr.io:
|
||||
#
|
||||
# - the dangerzone/dangerzone container image
|
||||
# - the dangerzone/build/{debian,ubuntu,fedora}:version
|
||||
# dev environments used to run the tests
|
||||
#
|
||||
# End-user environments are not published to the GHCR because
|
||||
# they need .rpm or .deb files to be built, which is what we
|
||||
# want to test.
|
||||
|
||||
jobs:
|
||||
build-dev-environment:
|
||||
name: "Build dev-env (${{ matrix.distro }}-${{ matrix.version }})"
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- distro: ubuntu
|
||||
version: "22.04"
|
||||
- distro: ubuntu
|
||||
version: "24.04"
|
||||
- distro: ubuntu
|
||||
version: "24.10"
|
||||
- distro: ubuntu
|
||||
version: "25.04"
|
||||
- distro: debian
|
||||
version: bullseye
|
||||
- distro: debian
|
||||
version: bookworm
|
||||
- distro: debian
|
||||
version: trixie
|
||||
- distro: fedora
|
||||
version: "40"
|
||||
- distro: fedora
|
||||
version: "41"
|
||||
- distro: fedora
|
||||
version: "42"
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.10"
|
||||
|
||||
- name: Login to GHCR
|
||||
run: |
|
||||
echo ${{ github.token }} | podman login ghcr.io -u USERNAME --password-stdin
|
||||
|
||||
- name: Build dev environment
|
||||
run: |
|
||||
./dev_scripts/env.py --distro ${{ matrix.distro }} \
|
||||
--version ${{ matrix.version }} \
|
||||
build-dev --sync
|
||||
|
||||
build-container-image:
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Get current date
|
||||
id: date
|
||||
run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Cache container image
|
||||
id: cache-container-image
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
key: v5-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/*.py', 'dangerzone/container_helpers/*', 'install/common/build-image.py') }}
|
||||
path: |
|
||||
share/container.tar
|
||||
share/image-id.txt
|
||||
|
||||
- name: Build Dangerzone image
|
||||
if: ${{ steps.cache-container-image.outputs.cache-hit != 'true' }}
|
||||
run: |
|
||||
python3 ./install/common/build-image.py
|
||||
|
|
@ -1,7 +1,6 @@
|
|||
name: Check branch conformity
|
||||
on:
|
||||
pull_request:
|
||||
types: ["opened", "labeled", "unlabeled", "reopened", "synchronize"]
|
||||
push:
|
||||
|
||||
jobs:
|
||||
prevent-fixup-commits:
|
||||
|
|
@ -18,13 +17,3 @@ jobs:
|
|||
git fetch origin
|
||||
git status
|
||||
git log --pretty=format:%s origin/main..HEAD | grep -ie '^fixup\|^wip' && exit 1 || true
|
||||
|
||||
check-changelog:
|
||||
runs-on: ubuntu-latest
|
||||
name: Ensure CHANGELOG.md is populated for user-visible changes
|
||||
steps:
|
||||
# Pin the GitHub action to a specific commit that we have audited and know
|
||||
# how it works.
|
||||
- uses: tarides/changelog-check-action@509965da3b8ac786a5e2da30c2ccf9661189121f
|
||||
with:
|
||||
changelog: CHANGELOG.md
|
||||
65
.github/workflows/check_repos.yml
vendored
65
.github/workflows/check_repos.yml
vendored
|
|
@ -9,7 +9,6 @@ name: Test official instructions for installing Dangerzone
|
|||
on:
|
||||
schedule:
|
||||
- cron: '0 0 * * *' # Run every day at 00:00 UTC.
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
install-from-apt-repo:
|
||||
|
|
@ -20,13 +19,11 @@ jobs:
|
|||
matrix:
|
||||
include:
|
||||
- distro: ubuntu
|
||||
version: "25.04" # plucky
|
||||
- distro: ubuntu
|
||||
version: "24.10" # oracular
|
||||
- distro: ubuntu
|
||||
version: "24.04" # noble
|
||||
version: "23.10" # mantic
|
||||
- distro: ubuntu
|
||||
version: "22.04" # jammy
|
||||
- distro: ubuntu
|
||||
version: "20.04" # focal
|
||||
- distro: debian
|
||||
version: "trixie" # 13
|
||||
- distro: debian
|
||||
|
|
@ -34,38 +31,33 @@ jobs:
|
|||
- distro: debian
|
||||
version: "11" # bullseye
|
||||
steps:
|
||||
- name: Add packages.freedom.press PGP key (gpg --keyring)
|
||||
if: matrix.version != 'trixie' && matrix.version != "25.04"
|
||||
- name: Add Podman repo for Ubuntu Focal
|
||||
if: matrix.distro == 'ubuntu' && matrix.version == 20.04
|
||||
run: |
|
||||
apt-get update && apt-get -y install curl wget gnupg2
|
||||
. /etc/os-release
|
||||
sh -c "echo 'deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_${VERSION_ID}/ /' \
|
||||
> /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list"
|
||||
wget -nv https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable/xUbuntu_${VERSION_ID}/Release.key -O- \
|
||||
| apt-key add -
|
||||
apt update
|
||||
apt-get install python-all -y
|
||||
|
||||
- name: Add GPG key for the packages.freedom.press
|
||||
run: |
|
||||
apt-get update && apt-get install -y gnupg2 ca-certificates
|
||||
dirmngr # NOTE: This is a command that's necessary only in containers
|
||||
# The key needs to be in the GPG keybox database format so the
|
||||
# signing subkey is detected by apt-secure.
|
||||
gpg --keyserver hkps://keys.openpgp.org \
|
||||
--no-default-keyring --keyring ./fpf-apt-tools-archive-keyring.gpg \
|
||||
--recv-keys "DE28 AB24 1FA4 8260 FAC9 B8BA A7C9 B385 2260 4281"
|
||||
mkdir -p /etc/apt/keyrings/
|
||||
mv ./fpf-apt-tools-archive-keyring.gpg /etc/apt/keyrings/.
|
||||
|
||||
- name: Add packages.freedom.press PGP key (sq)
|
||||
if: matrix.version == 'trixie' || matrix.version == '25.04'
|
||||
run: |
|
||||
apt-get update && apt-get install -y ca-certificates sq
|
||||
mkdir -p /etc/apt/keyrings/
|
||||
# On debian trixie, apt-secure uses `sqv` to verify the signatures
|
||||
# so we need to retrieve PGP keys and store them using the base64 format.
|
||||
sq network keyserver \
|
||||
--server hkps://keys.openpgp.org \
|
||||
search "DE28 AB24 1FA4 8260 FAC9 B8BA A7C9 B385 2260 4281" \
|
||||
--output - \
|
||||
| sq packet dearmor \
|
||||
> /etc/apt/keyrings/fpf-apt-tools-archive-keyring.gpg
|
||||
mv fpf-apt-tools-archive-keyring.gpg /etc/apt/keyrings
|
||||
|
||||
- name: Add packages.freedom.press to our APT sources
|
||||
run: |
|
||||
. /etc/os-release
|
||||
echo "deb [signed-by=/etc/apt/keyrings/fpf-apt-tools-archive-keyring.gpg] \
|
||||
https://packages.freedom.press/apt-tools-prod ${VERSION_CODENAME?} main" \
|
||||
echo deb [signed-by=/etc/apt/keyrings/fpf-apt-tools-archive-keyring.gpg] \
|
||||
https://packages.freedom.press/apt-tools-prod ${VERSION_CODENAME?} main \
|
||||
| tee /etc/apt/sources.list.d/fpf-apt-tools.list
|
||||
|
||||
- name: Install Dangerzone
|
||||
|
|
@ -81,27 +73,14 @@ jobs:
|
|||
matrix:
|
||||
include:
|
||||
- distro: fedora
|
||||
version: 40
|
||||
version: 38
|
||||
- distro: fedora
|
||||
version: 41
|
||||
- distro: fedora
|
||||
version: 42
|
||||
version: 39
|
||||
steps:
|
||||
- name: Add packages.freedom.press to our YUM sources
|
||||
run: |
|
||||
dnf install -y 'dnf-command(config-manager)'
|
||||
dnf-3 config-manager --add-repo=https://packages.freedom.press/yum-tools-prod/dangerzone/dangerzone.repo
|
||||
|
||||
- name: Replace 'rawhide' string with Fedora version
|
||||
# The previous command has created a `dangerzone.repo` file. The
|
||||
# config-manager plugin should have substituted the $releasever variable
|
||||
# with the Fedora version number. However, for unreleased Fedora
|
||||
# versions, this gets translated to "rawhide", even though they do have
|
||||
# a number. To fix this, we need to substitute the "rawhide" string
|
||||
# witht the proper Fedora version.
|
||||
run: |
|
||||
source /etc/os-release
|
||||
sed -i "s/rawhide/${VERSION_ID}/g" /etc/yum.repos.d/dangerzone.repo
|
||||
dnf config-manager --add-repo=https://packages.freedom.press/yum-tools-prod/dangerzone/dangerzone.repo
|
||||
|
||||
- name: Install Dangerzone
|
||||
# FIXME: We add the `-y` flag here, in lieu of a better way to check the
|
||||
|
|
|
|||
446
.github/workflows/ci.yml
vendored
446
.github/workflows/ci.yml
vendored
|
|
@ -1,273 +1,111 @@
|
|||
name: Tests
|
||||
on:
|
||||
pull_request:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
- "test/**"
|
||||
pull_request:
|
||||
branches: [ main ]
|
||||
schedule:
|
||||
- cron: "2 0 * * *" # Run every day at 02:00 UTC.
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
packages: write
|
||||
|
||||
env:
|
||||
REGISTRY_USER: ${{ github.actor }}
|
||||
REGISTRY_PASSWORD: ${{ github.token }}
|
||||
IMAGE_REGISTRY: ghcr.io/${{ github.repository_owner }}
|
||||
QT_SELECT: "qt6"
|
||||
|
||||
# Disable multiple concurrent runs on the same branch
|
||||
# When a new CI build is triggered, it will cancel the
|
||||
# other in-progress ones (for the same branch)
|
||||
concurrency:
|
||||
group: ${{ github.head_ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
- cron: '0 0 * * *' # Run every day at 00:00 UTC.
|
||||
|
||||
jobs:
|
||||
run-lint:
|
||||
runs-on: ubuntu-latest
|
||||
container:
|
||||
image: debian:bookworm
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Install dev. dependencies
|
||||
run: |-
|
||||
apt-get update
|
||||
apt-get install -y git make python3 python3-poetry --no-install-recommends
|
||||
poetry install --only lint,test
|
||||
- name: Run linters to enforce code style
|
||||
run: poetry run make lint
|
||||
- name: Check that the QA script is up to date with the docs
|
||||
run: "./dev_scripts/qa.py --check-refs"
|
||||
|
||||
# This is already built daily by the "build.yml" file
|
||||
# But we also want to include this in the checks that run on each push.
|
||||
build-container-image:
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Get current date
|
||||
id: date
|
||||
run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Cache container image
|
||||
id: cache-container-image
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
key: v5-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/*.py', 'dangerzone/container_helpers/*', 'install/common/build-image.py') }}
|
||||
path: |-
|
||||
share/container.tar
|
||||
share/image-id.txt
|
||||
|
||||
- name: Build Dangerzone container image
|
||||
if: ${{ steps.cache-container-image.outputs.cache-hit != 'true' }}
|
||||
run: |
|
||||
python3 ./install/common/build-image.py
|
||||
|
||||
- name: Upload container image
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: container.tar
|
||||
path: share/container.tar
|
||||
|
||||
download-tessdata:
|
||||
name: Download and cache Tesseract data
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Cache Tessdata
|
||||
id: cache-tessdata
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: share/tessdata/
|
||||
key: v1-tessdata-${{ hashFiles('./install/common/download-tessdata.py') }}
|
||||
enableCrossOsArchive: true
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.11'
|
||||
- name: Download Tessdata
|
||||
run: |-
|
||||
if [ -f "share/tessdata" ]; then
|
||||
echo "Already cached, skipping"
|
||||
else
|
||||
python3 ./install/common/download-tessdata.py
|
||||
fi
|
||||
|
||||
windows:
|
||||
runs-on: windows-latest
|
||||
needs:
|
||||
- download-tessdata
|
||||
env:
|
||||
DUMMY_CONVERSION: 1
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.12"
|
||||
python-version: '3.11'
|
||||
- run: pip install poetry
|
||||
- run: poetry install
|
||||
- name: Restore cached tessdata
|
||||
uses: actions/cache/restore@v4
|
||||
with:
|
||||
path: share/tessdata/
|
||||
enableCrossOsArchive: true
|
||||
fail-on-cache-miss: true
|
||||
key: v1-tessdata-${{ hashFiles('./install/common/download-tessdata.py') }}
|
||||
- name: Run CLI tests
|
||||
run: poetry run make test
|
||||
- name: Set up .NET CLI environment
|
||||
uses: actions/setup-dotnet@v4
|
||||
with:
|
||||
dotnet-version: "8.x"
|
||||
- name: Install WiX Toolset
|
||||
run: dotnet tool install --global wix --version 5.0.2
|
||||
- name: Add WiX UI extension
|
||||
run: wix extension add --global WixToolset.UI.wixext/5.0.2
|
||||
# Taken from: https://github.com/orgs/community/discussions/27149#discussioncomment-3254829
|
||||
- name: Set path for candle and light
|
||||
run: echo "C:\Program Files (x86)\WiX Toolset v3.14\bin" >> $GITHUB_PATH
|
||||
shell: bash
|
||||
- name: Build the MSI installer
|
||||
# NOTE: This also builds the .exe internally.
|
||||
run: poetry run .\install\windows\build-app.bat
|
||||
- name: Upload MSI installer
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: Dangerzone.msi
|
||||
path: "dist/Dangerzone.msi"
|
||||
if-no-files-found: error
|
||||
compression-level: 0
|
||||
|
||||
macOS:
|
||||
name: "macOS (${{ matrix.arch }})"
|
||||
runs-on: ${{ matrix.runner }}
|
||||
needs:
|
||||
- download-tessdata
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- runner: macos-latest # CPU type: Apple Silicon (M1)
|
||||
arch: arch64
|
||||
- runner: macos-13 # CPU type: Intel x86_64
|
||||
arch: x86_64
|
||||
runs-on: macos-latest
|
||||
env:
|
||||
DUMMY_CONVERSION: 1
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.12"
|
||||
- name: Restore cached tessdata
|
||||
uses: actions/cache/restore@v4
|
||||
with:
|
||||
path: share/tessdata/
|
||||
enableCrossOsArchive: true
|
||||
fail-on-cache-miss: true
|
||||
key: v1-tessdata-${{ hashFiles('./install/common/download-tessdata.py') }}
|
||||
python-version: '3.11'
|
||||
- run: pip install poetry
|
||||
- run: poetry install
|
||||
- name: Run CLI tests
|
||||
run: poetry run make test
|
||||
- name: Build macOS app
|
||||
run: poetry run python ./install/macos/build-app.py
|
||||
- name: Upload macOS app
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: Dangerzone-${{ matrix.arch }}.app
|
||||
path: "dist/Dangerzone.app"
|
||||
if-no-files-found: error
|
||||
compression-level: 0
|
||||
|
||||
build-deb:
|
||||
needs:
|
||||
- build-container-image
|
||||
name: "build-deb (${{ matrix.distro }} ${{ matrix.version }})"
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- distro: ubuntu
|
||||
version: "22.04"
|
||||
- distro: ubuntu
|
||||
version: "24.04"
|
||||
- distro: ubuntu
|
||||
version: "24.10"
|
||||
- distro: ubuntu
|
||||
version: "25.04"
|
||||
- distro: debian
|
||||
version: bullseye
|
||||
- distro: debian
|
||||
version: bookworm
|
||||
- distro: debian
|
||||
version: trixie
|
||||
env:
|
||||
target: debian-bookworm
|
||||
distro: debian
|
||||
version: bookworm
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.10"
|
||||
python-version: '3.10'
|
||||
|
||||
- name: Login to GHCR
|
||||
- name: Build dev environment
|
||||
run: |
|
||||
echo ${{ github.token }} | podman login ghcr.io -u USERNAME --password-stdin
|
||||
./dev_scripts/env.py --distro ${{ env.distro }} \
|
||||
--version ${{ env.version }} \
|
||||
build-dev
|
||||
|
||||
- name: Get the dev environment
|
||||
run: |
|
||||
./dev_scripts/env.py \
|
||||
--distro ${{ matrix.distro }} \
|
||||
--version ${{ matrix.version }} \
|
||||
build-dev --sync
|
||||
- name: Install container build dependencies
|
||||
run: sudo apt install pipx && pipx install poetry
|
||||
|
||||
- name: Get current date
|
||||
id: date
|
||||
run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Restore container cache
|
||||
uses: actions/cache/restore@v4
|
||||
with:
|
||||
key: v5-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/*.py', 'dangerzone/container_helpers/*', 'install/common/build-image.py') }}
|
||||
path: |-
|
||||
share/container.tar
|
||||
share/image-id.txt
|
||||
fail-on-cache-miss: true
|
||||
- name: Build Dangerzone image
|
||||
run: python3 ./install/common/build-image.py
|
||||
|
||||
- name: Build Dangerzone .deb
|
||||
run: |
|
||||
./dev_scripts/env.py --distro ${{ matrix.distro }} \
|
||||
--version ${{ matrix.version }} \
|
||||
./dev_scripts/env.py --distro ${{ env.distro }} \
|
||||
--version ${{ env.version }} \
|
||||
run --dev --no-gui ./dangerzone/install/linux/build-deb.py
|
||||
|
||||
- name: Upload Dangerzone .deb
|
||||
if: matrix.distro == 'debian' && matrix.version == 'bookworm'
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: dangerzone.deb
|
||||
path: "deb_dist/dangerzone_*_*.deb"
|
||||
if-no-files-found: error
|
||||
compression-level: 0
|
||||
path: "deb_dist/dangerzone_*_all.deb"
|
||||
|
||||
install-deb:
|
||||
name: "install-deb (${{ matrix.distro }} ${{ matrix.version }})"
|
||||
runs-on: ubuntu-latest
|
||||
needs:
|
||||
- build-deb
|
||||
needs: build-deb
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- distro: ubuntu
|
||||
- target: ubuntu-20.04
|
||||
distro: ubuntu
|
||||
version: "20.04"
|
||||
- target: ubuntu-22.04
|
||||
distro: ubuntu
|
||||
version: "22.04"
|
||||
- distro: ubuntu
|
||||
- target: ubuntu-23.10
|
||||
distro: ubuntu
|
||||
version: "23.10"
|
||||
- target: ubuntu-24.04
|
||||
distro: ubuntu
|
||||
version: "24.04"
|
||||
- distro: ubuntu
|
||||
version: "24.10"
|
||||
- distro: ubuntu
|
||||
version: "25.04"
|
||||
- distro: debian
|
||||
- target: debian-bullseye
|
||||
distro: debian
|
||||
version: bullseye
|
||||
- distro: debian
|
||||
- target: debian-bookworm
|
||||
distro: debian
|
||||
version: bookworm
|
||||
- distro: debian
|
||||
- target: debian-trixie
|
||||
distro: debian
|
||||
version: trixie
|
||||
|
||||
steps:
|
||||
|
|
@ -276,7 +114,7 @@ jobs:
|
|||
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.10"
|
||||
python-version: '3.10'
|
||||
|
||||
- name: Download Dangerzone .deb
|
||||
uses: actions/download-artifact@v4
|
||||
|
|
@ -284,17 +122,41 @@ jobs:
|
|||
name: dangerzone.deb
|
||||
path: "deb_dist/"
|
||||
|
||||
- name: Build end-user environment
|
||||
- name: Create end-user environment on (${{ matrix.target }})
|
||||
run: |
|
||||
./dev_scripts/env.py --distro ${{ matrix.distro }} \
|
||||
--version ${{ matrix.version }} \
|
||||
build
|
||||
|
||||
- name: Configure Podman for Debian Bullseye specifically
|
||||
if: matrix.target == 'debian-bullseye'
|
||||
run: |
|
||||
# Create a Podman config specifically for Bullseye (see #388).
|
||||
mkdir bullseye_fix
|
||||
cd bullseye_fix
|
||||
cat > containers.conf <<EOF
|
||||
[engine]
|
||||
cgroup_manager="cgroupfs"
|
||||
events_logger="file"
|
||||
EOF
|
||||
|
||||
# Copy the Podman config into the container image we created for the
|
||||
# Dangerzone environment.
|
||||
cat > Dockerfile.bullseye <<EOF
|
||||
FROM dangerzone.rocks/debian:bullseye-backports
|
||||
RUN mkdir -p /home/user/.config/containers
|
||||
COPY containers.conf /home/user/.config/containers/
|
||||
EOF
|
||||
|
||||
# Create a new image from the Dangerzone environment and re-tag it.
|
||||
podman build -t dangerzone.rocks/debian:bullseye-backports \
|
||||
-f Dockerfile.bullseye .
|
||||
|
||||
- name: Run a test command
|
||||
run: |
|
||||
./dev_scripts/env.py --distro ${{ matrix.distro }} \
|
||||
--version ${{ matrix.version }} \
|
||||
run dangerzone-cli dangerzone/tests/test_docs/sample-pdf.pdf --ocr-lang eng
|
||||
run dangerzone-cli dangerzone/tests/test_docs/sample-pdf.pdf
|
||||
|
||||
- name: Check that the Dangerzone GUI imports work
|
||||
run: |
|
||||
|
|
@ -303,55 +165,34 @@ jobs:
|
|||
run dangerzone --help
|
||||
|
||||
build-install-rpm:
|
||||
name: "build-install-rpm (${{ matrix.distro }} ${{matrix.version}})"
|
||||
name: "Build and install a Dangerzone RPM on Fedora ${{matrix.version}}"
|
||||
runs-on: ubuntu-latest
|
||||
needs:
|
||||
- build-container-image
|
||||
strategy:
|
||||
matrix:
|
||||
distro: ["fedora"]
|
||||
version: ["40", "41", "42"]
|
||||
include:
|
||||
- version: "38"
|
||||
- version: "39"
|
||||
- version: "40"
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Login to GHCR
|
||||
- name: Build dev environment
|
||||
run: |
|
||||
echo ${{ github.token }} | podman login ghcr.io -u USERNAME --password-stdin
|
||||
./dev_scripts/env.py --distro fedora --version ${{ matrix.version }} \
|
||||
build-dev
|
||||
|
||||
- name: Get the dev environment
|
||||
- name: Build Dangerzone image
|
||||
run: |
|
||||
./dev_scripts/env.py \
|
||||
--distro ${{ matrix.distro }} \
|
||||
--version ${{ matrix.version }} \
|
||||
build-dev --sync
|
||||
|
||||
- name: Get current date
|
||||
id: date
|
||||
run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Restore container image
|
||||
uses: actions/cache/restore@v4
|
||||
with:
|
||||
key: v5-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/*.py', 'dangerzone/container_helpers/*', 'install/common/build-image.py') }}
|
||||
path: |-
|
||||
share/container.tar
|
||||
share/image-id.txt
|
||||
fail-on-cache-miss: true
|
||||
./dev_scripts/env.py --distro fedora --version ${{ matrix.version }} \
|
||||
run --dev --no-gui \
|
||||
bash -c 'cd /home/user/dangerzone && python3 ./install/common/build-image.py'
|
||||
|
||||
- name: Build Dangerzone .rpm
|
||||
run: |
|
||||
./dev_scripts/env.py --distro ${{ matrix.distro }} --version ${{ matrix.version }} \
|
||||
./dev_scripts/env.py --distro fedora --version ${{ matrix.version }} \
|
||||
run --dev --no-gui ./dangerzone/install/linux/build-rpm.py
|
||||
|
||||
- name: Upload Dangerzone .rpm
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: dangerzone-${{ matrix.distro }}-${{ matrix.version }}.rpm
|
||||
path: "dist/dangerzone-*.x86_64.rpm"
|
||||
if-no-files-found: error
|
||||
compression-level: 0
|
||||
|
||||
# Reclaim some space in this step, now that the dev environment is no
|
||||
# longer necessary. Previously, we encountered out-of-space issues while
|
||||
# running this CI job.
|
||||
|
|
@ -360,124 +201,15 @@ jobs:
|
|||
|
||||
- name: Build end-user environment
|
||||
run: |
|
||||
./dev_scripts/env.py --distro ${{ matrix.distro }} \
|
||||
--version ${{ matrix.version }} \
|
||||
build
|
||||
./dev_scripts/env.py --distro fedora --version ${{ matrix.version }} \
|
||||
build --download-pyside6
|
||||
|
||||
- name: Run a test command
|
||||
run: |
|
||||
./dev_scripts/env.py --distro ${{ matrix.distro }} --version ${{ matrix.version }} \
|
||||
run dangerzone-cli dangerzone/tests/test_docs/sample-pdf.pdf --ocr-lang eng
|
||||
./dev_scripts/env.py --distro fedora --version ${{ matrix.version }} \
|
||||
run dangerzone-cli dangerzone/tests/test_docs/sample-pdf.pdf
|
||||
|
||||
- name: Check that the Dangerzone GUI imports work
|
||||
run: |
|
||||
./dev_scripts/env.py --distro ${{ matrix.distro }} --version ${{ matrix.version }} \
|
||||
./dev_scripts/env.py --distro fedora --version ${{ matrix.version }} \
|
||||
run dangerzone --help
|
||||
|
||||
run-tests:
|
||||
name: "run tests (${{ matrix.distro }} ${{ matrix.version }})"
|
||||
runs-on: ubuntu-latest
|
||||
needs:
|
||||
- build-container-image
|
||||
- download-tessdata
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- distro: ubuntu
|
||||
version: "22.04"
|
||||
- distro: ubuntu
|
||||
version: "24.04"
|
||||
- distro: ubuntu
|
||||
version: "24.10"
|
||||
- distro: ubuntu
|
||||
version: "25.04"
|
||||
- distro: debian
|
||||
version: bullseye
|
||||
- distro: debian
|
||||
version: bookworm
|
||||
- distro: debian
|
||||
version: trixie
|
||||
- distro: fedora
|
||||
version: "40"
|
||||
- distro: fedora
|
||||
version: "41"
|
||||
- distro: fedora
|
||||
version: "42"
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.10"
|
||||
|
||||
- name: Login to GHCR
|
||||
run: |
|
||||
echo ${{ github.token }} | podman login ghcr.io -u USERNAME --password-stdin
|
||||
|
||||
- name: Get current date
|
||||
id: date
|
||||
run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Get the dev environment
|
||||
run: |
|
||||
./dev_scripts/env.py \
|
||||
--distro ${{ matrix.distro }} \
|
||||
--version ${{ matrix.version }} \
|
||||
build-dev --sync
|
||||
|
||||
- name: Restore container image
|
||||
uses: actions/cache/restore@v4
|
||||
with:
|
||||
key: v5-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/*.py', 'dangerzone/container_helpers/*', 'install/common/build-image.py') }}
|
||||
path: |-
|
||||
share/container.tar
|
||||
share/image-id.txt
|
||||
fail-on-cache-miss: true
|
||||
|
||||
- name: Restore cached tessdata
|
||||
uses: actions/cache/restore@v4
|
||||
with:
|
||||
path: share/tessdata/
|
||||
enableCrossOsArchive: true
|
||||
fail-on-cache-miss: true
|
||||
key: v1-tessdata-${{ hashFiles('./install/common/download-tessdata.py') }}
|
||||
|
||||
- name: Setup xvfb (Linux)
|
||||
run: |
|
||||
sudo apt update
|
||||
# Stuff copied wildly from several stackoverflow posts
|
||||
sudo apt-get install -y xvfb libxkbcommon-x11-0 libxcb-icccm4 libxcb-image0 libxcb-keysyms1 libxcb-randr0 libxcb-render-util0 libxcb-xinerama0 libxcb-xinput0 libxcb-xfixes0 libxcb-shape0 libglib2.0-0 libgl1-mesa-dev '^libxcb.*-dev' libx11-xcb-dev libglu1-mesa-dev libxrender-dev libxi-dev libxkbcommon-dev libxkbcommon-x11-dev
|
||||
|
||||
# start xvfb in the background
|
||||
sudo /usr/bin/Xvfb $DISPLAY -screen 0 1280x1024x24 &
|
||||
|
||||
- name: Run CI tests
|
||||
run: |-
|
||||
# Pass the -ac Xserver flag, to disable host-based access controls.
|
||||
# This should be used ONLY for testing [1]. If we don't pass this
|
||||
# flag, the Podman container is not authorized [2] to access the Xvfb
|
||||
# server.
|
||||
#
|
||||
# [1] From https://www.x.org/releases/X11R6.7.0/doc/Xserver.1.html#sect4:
|
||||
#
|
||||
# disables host-based access control mechanisms. Enables access by
|
||||
# any host, and permits any host to modify the access control
|
||||
# list. Use with extreme caution. This option exists primarily for
|
||||
# running test suites remotely.
|
||||
#
|
||||
# [2] Fails with "Authorization required, but no authorization
|
||||
# protocol specified". However, we have verified with strace(1)
|
||||
# that the command in the Podman container can read the Xauthority
|
||||
# file successfully.
|
||||
xvfb-run -s '-ac' ./dev_scripts/env.py --distro ${{ matrix.distro }} --version ${{ matrix.version }} run --dev \
|
||||
bash -c 'cd dangerzone; poetry run make test'
|
||||
|
||||
- name: Upload PDF diffs
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: pdf-diffs-${{ matrix.distro }}-${{ matrix.version }}
|
||||
path: tests/test_docs/diffs/*.jpeg
|
||||
# Always run this step to publish test results, even on failures
|
||||
if: ${{ always() }}
|
||||
|
|
|
|||
22
.github/workflows/close-issues.yml
vendored
22
.github/workflows/close-issues.yml
vendored
|
|
@ -1,22 +0,0 @@
|
|||
name: Close inactive issues
|
||||
on:
|
||||
schedule:
|
||||
- cron: "30 1 * * *"
|
||||
|
||||
jobs:
|
||||
close-issues:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
issues: write
|
||||
steps:
|
||||
- uses: actions/stale@v9
|
||||
with:
|
||||
days-before-issue-stale: 30
|
||||
days-before-issue-close: 14
|
||||
stale-issue-label: "stale"
|
||||
stale-issue-message: "Marking this issue as stale because it has been open for 30 days with no activity. It will be closed in 14 days if there's no activity, or if the `stale` label is not removed. Does anyone want to add something?"
|
||||
close-issue-message: "Closing this issue now. Don't hesitate to reopen if you have anything to add :-)"
|
||||
days-before-pr-stale: -1
|
||||
days-before-pr-close: -1
|
||||
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
any-of-labels: needs info
|
||||
22
.github/workflows/release-container-image.yml
vendored
22
.github/workflows/release-container-image.yml
vendored
|
|
@ -1,22 +0,0 @@
|
|||
name: Release multi-arch container image
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
- "test/**"
|
||||
schedule:
|
||||
- cron: "0 0 * * *" # Run every day at 00:00 UTC.
|
||||
|
||||
|
||||
jobs:
|
||||
build-push-image:
|
||||
uses: ./.github/workflows/build-push-image.yml
|
||||
with:
|
||||
registry: ghcr.io/${{ github.repository_owner }}
|
||||
registry_user: ${{ github.actor }}
|
||||
image_name: dangerzone/dangerzone
|
||||
reproduce: true
|
||||
secrets:
|
||||
registry_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
43
.github/workflows/scan.yml
vendored
43
.github/workflows/scan.yml
vendored
|
|
@ -1,42 +1,28 @@
|
|||
name: Scan latest app and container
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
pull_request:
|
||||
branches: [ main ]
|
||||
schedule:
|
||||
- cron: '0 0 * * *' # Run every day at 00:00 UTC.
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
security-scan-container:
|
||||
strategy:
|
||||
matrix:
|
||||
runs-on:
|
||||
- ubuntu-24.04
|
||||
- ubuntu-24.04-arm
|
||||
runs-on: ${{ matrix.runs-on }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Install container build dependencies
|
||||
run: sudo apt install pipx && pipx install poetry
|
||||
- name: Build container image
|
||||
run: |
|
||||
python3 ./install/common/build-image.py \
|
||||
--debian-archive-date $(date "+%Y%m%d") \
|
||||
--runtime docker
|
||||
docker load -i share/container.tar
|
||||
- name: Get image tag
|
||||
id: tag
|
||||
run: echo "tag=$(cat share/image-id.txt)" >> $GITHUB_OUTPUT
|
||||
run: python3 ./install/common/build-image.py --runtime docker --no-save
|
||||
# NOTE: Scan first without failing, else we won't be able to read the scan
|
||||
# report.
|
||||
- name: Scan container image (no fail)
|
||||
uses: anchore/scan-action@v6
|
||||
uses: anchore/scan-action@v3
|
||||
id: scan_container
|
||||
with:
|
||||
image: "dangerzone.rocks/dangerzone:${{ steps.tag.outputs.tag }}"
|
||||
image: "dangerzone.rocks/dangerzone:latest"
|
||||
fail-build: false
|
||||
only-fixed: false
|
||||
severity-cutoff: critical
|
||||
|
|
@ -48,27 +34,22 @@ jobs:
|
|||
- name: Inspect container scan report
|
||||
run: cat ${{ steps.scan_container.outputs.sarif }}
|
||||
- name: Scan container image
|
||||
uses: anchore/scan-action@v6
|
||||
uses: anchore/scan-action@v3
|
||||
with:
|
||||
image: "dangerzone.rocks/dangerzone:${{ steps.tag.outputs.tag }}"
|
||||
image: "dangerzone.rocks/dangerzone:latest"
|
||||
fail-build: true
|
||||
only-fixed: false
|
||||
severity-cutoff: critical
|
||||
|
||||
security-scan-app:
|
||||
strategy:
|
||||
matrix:
|
||||
runs-on:
|
||||
- ubuntu-24.04
|
||||
- ubuntu-24.04-arm
|
||||
runs-on: ${{ matrix.runs-on }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
# NOTE: Scan first without failing, else we won't be able to read the scan
|
||||
# report.
|
||||
- name: Scan application (no fail)
|
||||
uses: anchore/scan-action@v6
|
||||
uses: anchore/scan-action@v3
|
||||
id: scan_app
|
||||
with:
|
||||
path: "."
|
||||
|
|
@ -83,7 +64,7 @@ jobs:
|
|||
- name: Inspect application scan report
|
||||
run: cat ${{ steps.scan_app.outputs.sarif }}
|
||||
- name: Scan application
|
||||
uses: anchore/scan-action@v6
|
||||
uses: anchore/scan-action@v3
|
||||
with:
|
||||
path: "."
|
||||
fail-build: true
|
||||
|
|
|
|||
50
.github/workflows/scan_released.yml
vendored
50
.github/workflows/scan_released.yml
vendored
|
|
@ -2,39 +2,26 @@ name: Scan released app and container
|
|||
on:
|
||||
schedule:
|
||||
- cron: '0 0 * * *' # Run every day at 00:00 UTC.
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
security-scan-container:
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- runs-on: ubuntu-24.04
|
||||
arch: i686
|
||||
- runs-on: ubuntu-24.04-arm
|
||||
arch: arm64
|
||||
runs-on: ${{ matrix.runs-on }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
- name: Download container image for the latest release and load it
|
||||
- name: Download container image for the latest release
|
||||
run: |
|
||||
VERSION=$(curl https://api.github.com/repos/freedomofpress/dangerzone/releases/latest | grep "tag_name" | cut -d '"' -f 4)
|
||||
CONTAINER_FILENAME=container-${VERSION:1}-${{ matrix.arch }}.tar
|
||||
wget https://github.com/freedomofpress/dangerzone/releases/download/${VERSION}/${CONTAINER_FILENAME} -O ${CONTAINER_FILENAME}
|
||||
docker load -i ${CONTAINER_FILENAME}
|
||||
- name: Get image tag
|
||||
id: tag
|
||||
run: |
|
||||
tag=$(docker images dangerzone.rocks/dangerzone --format '{{ .Tag }}')
|
||||
echo "tag=$tag" >> $GITHUB_OUTPUT
|
||||
VERSION=$(curl https://api.github.com/repos/freedomofpress/dangerzone/releases/latest | jq -r '.tag_name')
|
||||
wget https://github.com/freedomofpress/dangerzone/releases/download/${VERSION}/container.tar.gz
|
||||
- name: Load container image
|
||||
run: docker load -i container.tar.gz
|
||||
# NOTE: Scan first without failing, else we won't be able to read the scan
|
||||
# report.
|
||||
- name: Scan container image (no fail)
|
||||
uses: anchore/scan-action@v6
|
||||
uses: anchore/scan-action@v3
|
||||
id: scan_container
|
||||
with:
|
||||
image: "dangerzone.rocks/dangerzone:${{ steps.tag.outputs.tag }}"
|
||||
image: "dangerzone.rocks/dangerzone:latest"
|
||||
fail-build: false
|
||||
only-fixed: false
|
||||
severity-cutoff: critical
|
||||
|
|
@ -42,24 +29,19 @@ jobs:
|
|||
uses: github/codeql-action/upload-sarif@v3
|
||||
with:
|
||||
sarif_file: ${{ steps.scan_container.outputs.sarif }}
|
||||
category: container-${{ matrix.arch }}
|
||||
category: container
|
||||
- name: Inspect container scan report
|
||||
run: cat ${{ steps.scan_container.outputs.sarif }}
|
||||
- name: Scan container image
|
||||
uses: anchore/scan-action@v6
|
||||
uses: anchore/scan-action@v3
|
||||
with:
|
||||
image: "dangerzone.rocks/dangerzone:${{ steps.tag.outputs.tag }}"
|
||||
image: "dangerzone.rocks/dangerzone:latest"
|
||||
fail-build: true
|
||||
only-fixed: false
|
||||
severity-cutoff: critical
|
||||
|
||||
security-scan-app:
|
||||
strategy:
|
||||
matrix:
|
||||
runs-on:
|
||||
- ubuntu-24.04
|
||||
- ubuntu-24.04-arm
|
||||
runs-on: ${{ matrix.runs-on }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
|
@ -67,16 +49,12 @@ jobs:
|
|||
fetch-depth: 0
|
||||
- name: Checkout the latest released tag
|
||||
run: |
|
||||
# Grab the latest Grype ignore list before git checkout overwrites it.
|
||||
cp .grype.yaml .grype.yaml.new
|
||||
VERSION=$(curl https://api.github.com/repos/freedomofpress/dangerzone/releases/latest | jq -r '.tag_name')
|
||||
git checkout $VERSION
|
||||
# Restore the newest Grype ignore list.
|
||||
mv .grype.yaml.new .grype.yaml
|
||||
# NOTE: Scan first without failing, else we won't be able to read the scan
|
||||
# report.
|
||||
- name: Scan application (no fail)
|
||||
uses: anchore/scan-action@v6
|
||||
uses: anchore/scan-action@v3
|
||||
id: scan_app
|
||||
with:
|
||||
path: "."
|
||||
|
|
@ -91,7 +69,7 @@ jobs:
|
|||
- name: Inspect application scan report
|
||||
run: cat ${{ steps.scan_app.outputs.sarif }}
|
||||
- name: Scan application
|
||||
uses: anchore/scan-action@v6
|
||||
uses: anchore/scan-action@v3
|
||||
with:
|
||||
path: "."
|
||||
fail-build: true
|
||||
|
|
|
|||
13
.gitignore
vendored
13
.gitignore
vendored
|
|
@ -22,7 +22,6 @@ var/
|
|||
wheels/
|
||||
pip-wheel-metadata/
|
||||
share/python-wheels/
|
||||
share/tessdata/
|
||||
*.egg-info/
|
||||
.installed.cfg
|
||||
*.egg
|
||||
|
|
@ -128,15 +127,6 @@ dmypy.json
|
|||
# Pyre type checker
|
||||
.pyre/
|
||||
|
||||
# Debian packaging
|
||||
|
||||
debian/.debhelper
|
||||
debian/dangerzone
|
||||
debian/files
|
||||
debian/debhelper-build-stamp
|
||||
debian/dangerzone.*
|
||||
.pybuild/
|
||||
|
||||
# Other
|
||||
.vscode
|
||||
*.tar.gz
|
||||
|
|
@ -148,5 +138,4 @@ install/windows/Dangerzone.wxs
|
|||
share/container.tar
|
||||
share/container.tar.gz
|
||||
share/image-id.txt
|
||||
container/container-pip-requirements.txt
|
||||
.doit.db.db
|
||||
container/container-pip-requirements.txt
|
||||
65
.grype.yaml
65
.grype.yaml
|
|
@ -2,55 +2,24 @@
|
|||
# latest release of Dangerzone, and offer our analysis.
|
||||
|
||||
ignore:
|
||||
# CVE-2023-45853
|
||||
# ==============
|
||||
# CVE-2023-7104
|
||||
# =============
|
||||
#
|
||||
# Debian tracker: https://security-tracker.debian.org/tracker/CVE-2023-45853
|
||||
# Verdict: Dangerzone is not affected because the zlib library in Debian is
|
||||
# built in a way that is not vulnerable.
|
||||
- vulnerability: CVE-2023-45853
|
||||
# CVE-2024-38428
|
||||
# ==============
|
||||
# NVD Entry: https://nvd.nist.gov/vuln/detail/CVE-2023-7104
|
||||
# Verdict: Dangerzone is not affected. The rationale is the following:
|
||||
#
|
||||
# Debian tracker: https://security-tracker.debian.org/tracker/CVE-2024-38428
|
||||
# Verdict: Dangerzone is not affected because it doesn't use wget in the
|
||||
# container image (which also has no network connectivity).
|
||||
- vulnerability: CVE-2024-38428
|
||||
# CVE-2024-57823
|
||||
# ==============
|
||||
# 1. This CVE affects malicious/corrupted SQLite DBs.
|
||||
# 2. Databases can be loaded either via LibreOffice Calc or Base. Files for
|
||||
# the latter are not a valid input to Dangerzone.
|
||||
# 3. Based on the LibreOffice Calc guide [1], users can only refer to
|
||||
# external databases, not embed them in a spreadsheet.
|
||||
# 4. The actual CVSS score for this vulnerability is High, according to
|
||||
# NIST, not Critical.
|
||||
#
|
||||
# Debian tracker: https://security-tracker.debian.org/tracker/CVE-2024-57823
|
||||
# Verdict: Dangerzone is not affected. First things first, LibreOffice is
|
||||
# using this library for parsing RDF metadata in a document [1], and has
|
||||
# issued a fix for the vendored raptor2 package they have for other distros
|
||||
# [2].
|
||||
# [1]: From https://wiki.documentfoundation.org/images/f/f4/CG75-CalcGuide.pdf:
|
||||
#
|
||||
# On the other hand, the Debian security team has stated that this is a minor
|
||||
# issue [3], and there's no fix from the developers yet. It seems that the
|
||||
# Debian package is not affected somehow by this CVE, probably due to the way
|
||||
# it's packaged.
|
||||
#
|
||||
# [1] https://wiki.documentfoundation.org/Documentation/DevGuide/Office_Development#RDF_metadata
|
||||
# [2] https://cgit.freedesktop.org/libreoffice/core/commit/?id=2b50dc0e4482ac0ad27d69147b4175e05af4fba4
|
||||
# [2] From https://security-tracker.debian.org/tracker/CVE-2024-57823:
|
||||
#
|
||||
# [bookworm] - raptor2 <postponed> (Minor issue, revisit when fixed upstream)
|
||||
#
|
||||
- vulnerability: CVE-2024-57823
|
||||
# CVE-2025-0665
|
||||
# ==============
|
||||
#
|
||||
# Debian tracker: https://security-tracker.debian.org/tracker/CVE-2025-0665
|
||||
# Verdict: Dangerzone is not affected because the vulnerable code is not
|
||||
# present in Debian Bookworm. Also, libcurl is an HTTP client, and the
|
||||
# Dangerzone container does not make any network calls.
|
||||
- vulnerability: CVE-2025-0665
|
||||
# CVE-2025-43859
|
||||
# ==============
|
||||
#
|
||||
# GitHub advisory: https://github.com/advisories/GHSA-vqfr-h8mv-ghfj
|
||||
# Verdict: Dangerzone is not affected because the vulnerable code is triggered
|
||||
# when parsing HTTP requests, e.g., by web **servers**. Dangerzone on the
|
||||
# other hand performs HTTP requests, i.e., it operates as **client**.
|
||||
- vulnerability: CVE-2025-43859
|
||||
- vulnerability: GHSA-vqfr-h8mv-ghfj
|
||||
# > The possible data sources for the pivot table are a Calc spreadsheet
|
||||
# > or an external data source that is registered in LibreOffice. [...]
|
||||
# > A registered data source is a connection to data held in a database
|
||||
# > outside of LibreOffice.
|
||||
- vulnerability: CVE-2023-7104
|
||||
|
|
|
|||
|
|
@ -1 +0,0 @@
|
|||
https://dangerzone.rocks/assets/json/funding.json
|
||||
99
BUILD.md
99
BUILD.md
|
|
@ -33,10 +33,9 @@ Install dependencies:
|
|||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
```sh
|
||||
sudo apt install -y podman dh-python build-essential make libqt6gui6 \
|
||||
pipx python3 python3-dev
|
||||
sudo apt install -y podman dh-python build-essential fakeroot make libqt6gui6 \
|
||||
pipx python3 python3-dev python3-stdeb python3-all
|
||||
```
|
||||
|
||||
Install Poetry using `pipx` (recommended) and add it to your `$PATH`:
|
||||
|
|
@ -47,7 +46,6 @@ methods](https://python-poetry.org/docs/#installation))_
|
|||
```sh
|
||||
pipx ensurepath
|
||||
pipx install poetry
|
||||
pipx inject poetry poetry-plugin-export
|
||||
```
|
||||
|
||||
After this, restart the terminal window, for the `poetry` command to be in your
|
||||
|
|
@ -75,12 +73,6 @@ Build the latest container:
|
|||
python3 ./install/common/build-image.py
|
||||
```
|
||||
|
||||
Download the OCR language data:
|
||||
|
||||
```sh
|
||||
python3 ./install/common/download-tessdata.py
|
||||
```
|
||||
|
||||
Run from source tree:
|
||||
|
||||
```sh
|
||||
|
|
@ -113,7 +105,6 @@ Install Poetry using `pipx`:
|
|||
|
||||
```sh
|
||||
pipx install poetry
|
||||
pipx inject poetry
|
||||
```
|
||||
|
||||
Clone this repository:
|
||||
|
|
@ -137,12 +128,6 @@ Build the latest container:
|
|||
python3 ./install/common/build-image.py
|
||||
```
|
||||
|
||||
Download the OCR language data:
|
||||
|
||||
```sh
|
||||
python3 ./install/common/download-tessdata.py
|
||||
```
|
||||
|
||||
Run from source tree:
|
||||
|
||||
```sh
|
||||
|
|
@ -187,27 +172,27 @@ Overview of the qubes you'll create:
|
|||
|--------------|----------|---------|
|
||||
| dz | app qube | Dangerzone development |
|
||||
| dz-dvm | app qube | offline disposable template for performing conversions |
|
||||
| fedora-41-dz | template | template for the other two qubes |
|
||||
| fedora-38-dz | template | template for the other two qubes |
|
||||
|
||||
#### In `dom0`:
|
||||
|
||||
The following instructions require typing commands in a terminal in dom0.
|
||||
|
||||
1. Create a new Fedora **template** (`fedora-41-dz`) for Dangerzone development:
|
||||
1. Create a new Fedora **template** (`fedora-38-dz`) for Dangerzone development:
|
||||
|
||||
```
|
||||
qvm-clone fedora-41 fedora-41-dz
|
||||
qvm-clone fedora-38 fedora-38-dz
|
||||
```
|
||||
|
||||
> :bulb: Alternatively, you can use your base Fedora 40 template in the
|
||||
> :bulb: Alternatively, you can use your base Fedora 38 template in the
|
||||
> following instructions. In that case, skip this step and replace
|
||||
> `fedora-41-dz` with `fedora-41` in the steps below.
|
||||
> `fedora-38-dz` with `fedora-38` in the steps below.
|
||||
|
||||
2. Create an offline disposable template (app qube) called `dz-dvm`, based on the `fedora-41-dz`
|
||||
2. Create an offline disposable template (app qube) called `dz-dvm`, based on the `fedora-38-dz`
|
||||
template. This will be the qube where the documents will be sanitized:
|
||||
|
||||
```
|
||||
qvm-create --class AppVM --label red --template fedora-41-dz \
|
||||
qvm-create --class AppVM --label red --template fedora-38-dz \
|
||||
--prop netvm="" --prop template_for_dispvms=True \
|
||||
--prop default_dispvm='' dz-dvm
|
||||
```
|
||||
|
|
@ -216,18 +201,12 @@ The following instructions require typing commands in a terminal in dom0.
|
|||
and initiating the sanitization process:
|
||||
|
||||
```
|
||||
qvm-create --class AppVM --label red --template fedora-41-dz dz
|
||||
qvm-volume resize dz:private $(numfmt --from=auto 20Gi)
|
||||
qvm-create --class AppVM --label red --template fedora-38-dz dz
|
||||
```
|
||||
|
||||
> :bulb: Alternatively, you can use a different app qube for Dangerzone
|
||||
> development. In that case, replace `dz` with the qube of your choice in the
|
||||
> steps below.
|
||||
>
|
||||
> In the commands above, we also resize the private volume of the `dz` qube
|
||||
> to 20GiB, since you may need some extra storage space when developing on
|
||||
> Dangerzone (e.g., for container images, Tesseract data, and Python
|
||||
> virtualenvs).
|
||||
|
||||
4. Add an RPC policy (`/etc/qubes/policy.d/50-dangerzone.policy`) that will
|
||||
allow launching a disposable qube (`dz-dvm`) when Dangerzone converts a
|
||||
|
|
@ -253,7 +232,10 @@ test it.
|
|||
cd dangerzone
|
||||
```
|
||||
|
||||
2. Follow the Fedora instructions for setting up the development environment.
|
||||
2. Follow the Fedora instructions for setting up the development environment with the particularity of running the following instead of `poetry install`:
|
||||
```
|
||||
poetry install --with qubes
|
||||
```
|
||||
|
||||
3. Build a dangerzone `.rpm` for qubes with the command
|
||||
|
||||
|
|
@ -261,12 +243,12 @@ test it.
|
|||
./install/linux/build-rpm.py --qubes
|
||||
```
|
||||
|
||||
4. Copy the produced `.rpm` file into `fedora-41-dz`
|
||||
4. Copy the produced `.rpm` file into `fedora-38-dz`
|
||||
```sh
|
||||
qvm-copy dist/*.x86_64.rpm
|
||||
```
|
||||
|
||||
#### In the `fedora-41-dz` template
|
||||
#### In the `fedora-38-dz` template
|
||||
|
||||
1. Install the `.rpm` package you just copied
|
||||
|
||||
|
|
@ -274,7 +256,7 @@ test it.
|
|||
sudo dnf install ~/QubesIncoming/dz/*.rpm
|
||||
```
|
||||
|
||||
2. Shutdown the `fedora-41-dz` template
|
||||
2. Shutdown the `fedora-38-dz` template
|
||||
|
||||
### Developing Dangerzone
|
||||
|
||||
|
|
@ -305,7 +287,7 @@ For changes in the server side components, you can simply edit them locally,
|
|||
and they will be mirrored to the disposable qube through the `dz.ConvertDev`
|
||||
RPC call.
|
||||
|
||||
The only reason to build a new Qubes RPM and install it in the `fedora-41-dz`
|
||||
The only reason to build a new Qubes RPM and install it in the `fedora-38-dz`
|
||||
template for development is if:
|
||||
1. The project requires new server-side components.
|
||||
2. The code for `qubes/dz.ConvertDev` needs to be updated.
|
||||
|
|
@ -314,7 +296,7 @@ template for development is if:
|
|||
|
||||
Install [Docker Desktop](https://www.docker.com/products/docker-desktop). Make sure to choose your correct CPU, either Intel Chip or Apple Chip.
|
||||
|
||||
Install the latest version of Python 3.12 [from python.org](https://www.python.org/downloads/macos/), and make sure `/Library/Frameworks/Python.framework/Versions/3.12/bin` is in your `PATH`.
|
||||
Install the latest version of Python 3.11 [from python.org](https://www.python.org/downloads/macos/), and make sure `/Library/Frameworks/Python.framework/Versions/3.11/bin` is in your `PATH`.
|
||||
|
||||
Clone this repository:
|
||||
|
||||
|
|
@ -342,12 +324,6 @@ Build the dangerzone container image:
|
|||
python3 ./install/common/build-image.py
|
||||
```
|
||||
|
||||
Download the OCR language data:
|
||||
|
||||
```sh
|
||||
python3 ./install/common/download-tessdata.py
|
||||
```
|
||||
|
||||
Run from source tree:
|
||||
|
||||
```sh
|
||||
|
|
@ -379,7 +355,7 @@ The output is in the `dist` folder.
|
|||
|
||||
Install [Docker Desktop](https://www.docker.com/products/docker-desktop).
|
||||
|
||||
Install the latest version of Python 3.12 (64-bit) [from python.org](https://www.python.org/downloads/windows/). Make sure to check the "Add Python 3.12 to PATH" checkbox on the first page of the installer.
|
||||
Install the latest version of Python 3.11 (64-bit) [from python.org](https://www.python.org/downloads/windows/). Make sure to check the "Add Python 3.11 to PATH" checkbox on the first page of the installer.
|
||||
|
||||
|
||||
Install Microsoft Visual C++ 14.0 or greater. Get it with ["Microsoft C++ Build Tools"](https://visualstudio.microsoft.com/visual-cpp-build-tools/) and make sure to select "Desktop development with C++" when installing.
|
||||
|
|
@ -409,12 +385,6 @@ Build the dangerzone container image:
|
|||
python3 .\install\common\build-image.py
|
||||
```
|
||||
|
||||
Download the OCR language data:
|
||||
|
||||
```sh
|
||||
python3 .\install\common\download-tessdata.py
|
||||
```
|
||||
|
||||
After that you can launch dangerzone during development with:
|
||||
|
||||
```
|
||||
|
|
@ -428,24 +398,11 @@ poetry shell
|
|||
.\dev_scripts\dangerzone.bat
|
||||
```
|
||||
|
||||
### If you want to build the Windows installer
|
||||
### If you want to build the installer
|
||||
|
||||
Install [.NET SDK](https://dotnet.microsoft.com/en-us/download) version 6 or later. Then, open a terminal and install the latest version of [WiX Toolset .NET tool](https://wixtoolset.org/) **v5** with:
|
||||
|
||||
```sh
|
||||
dotnet tool install --global wix --version 5.0.2
|
||||
```
|
||||
|
||||
Install the WiX UI extension. You may need to open a new terminal in order to use the newly installed `wix` .NET tool:
|
||||
|
||||
```sh
|
||||
wix extension add --global WixToolset.UI.wixext/5.0.2
|
||||
```
|
||||
|
||||
> [!IMPORTANT]
|
||||
> To avoid compatibility issues, ensure the WiX UI extension version matches the version of the WiX Toolset.
|
||||
>
|
||||
> Run `wix --version` to check the version of WiX Toolset you have installed and replace `5.x.y` with the full version number without the Git revision.
|
||||
* Go to https://dotnet.microsoft.com/download/dotnet-framework and download and install .NET Framework 3.5 SP1 Runtime. I downloaded `dotnetfx35.exe`.
|
||||
* Go to https://wixtoolset.org/releases/ and download and install WiX toolset. I downloaded `wix314.exe`.
|
||||
* Add `C:\Program Files (x86)\WiX Toolset v3.14\bin` to the path ([instructions](https://web.archive.org/web/20230221104142/https://windowsloop.com/how-to-add-to-windows-path/)).
|
||||
|
||||
### If you want to sign binaries with Authenticode
|
||||
|
||||
|
|
@ -459,7 +416,7 @@ Open a command prompt, cd into the dangerzone directory, and run:
|
|||
poetry run python .\setup-windows.py build
|
||||
```
|
||||
|
||||
In `build\exe.win32-3.12\` you will find `dangerzone.exe`, `dangerzone-cli.exe`, and all supporting files.
|
||||
In `build\exe.win32-3.11\` you will find `dangerzone.exe`, `dangerzone-cli.exe`, and all supporting files.
|
||||
|
||||
### To build the installer
|
||||
|
||||
|
|
@ -470,9 +427,3 @@ poetry run .\install\windows\build-app.bat
|
|||
```
|
||||
|
||||
When you're done you will have `dist\Dangerzone.msi`.
|
||||
|
||||
## Updating the container image
|
||||
|
||||
The Dangezone container image is reproducible. This means that every time we
|
||||
build it, the result will be bit-for-bit the same, with some minor exceptions.
|
||||
Read more on how you can update it in `docs/developer/reproducibility.md`.
|
||||
|
|
|
|||
172
CHANGELOG.md
172
CHANGELOG.md
|
|
@ -5,177 +5,7 @@ All notable changes to this project will be documented in this file.
|
|||
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
|
||||
since 0.4.1, and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
||||
|
||||
## [Unreleased](https://github.com/freedomofpress/dangerzone/compare/v0.9.0...HEAD)
|
||||
|
||||
## Changed
|
||||
|
||||
- Update installation instructions (and CI checks) for Debian derivatives ([#1141](https://github.com/freedomofpress/dangerzone/pull/1141))
|
||||
|
||||
## [0.9.0](https://github.com/freedomofpress/dangerzone/compare/v0.9.0...0.8.1)
|
||||
|
||||
### Added
|
||||
|
||||
- Platform support: Add support for Fedora 42 ([#1091](https://github.com/freedomofpress/dangerzone/issues/1091))
|
||||
- Platform support: Add support for Ubuntu 25.04 (Plucky Puffin) ([#1090](https://github.com/freedomofpress/dangerzone/issues/1090))
|
||||
- (experimental): It is now possible to specify a custom container runtime in
|
||||
the settings, by using the `container_runtime` key. It should contain the path
|
||||
to the container runtime you want to use. Please note that this doesn't mean
|
||||
we support more container runtimes than Podman and Docker for the time being,
|
||||
but enables you to chose which one you want to use, independently of your
|
||||
platform. ([#925](https://github.com/freedomofpress/dangerzone/issues/925))
|
||||
- Document Operating System support [#986](https://github.com/freedomofpress/dangerzone/issues/986)
|
||||
- Tests: Look for regressions when converting PDFs [#321](https://github.com/freedomofpress/dangerzone/issues/321)
|
||||
- Ensure container image reproducibilty across different container runtimes and versions ([#1074](https://github.com/freedomofpress/dangerzone/issues/1074))
|
||||
- Implement container image attestations ([#1035](https://github.com/freedomofpress/dangerzone/issues/1035))
|
||||
- Inform user of outdated Docker Desktop Version ([#693](https://github.com/freedomofpress/dangerzone/issues/693))
|
||||
- Add support for Python 3.13 ([#992](https://github.com/freedomofpress/dangerzone/issues/992))
|
||||
- Publish the built artifacts in our CI pipelines ([#972](https://github.com/freedomofpress/dangerzone/pull/972))
|
||||
|
||||
### Fixed
|
||||
|
||||
- Fix our Debian Trixie installation instructions using Sequoia PGP ([#1052](https://github.com/freedomofpress/dangerzone/issues/1052))
|
||||
- Fix the way multiprocessing works on macOS ([#873](https://github.com/freedomofpress/dangerzone/issues/873))
|
||||
- Update minimum Docker Desktop version to fix an stdout truncation issue ([#1101](https://github.com/freedomofpress/dangerzone/issues/1101))
|
||||
|
||||
### Removed
|
||||
|
||||
- Platform support: Drop support for Ubuntu Focal, since it's nearing end-of-life ([#1018](https://github.com/freedomofpress/dangerzone/issues/1018))
|
||||
- Platform support: Drop support for Fedora 39 ([#999](https://github.com/freedomofpress/dangerzone/issues/999))
|
||||
|
||||
## Changed
|
||||
|
||||
- Switch base image to Debian Stable ([#1046](https://github.com/freedomofpress/dangerzone/issues/1046))
|
||||
- Track image tags instead of image IDs in `image-id.txt` ([#1020](https://github.com/freedomofpress/dangerzone/issues/1020))
|
||||
- Migrate to Wix 4 (windows building tool) ([#602](https://github.com/freedomofpress/dangerzone/issues/602)).
|
||||
Thanks [@jkarasti](https://github.com/jkarasti) for the contribution.
|
||||
- Add a `--debug` flag to the CLI to help retrieve more logs ([#941](https://github.com/freedomofpress/dangerzone/pull/941))
|
||||
- The `debian` base image is now fetched by digest. As a result, your local
|
||||
container storage will no longer show a tag for this dependency
|
||||
([#1116](https://github.com/freedomofpress/dangerzone/pull/1116)).
|
||||
Thanks [@sudoforge](https://github.com/sudoforge) for the contribution.
|
||||
- The `debian` base image is now referenced with a fully qualified URI,
|
||||
including the registry hostname ([#1118](https://github.com/freedomofpress/dangerzone/pull/1118)).
|
||||
Thanks [@sudoforge](https://github.com/sudoforge) for the contribution.
|
||||
- Update the Dangerzone container image and its dependencies (gVisor, Debian base image, H2Orestart) to the latest versions:
|
||||
* Debian image release: `bookworm-20250317-slim@sha256:1209d8fd77def86ceb6663deef7956481cc6c14a25e1e64daec12c0ceffcc19d`
|
||||
* Debian snapshots date: `2025-03-31`
|
||||
* gVisor release date: `2025-03-26`
|
||||
* H2Orestart plugin: `v0.7.2` (`d09bc5c93fe2483a7e4a57985d2a8d0e4efae2efb04375fe4b59a68afd7241e2`)
|
||||
|
||||
### Development changes
|
||||
|
||||
- Make container image scanning work for Silicon macOS ([#1008](https://github.com/freedomofpress/dangerzone/issues/1008))
|
||||
- Automate the main bulk of our release tasks ([#1016](https://github.com/freedomofpress/dangerzone/issues/1016))
|
||||
- CI: Enforce updating the CHANGELOG in the CI ([#1108](https://github.com/freedomofpress/dangerzone/pull/1108))
|
||||
- Add reference to funding.json (required by floss.fund application) ([#1092](https://github.com/freedomofpress/dangerzone/pull/1092))
|
||||
- Lint: add ruff for linting and formatting ([#1029](https://github.com/freedomofpress/dangerzone/pull/1029)).
|
||||
Thanks [@jkarasti](https://github.com/jkarasti) for the contribution.
|
||||
- Work around a `cx_freeze` build issue ([#974](https://github.com/freedomofpress/dangerzone/issues/974))
|
||||
- tests: mark the hancom office suite tests for rerun on failures ([#991](https://github.com/freedomofpress/dangerzone/pull/991))
|
||||
- Update reference template for Qubes to Fedora 41 ([#1078](https://github.com/freedomofpress/dangerzone/issues/1078))
|
||||
|
||||
## [0.8.1](https://github.com/freedomofpress/dangerzone/compare/v0.8.1...0.8.0)
|
||||
|
||||
- Update the container image
|
||||
|
||||
### Added
|
||||
|
||||
- Disable gVisor's DirectFS feature ([#226](https://github.com/freedomofpress/dangerzone/issues/226)).
|
||||
Thanks [EtiennePerot](https://github.com/EtiennePerot) for the contribution.
|
||||
|
||||
### Removed
|
||||
|
||||
- Platform support: Drop support for Fedora 39, since it's end-of-life ([#999](https://github.com/freedomofpress/dangerzone/pull/999))
|
||||
|
||||
## Updated
|
||||
|
||||
- Bump `slsa-framework/slsa-github-generator` from 2.0.0 to 2.1.0 ([#1109](https://github.com/freedomofpress/dangerzone/pull/1109))
|
||||
|
||||
### Development changes
|
||||
|
||||
Thanks [@jkarasti](https://github.com/jkarasti) for the contribution.
|
||||
- Automate a large portion of our release tasks with `doit` ([#1016](https://github.com/freedomofpress/dangerzone/issues/1016))
|
||||
|
||||
## [0.8.0](https://github.com/freedomofpress/dangerzone/compare/v0.8.0...0.7.1)
|
||||
|
||||
### Added
|
||||
|
||||
- Point to the installation instructions that the Tails team maintains for Dangerzone ([announcement](https://tails.net/news/dangerzone/index.en.html))
|
||||
- Installation and execution errors are now caught and displayed in the interface ([#193](https://github.com/freedomofpress/dangerzone/issues/193))
|
||||
- Prevent users from using illegal characters in output filename ([#362](https://github.com/freedomofpress/dangerzone/issues/362)). Thanks [@bnewc](https://github.com/bnewc) for the contribution!
|
||||
- Add support for Fedora 41 ([#947](https://github.com/freedomofpress/dangerzone/issues/947))
|
||||
- Add support for Ubuntu Oracular (24.10) ([#954](https://github.com/freedomofpress/dangerzone/pull/954))
|
||||
|
||||
### Fixed
|
||||
|
||||
- Update our macOS entitlements, removing now unneeded privileges ([#638](https://github.com/freedomofpress/dangerzone/issues/638))
|
||||
- Make Dangerzone work on Linux systems with SELinux in enforcing mode ([#880](https://github.com/freedomofpress/dangerzone/issues/880))
|
||||
- Process documents with embedded multimedia files without crashing ([#877](https://github.com/freedomofpress/dangerzone/issues/877))
|
||||
- Search for applications that can read PDF files in a more reliable way on Linux ([#899](https://github.com/freedomofpress/dangerzone/issues/899))
|
||||
- Handle and report some stray conversion errors ([#776](https://github.com/freedomofpress/dangerzone/issues/776)). Thanks [@amnak613](https://github.com/amnak613) for the contribution!
|
||||
- Replace occurrences of the word "Docker" in Podman-related error messages in Linux ([#212](https://github.com/freedomofpress/dangerzone/issues/212))
|
||||
|
||||
### Changed
|
||||
|
||||
- The second phase of the conversion (pixels to PDF) now happens on the host. Instead of first grabbing all of the pixel data from the first container, storing them on disk, and then reconstructing the PDF on a second container, Dangerzone now immediately reconstructs the PDF **on the host**, while the doc to pixels conversion is still running on the first container. The sanitation is no less safe, since the boundaries between the sandbox and the host are still respected ([#625](https://github.com/freedomofpress/dangerzone/issues/625))
|
||||
- PyMuPDF is now vendorized for Debian packages. This is done because the PyMuPDF package from the Debian repos lacks OCR support ([#940](https://github.com/freedomofpress/dangerzone/pull/940))
|
||||
- Always use our own seccomp policy as a default ([#908](https://github.com/freedomofpress/dangerzone/issues/908))
|
||||
- Debian packages are now amd64 only, which removes some warnings in Linux distros with 32-bit repos enabled ([#394](https://github.com/freedomofpress/dangerzone/issues/394))
|
||||
- Allow choosing installation directory on Windows platforms ([#148](https://github.com/freedomofpress/dangerzone/issues/148)). Thanks [@jkarasti](https://github.com/jkarasti) for the contribution!
|
||||
- Bumped H2ORestart LibreOffice extension to version 0.6.6 ([#943](https://github.com/freedomofpress/dangerzone/issues/943))
|
||||
- Platform support: Ubuntu Focal (20.04) is now deprecated, and support will be dropped with the next release ([#965](https://github.com/freedomofpress/dangerzone/issues/965))
|
||||
|
||||
### Removed
|
||||
|
||||
- Platform support: Drop Ubuntu Mantic (23.10), since it's end-of-life ([#977](https://github.com/freedomofpress/dangerzone/pull/977))
|
||||
|
||||
### Development changes
|
||||
|
||||
- Build Debian packages with pybuild ([#773](https://github.com/freedomofpress/dangerzone/issues/773))
|
||||
- Test Dangerzone on Intel macOS machines as well ([#932](https://github.com/freedomofpress/dangerzone/issues/932))
|
||||
- Switch from CircleCI runners to Github actions ([#674](https://github.com/freedomofpress/dangerzone/issues/674))
|
||||
- Sign Windows executables and installer with SHA256 rather than SHA1 ([#931](https://github.com/freedomofpress/dangerzone/pull/931)). Thanks [@jkarasti](https://github.com/jkarasti) for the contribution!
|
||||
|
||||
## [0.7.1](https://github.com/freedomofpress/dangerzone/compare/v0.7.1...v0.7.0)
|
||||
|
||||
### Fixed
|
||||
|
||||
- Fix an `image-id.txt` mismatch happening on Docker Desktop >= 4.30.0 ([#933](https://github.com/freedomofpress/dangerzone/issues/933))
|
||||
|
||||
## [0.7.0](https://github.com/freedomofpress/dangerzone/compare/v0.7.0...v0.6.1)
|
||||
|
||||
### Added
|
||||
|
||||
- Integrate Dangerzone with gVisor, a memory-safe application kernel, thanks to [@EtiennePerot](https://github.com/EtiennePerot) ([#126](https://github.com/freedomofpress/dangerzone/issues/126)).
|
||||
As a result of this integration, we have also improved Dangerzone's security in the following ways:
|
||||
* Prevent attacker from becoming root within the container ([#224](https://github.com/freedomofpress/dangerzone/issues/224))
|
||||
* Use a restricted seccomp profile ([#225](https://github.com/freedomofpress/dangerzone/issues/225))
|
||||
* Make use of user namespaces ([#228](https://github.com/freedomofpress/dangerzone/issues/228))
|
||||
- Files can now be drag-n-dropped to Dangerzone ([issue #409](https://github.com/freedomofpress/dangerzone/issues/409))
|
||||
|
||||
### Fixed
|
||||
|
||||
- Fix a deprecation warning in PySide6, thanks to [@naglis](https://github.com/naglis) ([issue #595](https://github.com/freedomofpress/dangerzone/issues/595))
|
||||
- Make update notifications work in systems with PySide2, thanks to [@naglis](https://github.com/naglis) ([issue #788](https://github.com/freedomofpress/dangerzone/issues/788))
|
||||
- Updated the Dangerzone container image to use Alpine Linux 3.20 ([#812](https://github.com/freedomofpress/dangerzone/pull/812))
|
||||
- Fix wrong file permissions in Fedora packages ([issue #727](https://github.com/freedomofpress/dangerzone/pull/727))
|
||||
- Quote commands in installation instructions, making it compatible with `zsh` based shells. (issue [#805](https://github.com/freedomofpress/dangerzone/issues/805))
|
||||
- Order the list of PDF viewers and return the default application first on Linux, thanks to [@rocodes](https://github.com/rocodes) (issue [#814](https://github.com/freedomofpress/dangerzone/pull/814))
|
||||
|
||||
### Removed
|
||||
|
||||
- Platform support: Drop Fedora 38, since it's end-of-life ([issue #840](https://github.com/freedomofpress/dangerzone/pull/840))
|
||||
|
||||
### Development changes
|
||||
|
||||
- Bumped the minimum python version to 3.9, due to Pyside6 dropping support for python 3.8 ([#780](https://github.com/freedomofpress/dangerzone/pull/780))
|
||||
- Minor amendments to the codebase (in [#811](https://github.com/freedomofpress/dangerzone/pull/811))
|
||||
- Use the original line ending (usually `LF`) for all content except images ([#838](https://github.com/freedomofpress/dangerzone/pull/838))
|
||||
- Explained how to create, sign, and verify source tarballs ([#823](https://github.com/freedomofpress/dangerzone/pull/823))
|
||||
- Added a design doc for the update notifications
|
||||
- Added a design doc for the gVisor integration ([#815](https://github.com/freedomofpress/dangerzone/pull/815))
|
||||
- Removed the python shebang from some files
|
||||
|
||||
## Unreleased
|
||||
|
||||
## Dangerzone 0.6.1
|
||||
|
||||
|
|
|
|||
277
Dockerfile
277
Dockerfile
|
|
@ -1,228 +1,81 @@
|
|||
# NOTE: Updating the packages to their latest versions requires bumping the
|
||||
# Dockerfile args below. For more info about this file, read
|
||||
# docs/developer/reproducibility.md.
|
||||
###########################################
|
||||
# Build PyMuPDF
|
||||
|
||||
ARG DEBIAN_IMAGE_DIGEST=sha256:1209d8fd77def86ceb6663deef7956481cc6c14a25e1e64daec12c0ceffcc19d
|
||||
FROM alpine:latest as pymupdf-build
|
||||
|
||||
FROM docker.io/library/debian@${DEBIAN_IMAGE_DIGEST} AS dangerzone-image
|
||||
ARG REQUIREMENTS_TXT
|
||||
|
||||
ARG GVISOR_ARCHIVE_DATE=20250326
|
||||
ARG DEBIAN_ARCHIVE_DATE=20250331
|
||||
ARG H2ORESTART_CHECKSUM=935e68671bde4ca63a364128077f1c733349bbcc90b7e6973bc7a2306494ec54
|
||||
ARG H2ORESTART_VERSION=v0.7.2
|
||||
# Install PyMuPDF via hash-checked requirements file
|
||||
COPY ${REQUIREMENTS_TXT} /tmp/requirements.txt
|
||||
RUN apk --no-cache add linux-headers g++ linux-headers gcc make python3-dev py3-pip clang-dev
|
||||
RUN pip install --break-system-packages --require-hashes -r /tmp/requirements.txt
|
||||
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
# The following way of installing packages is taken from
|
||||
# https://github.com/reproducible-containers/repro-sources-list.sh/blob/master/Dockerfile.debian-12,
|
||||
# and adapted to allow installing gVisor from each own repo as well.
|
||||
RUN \
|
||||
--mount=type=cache,target=/var/cache/apt,sharing=locked \
|
||||
--mount=type=cache,target=/var/lib/apt,sharing=locked \
|
||||
--mount=type=bind,source=./container_helpers/repro-sources-list.sh,target=/usr/local/bin/repro-sources-list.sh \
|
||||
--mount=type=bind,source=./container_helpers/gvisor.key,target=/tmp/gvisor.key \
|
||||
: "Hacky way to set a date for the Debian snapshot repos" && \
|
||||
touch -d ${DEBIAN_ARCHIVE_DATE}Z /etc/apt/sources.list.d/debian.sources && \
|
||||
touch -d ${DEBIAN_ARCHIVE_DATE}Z /etc/apt/sources.list && \
|
||||
repro-sources-list.sh && \
|
||||
: "Setup APT to install gVisor from its separate APT repo" && \
|
||||
apt-get update && \
|
||||
apt-get upgrade -y && \
|
||||
apt-get install -y --no-install-recommends apt-transport-https ca-certificates gnupg && \
|
||||
gpg -o /usr/share/keyrings/gvisor-archive-keyring.gpg --dearmor /tmp/gvisor.key && \
|
||||
echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/gvisor-archive-keyring.gpg] https://storage.googleapis.com/gvisor/releases ${GVISOR_ARCHIVE_DATE} main" > /etc/apt/sources.list.d/gvisor.list && \
|
||||
: "Install the necessary gVisor and Dangerzone dependencies" && \
|
||||
apt-get update && \
|
||||
apt-get install -y --no-install-recommends \
|
||||
python3 python3-fitz libreoffice-nogui libreoffice-java-common \
|
||||
python3 python3-magic default-jre-headless fonts-noto-cjk fonts-dejavu \
|
||||
runsc unzip wget && \
|
||||
: "Clean up for improving reproducibility (optional)" && \
|
||||
rm -rf /var/cache/fontconfig/ && \
|
||||
rm -rf /etc/ssl/certs/java/cacerts && \
|
||||
rm -rf /var/log/* /var/cache/ldconfig/aux-cache
|
||||
###########################################
|
||||
# Download Tesseract data
|
||||
|
||||
# Download H2ORestart from GitHub using a pinned version and hash. Note that
|
||||
# it's available in Debian repos, but not in Bookworm yet.
|
||||
RUN mkdir /opt/libreoffice_ext && cd /opt/libreoffice_ext \
|
||||
FROM alpine:latest as tessdata-dl
|
||||
ARG TESSDATA_CHECKSUM=d0e3bb6f3b4e75748680524a1d116f2bfb145618f8ceed55b279d15098a530f9
|
||||
|
||||
# Download the trained models from the latest GitHub release of Tesseract, and
|
||||
# store them under /usr/share/tessdata. This is basically what distro packages
|
||||
# do under the hood.
|
||||
#
|
||||
# Because the GitHub release contains more files than just the trained models,
|
||||
# we use `find` to fetch only the '*.traineddata' files in the top directory.
|
||||
#
|
||||
# Before we untar the models, we also check if the checksum is the expected one.
|
||||
RUN mkdir /usr/share/tessdata/ && mkdir tessdata && cd tessdata \
|
||||
&& TESSDATA_VERSION=$(wget -O- -nv https://api.github.com/repos/tesseract-ocr/tessdata_fast/releases/latest \
|
||||
| sed -n 's/^.*"tag_name": "\([0-9.]\+\)".*$/\1/p') \
|
||||
&& wget https://github.com/tesseract-ocr/tessdata_fast/archive/$TESSDATA_VERSION/tessdata_fast-$TESSDATA_VERSION.tar.gz \
|
||||
&& echo "$TESSDATA_CHECKSUM tessdata_fast-$TESSDATA_VERSION.tar.gz" | sha256sum -c \
|
||||
&& tar -xzvf tessdata_fast-$TESSDATA_VERSION.tar.gz -C . \
|
||||
&& find . -name '*.traineddata' -maxdepth 2 -exec cp {} /usr/share/tessdata/ \; \
|
||||
&& cd .. && rm -r tessdata
|
||||
|
||||
|
||||
###########################################
|
||||
# Download H2ORestart
|
||||
FROM alpine:latest as h2orestart-dl
|
||||
ARG H2ORESTART_CHECKSUM=5db816a1e57b510456633f55e693cb5ef3675ef8b35df4f31c90ab9d4c66071a
|
||||
RUN mkdir /libreoffice_ext && cd libreoffice_ext \
|
||||
&& H2ORESTART_FILENAME=h2orestart.oxt \
|
||||
&& H2ORESTART_VERSION="v0.5.7" \
|
||||
&& wget https://github.com/ebandal/H2Orestart/releases/download/$H2ORESTART_VERSION/$H2ORESTART_FILENAME \
|
||||
&& echo "$H2ORESTART_CHECKSUM $H2ORESTART_FILENAME" | sha256sum -c \
|
||||
&& install -dm777 "/usr/lib/libreoffice/share/extensions/" \
|
||||
&& rm /root/.wget-hsts
|
||||
&& install -dm777 "/usr/lib/libreoffice/share/extensions/"
|
||||
|
||||
# Create an unprivileged user both for gVisor and for running Dangerzone.
|
||||
# XXX: Make the shadow field "date of last password change" a constant
|
||||
# number.
|
||||
RUN addgroup --gid 1000 dangerzone
|
||||
RUN adduser --uid 1000 --ingroup dangerzone --shell /bin/true \
|
||||
--disabled-password --home /home/dangerzone dangerzone \
|
||||
&& chage -d 99999 dangerzone \
|
||||
&& rm /etc/shadow-
|
||||
|
||||
# Copy Dangerzone's conversion logic under /opt/dangerzone, and allow Python to
|
||||
# import it.
|
||||
###########################################
|
||||
# Dangerzone image
|
||||
|
||||
FROM alpine:latest
|
||||
|
||||
# Install dependencies
|
||||
RUN apk --no-cache -U upgrade && \
|
||||
apk --no-cache add \
|
||||
libreoffice \
|
||||
openjdk8 \
|
||||
python3 \
|
||||
py3-magic \
|
||||
font-noto-cjk
|
||||
|
||||
COPY --from=pymupdf-build /usr/lib/python3.11/site-packages/fitz/ /usr/lib/python3.11/site-packages/fitz
|
||||
COPY --from=tessdata-dl /usr/share/tessdata/ /usr/share/tessdata
|
||||
COPY --from=h2orestart-dl /libreoffice_ext/ /libreoffice_ext
|
||||
|
||||
RUN install -dm777 "/usr/lib/libreoffice/share/extensions/"
|
||||
|
||||
ENV PYTHONPATH=/opt/dangerzone
|
||||
|
||||
RUN mkdir -p /opt/dangerzone/dangerzone
|
||||
RUN touch /opt/dangerzone/dangerzone/__init__.py
|
||||
COPY conversion /opt/dangerzone/dangerzone/conversion
|
||||
|
||||
# Copy only the Python code, and not any produced .pyc files.
|
||||
COPY conversion/*.py /opt/dangerzone/dangerzone/conversion/
|
||||
|
||||
# Create a directory that will be used by gVisor as the place where it will
|
||||
# store the state of its containers.
|
||||
RUN mkdir /home/dangerzone/.containers
|
||||
|
||||
###############################################################################
|
||||
#
|
||||
# REUSING CONTAINER IMAGES:
|
||||
# Anatomy of a hack
|
||||
# ========================
|
||||
#
|
||||
# The rest of the Dockerfile aims to do one thing: allow the final container
|
||||
# image to actually contain two container images; one for the outer container
|
||||
# (spawned by Podman/Docker Desktop), and one for the inner container (spawned
|
||||
# by gVisor).
|
||||
#
|
||||
# This has already been done in the past, and we explain why and how in the
|
||||
# design document for gVisor integration (should be in
|
||||
# `docs/developer/gvisor.md`). In this iteration, we want to also
|
||||
# achieve the following:
|
||||
#
|
||||
# 1. Have a small final image, by sharing some system paths between the inner
|
||||
# and outer container image using symlinks.
|
||||
# 2. Allow our security scanning tool to see the contents of the inner
|
||||
# container image.
|
||||
# 3. Make the outer container image operational, in the sense that you can use
|
||||
# `apt` commands and perform a conversion with Dangerzone, outside the
|
||||
# gVisor sandbox. This is helpful for debugging purposes.
|
||||
#
|
||||
# Below we'll explain how our design choices are informed by the above
|
||||
# sub-goals.
|
||||
#
|
||||
# First, to achieve a small container image, we basically need to copy `/etc`,
|
||||
# `/usr` and `/opt` from the original Dangerzone image to the **inner**
|
||||
# container image (under `/home/dangerzone/dangerzone-image/rootfs/`)
|
||||
#
|
||||
# That's all we need. The rest of the files play no role, and we can actually
|
||||
# mask them in gVisor's OCI config.
|
||||
#
|
||||
# Second, in order to let our security scanner find the installed packages,
|
||||
# we need to copy the following dirs to the root of the **outer** container
|
||||
# image:
|
||||
# * `/etc`, so that the security scanner can detect the image type and its
|
||||
# sources
|
||||
# * `/var`, so that the security scanner can have access to the APT database.
|
||||
#
|
||||
# IMPORTANT: We don't symlink the `/etc` of the **outer** container image to
|
||||
# the **inner** one, in order to avoid leaking files like
|
||||
# `/etc/{hostname,hosts,resolv.conf}` that Podman/Docker mounts when running
|
||||
# the **outer** container image.
|
||||
#
|
||||
# Third, in order to have an operational Debian image, we are _mostly_ covered
|
||||
# by the dirs we have copied. There's a _rare_ case where during debugging, we
|
||||
# may want to install a system package that has components in `/etc` and
|
||||
# `/var`, which will not be available in the **inner** container image. In that
|
||||
# case, the developer can do the necessary symlinks in the live container.
|
||||
#
|
||||
# FILESYSTEM HIERARCHY
|
||||
# ====================
|
||||
#
|
||||
# The above plan leads to the following filesystem hierarchy:
|
||||
#
|
||||
# Outer container image:
|
||||
#
|
||||
# # ls -l /
|
||||
# lrwxrwxrwx 1 root root 7 Jan 27 10:46 bin -> usr/bin
|
||||
# -rwxr-xr-x 1 root root 7764 Jan 24 08:14 entrypoint.py
|
||||
# drwxr-xr-x 1 root root 4096 Jan 27 10:47 etc
|
||||
# drwxr-xr-x 1 root root 4096 Jan 27 10:46 home
|
||||
# lrwxrwxrwx 1 root root 7 Jan 27 10:46 lib -> usr/lib
|
||||
# lrwxrwxrwx 1 root root 9 Jan 27 10:46 lib64 -> usr/lib64
|
||||
# drwxr-xr-x 2 root root 4096 Jan 27 10:46 root
|
||||
# drwxr-xr-x 1 root root 4096 Jan 27 10:47 run
|
||||
# lrwxrwxrwx 1 root root 8 Jan 27 10:46 sbin -> usr/sbin
|
||||
# drwxrwxrwx 2 root root 4096 Jan 27 10:46 tmp
|
||||
# lrwxrwxrwx 1 root root 44 Jan 27 10:46 usr -> /home/dangerzone/dangerzone-image/rootfs/usr
|
||||
# drwxr-xr-x 11 root root 4096 Jan 27 10:47 var
|
||||
#
|
||||
# Inner container image:
|
||||
#
|
||||
# # ls -l /home/dangerzone/dangerzone-image/rootfs/
|
||||
# total 12
|
||||
# lrwxrwxrwx 1 root root 7 Jan 27 10:47 bin -> usr/bin
|
||||
# drwxr-xr-x 43 root root 4096 Jan 27 10:46 etc
|
||||
# lrwxrwxrwx 1 root root 7 Jan 27 10:47 lib -> usr/lib
|
||||
# lrwxrwxrwx 1 root root 9 Jan 27 10:47 lib64 -> usr/lib64
|
||||
# drwxr-xr-x 4 root root 4096 Jan 27 10:47 opt
|
||||
# drwxr-xr-x 12 root root 4096 Jan 27 10:47 usr
|
||||
#
|
||||
# SYMLINKING /USR
|
||||
# ===============
|
||||
#
|
||||
# It's surprisingly difficult (maybe even borderline impossible), to symlink
|
||||
# `/usr` to a different path during image build. The problem is that /usr
|
||||
# is very sensitive, and you can't manipulate it in a live system. That is, I
|
||||
# haven't found a way to do the following, or something equivalent:
|
||||
#
|
||||
# rm -r /usr && ln -s /home/dangerzone/dangerzone-image/rootfs/usr/ /usr
|
||||
#
|
||||
# The `ln` binary, even if you specify it by its full path, cannot run
|
||||
# (probably because `ld-linux.so` can't be found). For this reason, we have
|
||||
# to create the symlinks beforehand, in a previous build stage. Then, in an
|
||||
# empty container image (scratch images), we can copy these symlinks and the
|
||||
# /usr, and stitch everything together.
|
||||
###############################################################################
|
||||
|
||||
# Create the filesystem hierarchy that will be used to symlink /usr.
|
||||
|
||||
RUN mkdir -p \
|
||||
/new_root \
|
||||
/new_root/root \
|
||||
/new_root/run \
|
||||
/new_root/tmp \
|
||||
/new_root/home/dangerzone/dangerzone-image/rootfs
|
||||
|
||||
# Copy the /etc and /var directories under the new root directory. Also,
|
||||
# copy /etc/, /opt, and /usr to the Dangerzone image rootfs.
|
||||
#
|
||||
# NOTE: We also have to remove the resolv.conf file, in order to not leak any
|
||||
# DNS servers added there during image build time.
|
||||
RUN cp -r /etc /var /new_root/ \
|
||||
&& rm /new_root/etc/resolv.conf
|
||||
RUN cp -r /etc /opt /usr /new_root/home/dangerzone/dangerzone-image/rootfs \
|
||||
&& rm /new_root/home/dangerzone/dangerzone-image/rootfs/etc/resolv.conf
|
||||
|
||||
RUN ln -s /home/dangerzone/dangerzone-image/rootfs/usr /new_root/usr
|
||||
RUN ln -s usr/bin /new_root/bin
|
||||
RUN ln -s usr/lib /new_root/lib
|
||||
RUN ln -s usr/lib64 /new_root/lib64
|
||||
RUN ln -s usr/sbin /new_root/sbin
|
||||
RUN ln -s usr/bin /new_root/home/dangerzone/dangerzone-image/rootfs/bin
|
||||
RUN ln -s usr/lib /new_root/home/dangerzone/dangerzone-image/rootfs/lib
|
||||
RUN ln -s usr/lib64 /new_root/home/dangerzone/dangerzone-image/rootfs/lib64
|
||||
|
||||
# Fix permissions in /home/dangerzone, so that our entrypoint script can make
|
||||
# changes in the following folders.
|
||||
RUN chown dangerzone:dangerzone \
|
||||
/new_root/home/dangerzone \
|
||||
/new_root/home/dangerzone/dangerzone-image/
|
||||
# Fix permissions in /tmp, so that it can be used by unprivileged users.
|
||||
RUN chmod 777 /new_root/tmp
|
||||
|
||||
COPY container_helpers/entrypoint.py /new_root
|
||||
# HACK: For reasons that we are not sure yet, we need to explicitly specify the
|
||||
# modification time of this file.
|
||||
RUN touch -d ${DEBIAN_ARCHIVE_DATE}Z /new_root/entrypoint.py
|
||||
|
||||
## Final image
|
||||
|
||||
FROM scratch
|
||||
|
||||
# Copy the filesystem hierarchy that we created in the previous stage, so that
|
||||
# /usr can be a symlink.
|
||||
COPY --from=dangerzone-image /new_root/ /
|
||||
|
||||
# Switch to the dangerzone user for the rest of the script.
|
||||
# Add the unprivileged user
|
||||
RUN adduser -s /bin/sh -D dangerzone
|
||||
USER dangerzone
|
||||
|
||||
ENTRYPOINT ["/entrypoint.py"]
|
||||
# /safezone is a directory through which Pixels to PDF receives files
|
||||
VOLUME /safezone
|
||||
|
|
|
|||
|
|
@ -1,16 +0,0 @@
|
|||
# Should be the INDEX DIGEST from an image tagged `bookworm-<DATE>-slim`:
|
||||
# https://hub.docker.com/_/debian/tags?name=bookworm-
|
||||
#
|
||||
# Tag for this digest: bookworm-20250317-slim
|
||||
DEBIAN_IMAGE_DIGEST=sha256:1209d8fd77def86ceb6663deef7956481cc6c14a25e1e64daec12c0ceffcc19d
|
||||
# Can be bumped to today's date
|
||||
DEBIAN_ARCHIVE_DATE=20250331
|
||||
# Can be bumped to the latest date in https://github.com/google/gvisor/tags
|
||||
GVISOR_ARCHIVE_DATE=20250326
|
||||
# Can be bumped to the latest version and checksum from https://github.com/ebandal/H2Orestart/releases
|
||||
H2ORESTART_CHECKSUM=935e68671bde4ca63a364128077f1c733349bbcc90b7e6973bc7a2306494ec54
|
||||
H2ORESTART_VERSION=v0.7.2
|
||||
|
||||
# Buildkit image (taken from freedomofpress/repro-build)
|
||||
BUILDKIT_IMAGE="docker.io/moby/buildkit:v19.0@sha256:14aa1b4dd92ea0a4cd03a54d0c6079046ea98cd0c0ae6176bdd7036ba370cbbe"
|
||||
BUILDKIT_IMAGE_ROOTLESS="docker.io/moby/buildkit:v0.19.0-rootless@sha256:e901cffdad753892a7c3afb8b9972549fca02c73888cf340c91ed801fdd96d71"
|
||||
228
Dockerfile.in
228
Dockerfile.in
|
|
@ -1,228 +0,0 @@
|
|||
# NOTE: Updating the packages to their latest versions requires bumping the
|
||||
# Dockerfile args below. For more info about this file, read
|
||||
# docs/developer/reproducibility.md.
|
||||
|
||||
ARG DEBIAN_IMAGE_DIGEST={{DEBIAN_IMAGE_DIGEST}}
|
||||
|
||||
FROM docker.io/library/debian@${DEBIAN_IMAGE_DIGEST} AS dangerzone-image
|
||||
|
||||
ARG GVISOR_ARCHIVE_DATE={{GVISOR_ARCHIVE_DATE}}
|
||||
ARG DEBIAN_ARCHIVE_DATE={{DEBIAN_ARCHIVE_DATE}}
|
||||
ARG H2ORESTART_CHECKSUM={{H2ORESTART_CHECKSUM}}
|
||||
ARG H2ORESTART_VERSION={{H2ORESTART_VERSION}}
|
||||
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
# The following way of installing packages is taken from
|
||||
# https://github.com/reproducible-containers/repro-sources-list.sh/blob/master/Dockerfile.debian-12,
|
||||
# and adapted to allow installing gVisor from each own repo as well.
|
||||
RUN \
|
||||
--mount=type=cache,target=/var/cache/apt,sharing=locked \
|
||||
--mount=type=cache,target=/var/lib/apt,sharing=locked \
|
||||
--mount=type=bind,source=./container_helpers/repro-sources-list.sh,target=/usr/local/bin/repro-sources-list.sh \
|
||||
--mount=type=bind,source=./container_helpers/gvisor.key,target=/tmp/gvisor.key \
|
||||
: "Hacky way to set a date for the Debian snapshot repos" && \
|
||||
touch -d ${DEBIAN_ARCHIVE_DATE}Z /etc/apt/sources.list.d/debian.sources && \
|
||||
touch -d ${DEBIAN_ARCHIVE_DATE}Z /etc/apt/sources.list && \
|
||||
repro-sources-list.sh && \
|
||||
: "Setup APT to install gVisor from its separate APT repo" && \
|
||||
apt-get update && \
|
||||
apt-get upgrade -y && \
|
||||
apt-get install -y --no-install-recommends apt-transport-https ca-certificates gnupg && \
|
||||
gpg -o /usr/share/keyrings/gvisor-archive-keyring.gpg --dearmor /tmp/gvisor.key && \
|
||||
echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/gvisor-archive-keyring.gpg] https://storage.googleapis.com/gvisor/releases ${GVISOR_ARCHIVE_DATE} main" > /etc/apt/sources.list.d/gvisor.list && \
|
||||
: "Install the necessary gVisor and Dangerzone dependencies" && \
|
||||
apt-get update && \
|
||||
apt-get install -y --no-install-recommends \
|
||||
python3 python3-fitz libreoffice-nogui libreoffice-java-common \
|
||||
python3 python3-magic default-jre-headless fonts-noto-cjk fonts-dejavu \
|
||||
runsc unzip wget && \
|
||||
: "Clean up for improving reproducibility (optional)" && \
|
||||
rm -rf /var/cache/fontconfig/ && \
|
||||
rm -rf /etc/ssl/certs/java/cacerts && \
|
||||
rm -rf /var/log/* /var/cache/ldconfig/aux-cache
|
||||
|
||||
# Download H2ORestart from GitHub using a pinned version and hash. Note that
|
||||
# it's available in Debian repos, but not in Bookworm yet.
|
||||
RUN mkdir /opt/libreoffice_ext && cd /opt/libreoffice_ext \
|
||||
&& H2ORESTART_FILENAME=h2orestart.oxt \
|
||||
&& wget https://github.com/ebandal/H2Orestart/releases/download/$H2ORESTART_VERSION/$H2ORESTART_FILENAME \
|
||||
&& echo "$H2ORESTART_CHECKSUM $H2ORESTART_FILENAME" | sha256sum -c \
|
||||
&& install -dm777 "/usr/lib/libreoffice/share/extensions/" \
|
||||
&& rm /root/.wget-hsts
|
||||
|
||||
# Create an unprivileged user both for gVisor and for running Dangerzone.
|
||||
# XXX: Make the shadow field "date of last password change" a constant
|
||||
# number.
|
||||
RUN addgroup --gid 1000 dangerzone
|
||||
RUN adduser --uid 1000 --ingroup dangerzone --shell /bin/true \
|
||||
--disabled-password --home /home/dangerzone dangerzone \
|
||||
&& chage -d 99999 dangerzone \
|
||||
&& rm /etc/shadow-
|
||||
|
||||
# Copy Dangerzone's conversion logic under /opt/dangerzone, and allow Python to
|
||||
# import it.
|
||||
RUN mkdir -p /opt/dangerzone/dangerzone
|
||||
RUN touch /opt/dangerzone/dangerzone/__init__.py
|
||||
|
||||
# Copy only the Python code, and not any produced .pyc files.
|
||||
COPY conversion/*.py /opt/dangerzone/dangerzone/conversion/
|
||||
|
||||
# Create a directory that will be used by gVisor as the place where it will
|
||||
# store the state of its containers.
|
||||
RUN mkdir /home/dangerzone/.containers
|
||||
|
||||
###############################################################################
|
||||
#
|
||||
# REUSING CONTAINER IMAGES:
|
||||
# Anatomy of a hack
|
||||
# ========================
|
||||
#
|
||||
# The rest of the Dockerfile aims to do one thing: allow the final container
|
||||
# image to actually contain two container images; one for the outer container
|
||||
# (spawned by Podman/Docker Desktop), and one for the inner container (spawned
|
||||
# by gVisor).
|
||||
#
|
||||
# This has already been done in the past, and we explain why and how in the
|
||||
# design document for gVisor integration (should be in
|
||||
# `docs/developer/gvisor.md`). In this iteration, we want to also
|
||||
# achieve the following:
|
||||
#
|
||||
# 1. Have a small final image, by sharing some system paths between the inner
|
||||
# and outer container image using symlinks.
|
||||
# 2. Allow our security scanning tool to see the contents of the inner
|
||||
# container image.
|
||||
# 3. Make the outer container image operational, in the sense that you can use
|
||||
# `apt` commands and perform a conversion with Dangerzone, outside the
|
||||
# gVisor sandbox. This is helpful for debugging purposes.
|
||||
#
|
||||
# Below we'll explain how our design choices are informed by the above
|
||||
# sub-goals.
|
||||
#
|
||||
# First, to achieve a small container image, we basically need to copy `/etc`,
|
||||
# `/usr` and `/opt` from the original Dangerzone image to the **inner**
|
||||
# container image (under `/home/dangerzone/dangerzone-image/rootfs/`)
|
||||
#
|
||||
# That's all we need. The rest of the files play no role, and we can actually
|
||||
# mask them in gVisor's OCI config.
|
||||
#
|
||||
# Second, in order to let our security scanner find the installed packages,
|
||||
# we need to copy the following dirs to the root of the **outer** container
|
||||
# image:
|
||||
# * `/etc`, so that the security scanner can detect the image type and its
|
||||
# sources
|
||||
# * `/var`, so that the security scanner can have access to the APT database.
|
||||
#
|
||||
# IMPORTANT: We don't symlink the `/etc` of the **outer** container image to
|
||||
# the **inner** one, in order to avoid leaking files like
|
||||
# `/etc/{hostname,hosts,resolv.conf}` that Podman/Docker mounts when running
|
||||
# the **outer** container image.
|
||||
#
|
||||
# Third, in order to have an operational Debian image, we are _mostly_ covered
|
||||
# by the dirs we have copied. There's a _rare_ case where during debugging, we
|
||||
# may want to install a system package that has components in `/etc` and
|
||||
# `/var`, which will not be available in the **inner** container image. In that
|
||||
# case, the developer can do the necessary symlinks in the live container.
|
||||
#
|
||||
# FILESYSTEM HIERARCHY
|
||||
# ====================
|
||||
#
|
||||
# The above plan leads to the following filesystem hierarchy:
|
||||
#
|
||||
# Outer container image:
|
||||
#
|
||||
# # ls -l /
|
||||
# lrwxrwxrwx 1 root root 7 Jan 27 10:46 bin -> usr/bin
|
||||
# -rwxr-xr-x 1 root root 7764 Jan 24 08:14 entrypoint.py
|
||||
# drwxr-xr-x 1 root root 4096 Jan 27 10:47 etc
|
||||
# drwxr-xr-x 1 root root 4096 Jan 27 10:46 home
|
||||
# lrwxrwxrwx 1 root root 7 Jan 27 10:46 lib -> usr/lib
|
||||
# lrwxrwxrwx 1 root root 9 Jan 27 10:46 lib64 -> usr/lib64
|
||||
# drwxr-xr-x 2 root root 4096 Jan 27 10:46 root
|
||||
# drwxr-xr-x 1 root root 4096 Jan 27 10:47 run
|
||||
# lrwxrwxrwx 1 root root 8 Jan 27 10:46 sbin -> usr/sbin
|
||||
# drwxrwxrwx 2 root root 4096 Jan 27 10:46 tmp
|
||||
# lrwxrwxrwx 1 root root 44 Jan 27 10:46 usr -> /home/dangerzone/dangerzone-image/rootfs/usr
|
||||
# drwxr-xr-x 11 root root 4096 Jan 27 10:47 var
|
||||
#
|
||||
# Inner container image:
|
||||
#
|
||||
# # ls -l /home/dangerzone/dangerzone-image/rootfs/
|
||||
# total 12
|
||||
# lrwxrwxrwx 1 root root 7 Jan 27 10:47 bin -> usr/bin
|
||||
# drwxr-xr-x 43 root root 4096 Jan 27 10:46 etc
|
||||
# lrwxrwxrwx 1 root root 7 Jan 27 10:47 lib -> usr/lib
|
||||
# lrwxrwxrwx 1 root root 9 Jan 27 10:47 lib64 -> usr/lib64
|
||||
# drwxr-xr-x 4 root root 4096 Jan 27 10:47 opt
|
||||
# drwxr-xr-x 12 root root 4096 Jan 27 10:47 usr
|
||||
#
|
||||
# SYMLINKING /USR
|
||||
# ===============
|
||||
#
|
||||
# It's surprisingly difficult (maybe even borderline impossible), to symlink
|
||||
# `/usr` to a different path during image build. The problem is that /usr
|
||||
# is very sensitive, and you can't manipulate it in a live system. That is, I
|
||||
# haven't found a way to do the following, or something equivalent:
|
||||
#
|
||||
# rm -r /usr && ln -s /home/dangerzone/dangerzone-image/rootfs/usr/ /usr
|
||||
#
|
||||
# The `ln` binary, even if you specify it by its full path, cannot run
|
||||
# (probably because `ld-linux.so` can't be found). For this reason, we have
|
||||
# to create the symlinks beforehand, in a previous build stage. Then, in an
|
||||
# empty container image (scratch images), we can copy these symlinks and the
|
||||
# /usr, and stitch everything together.
|
||||
###############################################################################
|
||||
|
||||
# Create the filesystem hierarchy that will be used to symlink /usr.
|
||||
|
||||
RUN mkdir -p \
|
||||
/new_root \
|
||||
/new_root/root \
|
||||
/new_root/run \
|
||||
/new_root/tmp \
|
||||
/new_root/home/dangerzone/dangerzone-image/rootfs
|
||||
|
||||
# Copy the /etc and /var directories under the new root directory. Also,
|
||||
# copy /etc/, /opt, and /usr to the Dangerzone image rootfs.
|
||||
#
|
||||
# NOTE: We also have to remove the resolv.conf file, in order to not leak any
|
||||
# DNS servers added there during image build time.
|
||||
RUN cp -r /etc /var /new_root/ \
|
||||
&& rm /new_root/etc/resolv.conf
|
||||
RUN cp -r /etc /opt /usr /new_root/home/dangerzone/dangerzone-image/rootfs \
|
||||
&& rm /new_root/home/dangerzone/dangerzone-image/rootfs/etc/resolv.conf
|
||||
|
||||
RUN ln -s /home/dangerzone/dangerzone-image/rootfs/usr /new_root/usr
|
||||
RUN ln -s usr/bin /new_root/bin
|
||||
RUN ln -s usr/lib /new_root/lib
|
||||
RUN ln -s usr/lib64 /new_root/lib64
|
||||
RUN ln -s usr/sbin /new_root/sbin
|
||||
RUN ln -s usr/bin /new_root/home/dangerzone/dangerzone-image/rootfs/bin
|
||||
RUN ln -s usr/lib /new_root/home/dangerzone/dangerzone-image/rootfs/lib
|
||||
RUN ln -s usr/lib64 /new_root/home/dangerzone/dangerzone-image/rootfs/lib64
|
||||
|
||||
# Fix permissions in /home/dangerzone, so that our entrypoint script can make
|
||||
# changes in the following folders.
|
||||
RUN chown dangerzone:dangerzone \
|
||||
/new_root/home/dangerzone \
|
||||
/new_root/home/dangerzone/dangerzone-image/
|
||||
# Fix permissions in /tmp, so that it can be used by unprivileged users.
|
||||
RUN chmod 777 /new_root/tmp
|
||||
|
||||
COPY container_helpers/entrypoint.py /new_root
|
||||
# HACK: For reasons that we are not sure yet, we need to explicitly specify the
|
||||
# modification time of this file.
|
||||
RUN touch -d ${DEBIAN_ARCHIVE_DATE}Z /new_root/entrypoint.py
|
||||
|
||||
## Final image
|
||||
|
||||
FROM scratch
|
||||
|
||||
# Copy the filesystem hierarchy that we created in the previous stage, so that
|
||||
# /usr can be a symlink.
|
||||
COPY --from=dangerzone-image /new_root/ /
|
||||
|
||||
# Switch to the dangerzone user for the rest of the script.
|
||||
USER dangerzone
|
||||
|
||||
ENTRYPOINT ["/entrypoint.py"]
|
||||
193
INSTALL.md
193
INSTALL.md
|
|
@ -1,91 +1,24 @@
|
|||
## Operating System support
|
||||
|
||||
Dangerzone can run on various Operating Systems (OS), and has automated tests
|
||||
for most of them.
|
||||
This section explains which OS we support, how long we support each version, and
|
||||
how do we test Dangerzone against these.
|
||||
|
||||
You can find general support information in this table, and more details in the
|
||||
following sections.
|
||||
|
||||
(Unless specified, the architecture of the OS is AMD64)
|
||||
|
||||
| Distribution | Supported releases | Automated tests | Manual QA |
|
||||
| ------------ | ------------------------- | ---------------------- | --------- |
|
||||
| Windows | 2 last releases | 🗹 (`windows-latest`) ◎ | 🗹 |
|
||||
| macOS intel | 3 last releases | 🗹 (`macos-13`) ◎ | 🗹 |
|
||||
| macOS silicon | 3 last releases | 🗹 (`macos-latest`) ◎ | 🗹 |
|
||||
| Ubuntu | Follow upstream support ✰ | 🗹 | 🗹 |
|
||||
| Debian | Current stable, Oldstable and LTS releases | 🗹 | 🗹 |
|
||||
| Fedora | Follow upstream support | 🗹 | 🗹 |
|
||||
| Qubes OS | [Beta support](https://github.com/freedomofpress/dangerzone/issues/413) ✢ | 🗷 | Latest Fedora template |
|
||||
| Tails | Only the last release | 🗷 | Last release only |
|
||||
|
||||
Notes:
|
||||
|
||||
✰ Support for Ubuntu Focal [was dropped](https://github.com/freedomofpress/dangerzone/issues/1018)
|
||||
|
||||
✢ Qubes OS support assumes the use of a Fedora template. The supported releases follow our general support for Fedora.
|
||||
|
||||
◎ More information about where that points [in the runner-images repository](https://github.com/actions/runner-images/tree/main)
|
||||
|
||||
## MacOS
|
||||
|
||||
- Download [Dangerzone 0.9.0 for Mac (Apple Silicon CPU)](https://github.com/freedomofpress/dangerzone/releases/download/v0.9.0/Dangerzone-0.9.0-arm64.dmg)
|
||||
- Download [Dangerzone 0.9.0 for Mac (Intel CPU)](https://github.com/freedomofpress/dangerzone/releases/download/v0.9.0/Dangerzone-0.9.0-i686.dmg)
|
||||
|
||||
> [!TIP]
|
||||
> We support the releases of macOS that are still within Apple's servicing timeline. Apple usually provides security updates for the latest 3 releases, but this isn’t consistently applied and security fixes aren’t guaranteed for the non-latest releases. We are also dependent on [Docker Desktop windows support](https://docs.docker.com/desktop/setup/install/mac-install/)
|
||||
|
||||
You can also install Dangerzone for Mac using [Homebrew](https://brew.sh/): `brew install --cask dangerzone`
|
||||
|
||||
> **Note**: you will also need to install [Docker Desktop](https://www.docker.com/products/docker-desktop/).
|
||||
> This program needs to run alongside Dangerzone at all times, since it is what allows Dangerzone to
|
||||
> create the secure environment.
|
||||
See instructions in [README.md](README.md#macos).
|
||||
|
||||
## Windows
|
||||
|
||||
- Download [Dangerzone 0.9.0 for Windows](https://github.com/freedomofpress/dangerzone/releases/download/v0.9.0/Dangerzone-0.9.0.msi)
|
||||
|
||||
> **Note**: you will also need to install [Docker Desktop](https://www.docker.com/products/docker-desktop/).
|
||||
> This program needs to run alongside Dangerzone at all times, since it is what allows Dangerzone to
|
||||
> create the secure environment.
|
||||
|
||||
> [!TIP]
|
||||
> We generally support Windows releases that are still within [Microsoft’s servicing timeline](https://support.microsoft.com/en-us/help/13853/windows-lifecycle-fact-sheet).
|
||||
>
|
||||
> Docker sets the bottom line:
|
||||
>
|
||||
> > Docker only supports Docker Desktop on Windows for those versions of Windows that are still within [Microsoft’s servicing timeline](https://support.microsoft.com/en-us/help/13853/windows-lifecycle-fact-sheet). Docker Desktop is not supported on server versions of Windows, such as Windows Server 2019 or Windows Server 2022.
|
||||
|
||||
See instructions in [README.md](README.md#windows).
|
||||
|
||||
## Linux
|
||||
|
||||
On Linux, Dangerzone uses [Podman](https://podman.io/) instead of Docker Desktop for creating
|
||||
an isolated environment. It will be installed automatically when installing Dangerzone.
|
||||
|
||||
> [!TIP]
|
||||
> We support Ubuntu, Debian, and Fedora releases that are still within
|
||||
> their respective servicing timelines, with a few twists:
|
||||
>
|
||||
> - Ubuntu: We follow upstream support with an extra cutoff date. No support for
|
||||
> versions prior to the second oldest LTS release.
|
||||
> - Fedora: We follow upstream support
|
||||
> - Debian: current stable, oldstable and LTS releases.
|
||||
|
||||
Dangerzone is available for:
|
||||
|
||||
- Ubuntu 25.04 (plucky)
|
||||
- Ubuntu 24.10 (oracular)
|
||||
- Ubuntu 24.04 (noble)
|
||||
- Ubuntu 23.10 (mantic)
|
||||
- Ubuntu 22.04 (jammy)
|
||||
- Ubuntu 20.04 (focal)
|
||||
- Debian 13 (trixie)
|
||||
- Debian 12 (bookworm)
|
||||
- Debian 11 (bullseye)
|
||||
- Fedora 42
|
||||
- Fedora 41
|
||||
- Fedora 40
|
||||
- Tails
|
||||
- Fedora 39
|
||||
- Fedora 38
|
||||
- Qubes OS (beta support)
|
||||
|
||||
### Ubuntu, Debian
|
||||
|
|
@ -94,7 +27,41 @@ Dangerzone is available for:
|
|||
<tr>
|
||||
<td>
|
||||
<details>
|
||||
<summary><i>:information_source: Backport notice for Ubuntu 22.04 (Jammy) users regarding the <code>conmon</code> package</i></summary>
|
||||
<summary><i>:memo: Expand this section if you are on Ubuntu 20.04 (Focal).</i></summary>
|
||||
</br>
|
||||
|
||||
Dangerzone requires [Podman](https://podman.io/), which is not available
|
||||
through the official Ubuntu Focal repos. To proceed with the Dangerzone
|
||||
installation, you need to add an extra OpenSUSE repo that provides Podman to
|
||||
Ubuntu Focal users. You can follow the instructions below, which have been
|
||||
copied from the [official Podman blog](https://podman.io/new/2021/06/16/new.html):
|
||||
|
||||
```bash
|
||||
sudo apt-get update && sudo apt-get install curl wget gnupg2 -y
|
||||
. /etc/os-release
|
||||
sudo sh -c "echo 'deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_${VERSION_ID}/ /' \
|
||||
> /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list"
|
||||
wget -nv https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable/xUbuntu_${VERSION_ID}/Release.key -O- \
|
||||
| sudo apt-key add -
|
||||
sudo apt update
|
||||
```
|
||||
|
||||
Also, you need to install the `python-all` package, due to an `stdeb` bug that
|
||||
existed before v0.9.1:
|
||||
|
||||
```
|
||||
sudo apt-get install python-all -y
|
||||
```
|
||||
</details>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
<table>
|
||||
<tr>
|
||||
<td>
|
||||
<details>
|
||||
<summary><i>:information_source: Backport notice for Ubuntu 24.04 (Noble) users regarding the <code>conmon</code> package</i></summary>
|
||||
</br>
|
||||
|
||||
The `conmon` version that Podman uses and Ubuntu Jammy ships, has a bug
|
||||
|
|
@ -110,38 +77,25 @@ Dangerzone is available for:
|
|||
</tr>
|
||||
</table>
|
||||
|
||||
First, retrieve the PGP keys. The instructions differ depending on the specific
|
||||
distribution you are using:
|
||||
Add our repository following these instructions:
|
||||
|
||||
For Debian Trixie and Ubuntu Plucky (25.04), follow these instructions to
|
||||
download the PGP keys:
|
||||
|
||||
```bash
|
||||
sudo apt-get update && sudo apt-get install sq ca-certificates -y
|
||||
sq network keyserver \
|
||||
--server hkps://keys.openpgp.org \
|
||||
search "DE28 AB24 1FA4 8260 FAC9 B8BA A7C9 B385 2260 4281" \
|
||||
--output - | sq packet dearmor fpfdz.gpg
|
||||
sudo mkdir -p /etc/apt/keyrings/
|
||||
sudo mv fpfdz.gpg /etc/apt/keyrings/fpf-apt-tools-archive-keyring.gpg
|
||||
```
|
||||
|
||||
On other Debian-derivatives:
|
||||
Download the GPG key for the repo:
|
||||
|
||||
```sh
|
||||
sudo apt-get update && sudo apt-get install gnupg2 ca-certificates -y
|
||||
sudo mkdir -p /etc/apt/keyrings/
|
||||
sudo gpg --keyserver hkps://keys.openpgp.org \
|
||||
--no-default-keyring --keyring /etc/apt/keyrings/fpf-apt-tools-archive-keyring.gpg \
|
||||
gpg --keyserver hkps://keys.openpgp.org \
|
||||
--no-default-keyring --keyring ./fpf-apt-tools-archive-keyring.gpg \
|
||||
--recv-keys "DE28 AB24 1FA4 8260 FAC9 B8BA A7C9 B385 2260 4281"
|
||||
sudo mkdir -p /etc/apt/keyrings/
|
||||
sudo mv fpf-apt-tools-archive-keyring.gpg /etc/apt/keyrings
|
||||
```
|
||||
|
||||
Then, on all distributions, add the URL of the repo in your APT sources:
|
||||
Add the URL of the repo in your APT sources:
|
||||
|
||||
```sh
|
||||
. /etc/os-release
|
||||
echo "deb [signed-by=/etc/apt/keyrings/fpf-apt-tools-archive-keyring.gpg] \
|
||||
https://packages.freedom.press/apt-tools-prod ${VERSION_CODENAME?} main" \
|
||||
echo deb [signed-by=/etc/apt/keyrings/fpf-apt-tools-archive-keyring.gpg] \
|
||||
https://packages.freedom.press/apt-tools-prod ${VERSION_CODENAME?} main \
|
||||
| sudo tee /etc/apt/sources.list.d/fpf-apt-tools.list
|
||||
```
|
||||
|
||||
|
|
@ -176,11 +130,28 @@ sudo apt install -y dangerzone
|
|||
|
||||
### Fedora
|
||||
|
||||
<table>
|
||||
<tr>
|
||||
<td>
|
||||
<details>
|
||||
<summary><i>:information_source: Backport notice for Fedora users regarding the <code>python3-pyside6</code> package</i></summary>
|
||||
</br>
|
||||
|
||||
Fedora 39+ onwards does not provide official Python bindings for Qt. For
|
||||
this reason, we provide our own `python3-pyside6` package (see
|
||||
[build instructions](https://github.com/freedomofpress/maint-dangerzone-pyside6))
|
||||
from our YUM repo. For a deeper dive on this subject, you may read
|
||||
[this issue](https://github.com/freedomofpress/dangerzone/issues/211#issuecomment-1827777122).
|
||||
</details>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
Type the following commands in a terminal:
|
||||
|
||||
```
|
||||
sudo dnf install 'dnf-command(config-manager)'
|
||||
sudo dnf-3 config-manager --add-repo=https://packages.freedom.press/yum-tools-prod/dangerzone/dangerzone.repo
|
||||
sudo dnf config-manager --add-repo=https://packages.freedom.press/yum-tools-prod/dangerzone/dangerzone.repo
|
||||
sudo dnf install dangerzone
|
||||
```
|
||||
|
||||
|
|
@ -231,8 +202,8 @@ After confirming that it matches, type `y` (for yes) and the installation should
|
|||
|
||||
> [!IMPORTANT]
|
||||
> This section will install Dangerzone in your **default template**
|
||||
> (`fedora-41` as of writing this). If you want to install it in a different
|
||||
> one, make sure to replace `fedora-41` with the template of your choice.
|
||||
> (`fedora-38` as of writing this). If you want to install it in a different
|
||||
> one, make sure to replace `fedora-38` with the template of your choice.
|
||||
|
||||
The following steps must be completed once. Make sure you run them in the
|
||||
specified qubes.
|
||||
|
|
@ -249,7 +220,7 @@ Create a **disposable**, offline app qube (`dz-dvm`), based on your default
|
|||
template. This will be the qube where the documents will be sanitized:
|
||||
|
||||
```
|
||||
qvm-create --class AppVM --label red --template fedora-41 \
|
||||
qvm-create --class AppVM --label red --template fedora-38 \
|
||||
--prop netvm="" --prop template_for_dispvms=True \
|
||||
--prop default_dispvm='' dz-dvm
|
||||
```
|
||||
|
|
@ -262,12 +233,12 @@ document, with the following contents:
|
|||
dz.Convert * @anyvm @dispvm:dz-dvm allow
|
||||
```
|
||||
|
||||
#### In the `fedora-41` template
|
||||
#### In the `fedora-38` template
|
||||
|
||||
Install Dangerzone:
|
||||
|
||||
```
|
||||
sudo dnf-3 config-manager --add-repo=https://packages.freedom.press/yum-tools-prod/dangerzone/dangerzone.repo
|
||||
sudo dnf config-manager --add-repo=https://packages.freedom.press/yum-tools-prod/dangerzone/dangerzone.repo
|
||||
sudo dnf install dangerzone-qubes
|
||||
```
|
||||
|
||||
|
|
@ -283,12 +254,6 @@ column to "Selected".
|
|||
You can now launch Dangerzone from the list of applications for your qube, and
|
||||
pass it a file to sanitize.
|
||||
|
||||
## Tails
|
||||
|
||||
Dangerzone is not yet available by default in Tails, but we have collaborated
|
||||
with the Tails team to offer manual
|
||||
[installation instructions](https://tails.net/doc/persistent_storage/additional_software/dangerzone/index.en.html)
|
||||
for Tails users.
|
||||
|
||||
## Build from source
|
||||
|
||||
|
|
@ -323,7 +288,7 @@ Our [GitHub Releases page](https://github.com/freedomofpress/dangerzone/releases
|
|||
hosts the following files:
|
||||
* Windows installer (`Dangerzone-<version>.msi`)
|
||||
* macOS archives (`Dangerzone-<version>-<arch>.dmg`)
|
||||
* Container images (`container-<version>-<arch>.tar`)
|
||||
* Container image (`container.tar.gz`)
|
||||
* Source package (`dangerzone-<version>.tar.gz`)
|
||||
|
||||
All these files are accompanied by signatures (as `.asc` files). We'll explain
|
||||
|
|
@ -348,16 +313,10 @@ gpg --verify Dangerzone-0.6.1-arm64.dmg.asc Dangerzone-0.6.1-arm64.dmg
|
|||
gpg --verify Dangerzone-0.6.1-i686.dmg.asc Dangerzone-0.6.1-i686.dmg
|
||||
```
|
||||
|
||||
For the container images:
|
||||
For the container image:
|
||||
|
||||
```
|
||||
gpg --verify container-0.6.1-i686.tar.asc container-0.6.1-i686.tar
|
||||
```
|
||||
|
||||
For the source package:
|
||||
|
||||
```
|
||||
gpg --verify dangerzone-0.6.1.tar.gz.asc dangerzone-0.6.1.tar.gz
|
||||
gpg --verify container.tar.gz.asc container.tar.gz
|
||||
```
|
||||
|
||||
We also hash all the above files with SHA-256, and provide a list of these
|
||||
|
|
|
|||
65
Makefile
65
Makefile
|
|
@ -1,6 +1,23 @@
|
|||
LARGE_TEST_REPO_DIR:=tests/test_docs_large
|
||||
GIT_DESC=$$(git describe)
|
||||
JUNIT_FLAGS := --capture=sys -o junit_logging=all
|
||||
|
||||
.PHONY: lint-black
|
||||
lint-black: ## check python source code formatting issues, with black
|
||||
black --check --diff ./
|
||||
|
||||
.PHONY: lint-black-apply
|
||||
lint-black-apply: ## apply black's source code formatting suggestions
|
||||
black ./
|
||||
|
||||
.PHONY: lint-isort
|
||||
lint-isort: ## check imports are organized, with isort
|
||||
isort --check --diff ./
|
||||
|
||||
.PHONY: lint-isort-apply
|
||||
lint-isort-apply: ## apply isort's imports organization suggestions
|
||||
isort ./
|
||||
|
||||
MYPY_ARGS := --ignore-missing-imports \
|
||||
--disallow-incomplete-defs \
|
||||
--disallow-untyped-defs \
|
||||
|
|
@ -9,24 +26,26 @@ MYPY_ARGS := --ignore-missing-imports \
|
|||
--warn-unused-ignores \
|
||||
--exclude $(LARGE_TEST_REPO_DIR)/*.py
|
||||
|
||||
.PHONY: lint
|
||||
lint: ## Check the code for linting, formatting, and typing issues with ruff and mypy
|
||||
ruff check
|
||||
ruff format --check
|
||||
mypy-host:
|
||||
mypy $(MYPY_ARGS) dangerzone
|
||||
|
||||
mypy-tests:
|
||||
mypy $(MYPY_ARGS) tests
|
||||
|
||||
.PHONY: fix
|
||||
fix: ## apply all the suggestions from ruff
|
||||
ruff check --fix
|
||||
ruff format
|
||||
mypy: mypy-host mypy-tests ## check type hints with mypy
|
||||
|
||||
.PHONY: lint
|
||||
lint: lint-black lint-isort mypy ## check the code with various linters
|
||||
|
||||
.PHONY: lint-apply
|
||||
lint-apply: lint-black-apply lint-isort-apply ## apply all the linter's suggestions
|
||||
|
||||
.PHONY: test
|
||||
test: ## Run the tests
|
||||
test:
|
||||
# Make each GUI test run as a separate process, to avoid segfaults due to
|
||||
# shared state.
|
||||
# See more in https://github.com/freedomofpress/dangerzone/issues/493
|
||||
pytest --co -q tests/gui | grep -e '^tests/' | xargs -n 1 pytest -v
|
||||
pytest --co -q tests/gui | grep -v ' collected' | xargs -n 1 pytest -v
|
||||
pytest -v --cov --ignore dev_scripts --ignore tests/gui --ignore tests/test_large_set.py
|
||||
|
||||
|
||||
|
|
@ -47,32 +66,6 @@ test-large: test-large-init ## Run large test set
|
|||
python -m pytest --tb=no tests/test_large_set.py::TestLargeSet -v $(JUNIT_FLAGS) --junitxml=$(TEST_LARGE_RESULTS)
|
||||
python $(TEST_LARGE_RESULTS)/report.py $(TEST_LARGE_RESULTS)
|
||||
|
||||
Dockerfile: Dockerfile.env Dockerfile.in ## Regenerate the Dockerfile from its template
|
||||
poetry run jinja2 Dockerfile.in Dockerfile.env > Dockerfile
|
||||
|
||||
.PHONY: poetry-install
|
||||
poetry-install: ## Install project dependencies
|
||||
poetry install
|
||||
|
||||
.PHONY: build-clean
|
||||
build-clean:
|
||||
poetry run doit clean
|
||||
|
||||
.PHONY: build-macos-intel
|
||||
build-macos-intel: build-clean poetry-install ## Build macOS intel package (.dmg)
|
||||
poetry run doit -n 8
|
||||
|
||||
.PHONY: build-macos-arm
|
||||
build-macos-arm: build-clean poetry-install ## Build macOS Apple Silicon package (.dmg)
|
||||
poetry run doit -n 8 macos_build_dmg
|
||||
|
||||
.PHONY: build-linux
|
||||
build-linux: build-clean poetry-install ## Build linux packages (.rpm and .deb)
|
||||
poetry run doit -n 8 fedora_rpm debian_deb
|
||||
|
||||
.PHONY: regenerate-reference-pdfs
|
||||
regenerate-reference-pdfs: ## Regenerate the reference PDFs
|
||||
pytest tests/test_cli.py -k regenerate --generate-reference-pdfs
|
||||
# Makefile self-help borrowed from the securedrop-client project
|
||||
# Explaination of the below shell command should it ever break.
|
||||
# 1. Set the field separator to ": ##" and any make targets that might appear between : and ##
|
||||
|
|
|
|||
197
QA.md
197
QA.md
|
|
@ -1,197 +0,0 @@
|
|||
## QA
|
||||
|
||||
To ensure that new releases do not introduce regressions, and support existing
|
||||
and newer platforms, we have to test that the produced packages work as expected.
|
||||
|
||||
Check the following:
|
||||
|
||||
- [ ] Make sure that the tip of the `main` branch passes the CI tests.
|
||||
- [ ] Make sure that the Apple account has a valid application password and has
|
||||
agreed to the latest Apple terms (see [macOS release](#macos-release)
|
||||
section).
|
||||
|
||||
Because it is repetitive, we wrote a script to help with the QA.
|
||||
It can run the tasks for you, pausing when it needs manual intervention.
|
||||
|
||||
You can run it with a command like:
|
||||
|
||||
```bash
|
||||
poetry run ./dev_scripts/qa.py {distro}-{version}
|
||||
```
|
||||
|
||||
### The checklist
|
||||
|
||||
- [ ] Create a test build in Windows and make sure it works:
|
||||
- [ ] Check if the suggested Python version is still supported.
|
||||
- [ ] Create a new development environment with Poetry.
|
||||
- [ ] Build the container image and ensure the development environment uses
|
||||
the new image.
|
||||
- [ ] Download the OCR language data using `./install/common/download-tessdata.py`
|
||||
- [ ] Run the Dangerzone tests.
|
||||
- [ ] Build and run the Dangerzone .exe
|
||||
- [ ] Test some QA scenarios (see [Scenarios](#Scenarios) below).
|
||||
- [ ] Create a test build in macOS (Intel CPU) and make sure it works:
|
||||
- [ ] Check if the suggested Python version is still supported.
|
||||
- [ ] Create a new development environment with Poetry.
|
||||
- [ ] Build the container image and ensure the development environment uses
|
||||
the new image.
|
||||
- [ ] Download the OCR language data using `./install/common/download-tessdata.py`
|
||||
- [ ] Run the Dangerzone tests.
|
||||
- [ ] Create and run an app bundle.
|
||||
- [ ] Test some QA scenarios (see [Scenarios](#Scenarios) below).
|
||||
- [ ] Create a test build in macOS (M1/2 CPU) and make sure it works:
|
||||
- [ ] Check if the suggested Python version is still supported.
|
||||
- [ ] Create a new development environment with Poetry.
|
||||
- [ ] Build the container image and ensure the development environment uses
|
||||
the new image.
|
||||
- [ ] Download the OCR language data using `./install/common/download-tessdata.py`
|
||||
- [ ] Run the Dangerzone tests.
|
||||
- [ ] Create and run an app bundle.
|
||||
- [ ] Test some QA scenarios (see [Scenarios](#Scenarios) below).
|
||||
- [ ] Create a test build in the most recent Ubuntu LTS platform (Ubuntu 24.04
|
||||
as of writing this) and make sure it works:
|
||||
- [ ] Create a new development environment with Poetry.
|
||||
- [ ] Build the container image and ensure the development environment uses
|
||||
the new image.
|
||||
- [ ] Download the OCR language data using `./install/common/download-tessdata.py`
|
||||
- [ ] Run the Dangerzone tests.
|
||||
- [ ] Create a .deb package and install it system-wide.
|
||||
- [ ] Test some QA scenarios (see [Scenarios](#Scenarios) below).
|
||||
- [ ] Create a test build in the most recent Fedora platform (Fedora 41 as of
|
||||
writing this) and make sure it works:
|
||||
- [ ] Create a new development environment with Poetry.
|
||||
- [ ] Build the container image and ensure the development environment uses
|
||||
the new image.
|
||||
- [ ] Download the OCR language data using `./install/common/download-tessdata.py`
|
||||
- [ ] Run the Dangerzone tests.
|
||||
- [ ] Create an .rpm package and install it system-wide.
|
||||
- [ ] Test some QA scenarios (see [Scenarios](#Scenarios) below).
|
||||
- [ ] Create a test build in the most recent Qubes Fedora template (Fedora 40 as
|
||||
of writing this) and make sure it works:
|
||||
- [ ] Create a new development environment with Poetry.
|
||||
- [ ] Run the Dangerzone tests.
|
||||
- [ ] Create a Qubes .rpm package and install it system-wide.
|
||||
- [ ] Ensure that the Dangerzone application appears in the "Applications"
|
||||
tab.
|
||||
- [ ] Test some QA scenarios (see [Scenarios](#Scenarios) below) and make sure
|
||||
they spawn disposable qubes.
|
||||
|
||||
### Scenarios
|
||||
|
||||
#### 1. Dangerzone correctly identifies that Docker/Podman is not installed
|
||||
|
||||
_(Only for MacOS / Windows)_
|
||||
|
||||
Temporarily hide the Docker/Podman binaries, e.g., rename the `docker` /
|
||||
`podman` binaries to something else. Then run Dangerzone. Dangerzone should
|
||||
prompt the user to install Docker/Podman.
|
||||
|
||||
#### 2. Dangerzone correctly identifies that Docker is not running
|
||||
|
||||
_(Only for MacOS / Windows)_
|
||||
|
||||
Stop the Docker Desktop application. Then run Dangerzone. Dangerzone should
|
||||
prompt the user to start Docker Desktop.
|
||||
|
||||
|
||||
#### 3. Updating Dangerzone handles external state correctly.
|
||||
|
||||
_(Applies to Windows/MacOS)_
|
||||
|
||||
Install the previous version of Dangerzone, downloaded from the website.
|
||||
|
||||
Open the Dangerzone application and enable some non-default settings.
|
||||
**If there are new settings, make sure to change those as well**.
|
||||
|
||||
Close the Dangerzone application and get the container image for that
|
||||
version. For example:
|
||||
|
||||
```
|
||||
$ docker images dangerzone.rocks/dangerzone
|
||||
REPOSITORY TAG IMAGE ID CREATED SIZE
|
||||
dangerzone.rocks/dangerzone <tag> <image ID> <date> <size>
|
||||
```
|
||||
|
||||
Then run the version under QA and ensure that the settings remain changed.
|
||||
|
||||
Afterwards check that new docker image was installed by running the same command
|
||||
and seeing the following differences:
|
||||
|
||||
```
|
||||
$ docker images dangerzone.rocks/dangerzone
|
||||
REPOSITORY TAG IMAGE ID CREATED SIZE
|
||||
dangerzone.rocks/dangerzone <other tag> <different ID> <newer date> <different size>
|
||||
```
|
||||
|
||||
#### 4. Dangerzone successfully installs the container image
|
||||
|
||||
_(Only for Linux)_
|
||||
|
||||
Remove the Dangerzone container image from Docker/Podman. Then run Dangerzone.
|
||||
Dangerzone should install the container image successfully.
|
||||
|
||||
#### 5. Dangerzone retains the settings of previous runs
|
||||
|
||||
Run Dangerzone and make some changes in the settings (e.g., change the OCR
|
||||
language, toggle whether to open the document after conversion, etc.). Restart
|
||||
Dangerzone. Dangerzone should show the settings that the user chose.
|
||||
|
||||
#### 6. Dangerzone reports failed conversions
|
||||
|
||||
Run Dangerzone and convert the `tests/test_docs/sample_bad_pdf.pdf` document.
|
||||
Dangerzone should fail gracefully, by reporting that the operation failed, and
|
||||
showing the following error message:
|
||||
|
||||
> The document format is not supported
|
||||
|
||||
#### 7. Dangerzone succeeds in converting multiple documents
|
||||
|
||||
Run Dangerzone against a list of documents, and tick all options. Ensure that:
|
||||
* Conversions take place sequentially.
|
||||
* Attempting to close the window while converting asks the user if they want to
|
||||
abort the conversions.
|
||||
* Conversions are completed successfully.
|
||||
* Conversions show individual progress in real-time (double-check for Qubes).
|
||||
* _(Only for Linux)_ The resulting files open with the PDF viewer of our choice.
|
||||
* OCR seems to have detected characters in the PDF files.
|
||||
* The resulting files have been saved with the proper suffix, in the proper
|
||||
location.
|
||||
* The original files have been saved in the `unsafe/` directory.
|
||||
|
||||
#### 8. Dangerzone is able to handle drag-n-drop
|
||||
|
||||
Run Dangerzone against a set of documents that you drag-n-drop. Files should be
|
||||
added and conversion should run without issue.
|
||||
|
||||
> [!TIP]
|
||||
> On our end-user container environments for Linux, we can start a file manager
|
||||
> with `thunar &`.
|
||||
|
||||
#### 9. Dangerzone CLI succeeds in converting multiple documents
|
||||
|
||||
_(Only for Windows and Linux)_
|
||||
|
||||
Run Dangerzone CLI against a list of documents. Ensure that conversions happen
|
||||
sequentially, are completed successfully, and we see their progress.
|
||||
|
||||
#### 10. Dangerzone can open a document for conversion via right-click -> "Open With"
|
||||
|
||||
_(Only for Windows, MacOS and Qubes)_
|
||||
|
||||
Go to a directory with office documents, right-click on one, and click on "Open
|
||||
With". We should be able to open the file with Dangerzone, and then convert it.
|
||||
|
||||
#### 11. Dangerzone shows helpful errors for setup issues on Qubes
|
||||
|
||||
_(Only for Qubes)_
|
||||
|
||||
Check what errors does Dangerzone throw in the following scenarios. The errors
|
||||
should point the user to the Qubes notifications in the top-right corner:
|
||||
|
||||
1. The `dz-dvm` template does not exist. We can trigger this scenario by
|
||||
temporarily renaming this template.
|
||||
2. The Dangerzone RPC policy does not exist. We can trigger this scenario by
|
||||
temporarily renaming the `dz.Convert` policy.
|
||||
3. The `dz-dvm` disposable Qube cannot start due to insufficient resources. We
|
||||
can trigger this scenario by temporarily increasing the minimum required RAM
|
||||
of the `dz-dvm` template to more than the available amount.
|
||||
58
README.md
58
README.md
|
|
@ -6,28 +6,37 @@ Take potentially dangerous PDFs, office documents, or images and convert them to
|
|||
|  | 
|
||||
|--|--|
|
||||
|
||||
Dangerzone works like this: You give it a document that you don't know if you can trust (for example, an email attachment). Inside of a sandbox, Dangerzone converts the document to a PDF (if it isn't already one), and then converts the PDF into raw pixel data: a huge list of RGB color values for each page. Then, outside of the sandbox, Dangerzone takes this pixel data and converts it back into a PDF.
|
||||
Dangerzone works like this: You give it a document that you don't know if you can trust (for example, an email attachment). Inside of a sandbox, Dangerzone converts the document to a PDF (if it isn't already one), and then converts the PDF into raw pixel data: a huge list of RGB color values for each page. Then, in a separate sandbox, Dangerzone takes this pixel data and converts it back into a PDF.
|
||||
|
||||
_Read more about Dangerzone in the [official site](https://dangerzone.rocks/about/)._
|
||||
_Read more about Dangerzone in the [official site](https://dangerzone.rocks/about.html)._
|
||||
|
||||
## Getting started
|
||||
|
||||
Follow the instructions for each platform:
|
||||
### MacOS
|
||||
- Download [Dangerzone 0.6.0 for Mac (Apple Silicon CPU)](https://github.com/freedomofpress/dangerzone/releases/download/v0.6.0/Dangerzone-0.6.0-arm64.dmg)
|
||||
- Download [Dangerzone 0.6.0 for Mac (Intel CPU)](https://github.com/freedomofpress/dangerzone/releases/download/v0.6.0/Dangerzone-0.6.0-i686.dmg)
|
||||
|
||||
* [macOS](https://github.com/freedomofpress/dangerzone/blob/v0.9.0/INSTALL.md#macos)
|
||||
* [Windows](https://github.com/freedomofpress/dangerzone/blob/v0.9.0//INSTALL.md#windows)
|
||||
* [Ubuntu Linux](https://github.com/freedomofpress/dangerzone/blob/v0.9.0/INSTALL.md#ubuntu-debian)
|
||||
* [Debian Linux](https://github.com/freedomofpress/dangerzone/blob/v0.9.0/INSTALL.md#ubuntu-debian)
|
||||
* [Fedora Linux](https://github.com/freedomofpress/dangerzone/blob/v0.9.0/INSTALL.md#fedora)
|
||||
* [Qubes OS (beta)](https://github.com/freedomofpress/dangerzone/blob/v0.9.0/INSTALL.md#qubes-os)
|
||||
* [Tails](https://github.com/freedomofpress/dangerzone/blob/v0.9.0/INSTALL.md#tails)
|
||||
You can also install Dangerzone for Mac using [Homebrew](https://brew.sh/): `brew install --cask dangerzone`
|
||||
|
||||
You can read more about our operating system support [here](https://github.com/freedomofpress/dangerzone/blob/v0.9.0/INSTALL.md#operating-system-support).
|
||||
> **Note**: you will also need to install [Docker Desktop](https://www.docker.com/products/docker-desktop/).
|
||||
> This program needs to run alongside Dangerzone at all times, since it is what allows Dangerzone to
|
||||
> create the secure environment.
|
||||
|
||||
### Windows
|
||||
|
||||
- Download [Dangerzone 0.6.0 for Windows](https://github.com/freedomofpress/dangerzone/releases/download/v0.6.0/Dangerzone-0.6.0-1.msi)
|
||||
|
||||
> **Note**: you will also need to install [Docker Desktop](https://www.docker.com/products/docker-desktop/).
|
||||
> This program needs to run alongside Dangerzone at all times, since it is what allows Dangerzone to
|
||||
> create the secure environment.
|
||||
|
||||
### Linux
|
||||
|
||||
See [installing Dangerzone](INSTALL.md#linux) for adding the Linux repositories to your system.
|
||||
|
||||
## Some features
|
||||
|
||||
- Sandboxes don't have network access, so if a malicious document can compromise one, it can't phone home
|
||||
- Sandboxes use [gVisor](https://gvisor.dev/), an application kernel written in Go, that implements a substantial portion of the Linux system call interface.
|
||||
- Dangerzone can optionally OCR the safe PDFs it creates, so it will have a text layer again
|
||||
- Dangerzone compresses the safe PDF to reduce file size
|
||||
- After converting, Dangerzone lets you open the safe PDF in the PDF viewer of your choice, which allows you to open PDFs and office docs in Dangerzone by default so you never accidentally open a dangerous document
|
||||
|
|
@ -64,17 +73,8 @@ Licensed under the AGPLv3: [https://opensource.org/licenses/agpl-3.0](https://op
|
|||
Copyright (c) 2022-2024 Freedom of the Press Foundation and Dangerzone contributors
|
||||
Copyright (c) 2020-2021 First Look Media
|
||||
|
||||
## See also
|
||||
|
||||
* [GIJN Toolbox: Cutting-Edge — and Free — Online Investigative Tools You Can Try Right Now](https://gijn.org/stories/cutting-edge-free-online-investigative-tools/)
|
||||
* [When security matters: working with Qubes OS at the Guardian](https://www.theguardian.com/info/2024/apr/04/when-security-matters-working-with-qubes-os-at-the-guardian)
|
||||
|
||||
## FAQ
|
||||
|
||||
### Has Dangerzone received a security audit?
|
||||
|
||||
Yes, Dangerzone received its [first security audit](https://freedom.press/news/dangerzone-receives-favorable-audit/) by [Include Security](https://includesecurity.com/) in December 2023. The audit was generally favorable, as it didn't identify any high-risk findings, except for 3 low-risk and 7 informational findings.
|
||||
|
||||
### "I'm experiencing an issue while using Dangerzone."
|
||||
|
||||
Dangerzone gets updates to improve its features _and_ to fix problems. So, updating may be the simplest path to resolving the issue which brought you here. Here is how to update:
|
||||
|
|
@ -83,6 +83,18 @@ Dangerzone gets updates to improve its features _and_ to fix problems. So, updat
|
|||
2. Now find the latest available version of Dangerzone: go to the [download page](https://dangerzone.rocks/#downloads). Look for the version number displayed. The number will be using the same format as in Step 1.
|
||||
3. Is the version on the Dangerzone download page higher than the version of your installed app? Go ahead and update.
|
||||
|
||||
### Can I use Podman Desktop?
|
||||
### "I get `invalid json returned from container` on MacOS Big Sur or newer (MacOS 11.x.x or higher)"
|
||||
|
||||
Yes! We've introduced [experimental support for Podman Desktop](https://github.com/freedomofpress/dangerzone/blob/main/docs/podman-desktop.md) on Windows and macOS.
|
||||
Are you using the latest version of Dangerzone? See the FAQ for: "I'm experiencing an issue while using Dangerzone."
|
||||
|
||||
You _may_ be attempting to convert a file in a directory to which Docker Desktop does not have access. Dangerzone for Mac requires Docker Desktop for conversion. Docker Desktop, in turn, requires permission from MacOS to access the directory in which your target file is located.
|
||||
|
||||
To grant this permission:
|
||||
|
||||
1. On MacOS 13, choose Apple menu > System Settings. On lower versions, choose System Preferences.
|
||||
2. Tap into Privacy & Security in the sidebar. (You may need to scroll down.)
|
||||
3. In the Privacy section, tap into Files & Folders. (Again, you may need to scroll down.)
|
||||
4. Scroll to the entry for Docker. Tap the > to expand the entry.
|
||||
5. Enable the toggle beside the directory where your file is present. For example, if the file to be converted is in the Downloads folder, enable the toggle beside Downloads.
|
||||
|
||||
(Full Disk Access permission has a similar effect, but it's enough to give Docker access to _only_ the directory containing the intended file(s) to be converted. Full Disk is unnecessary. As of 2023.04.28, granting one of these permissions continues to be required for successful conversion. Apologies for the extra steps. Dangerzone depends on Docker, and the fix for this issue needs to come from upstream. Read more on [#371](https://github.com/freedomofpress/dangerzone/issues/371#issuecomment-1516863056).)
|
||||
|
|
|
|||
447
RELEASE.md
447
RELEASE.md
|
|
@ -1,28 +1,27 @@
|
|||
# Release instructions
|
||||
|
||||
This section documents how we currently release Dangerzone for the different distributions we support.
|
||||
This section documents the release process. Unless you're a dangerzone developer making a release, you'll probably never need to follow it.
|
||||
|
||||
## Pre-release
|
||||
|
||||
Here is a list of tasks that should be done before issuing the release:
|
||||
Before making a release, all of these should be complete:
|
||||
|
||||
- [ ] Create a new issue named **QA and Release for version \<VERSION\>**, to track the general progress.
|
||||
You can generate its content with the the `poetry run ./dev_scripts/generate-release-tasks.py` command.
|
||||
- [ ] [Add new Linux platforms and remove obsolete ones](https://github.com/freedomofpress/dangerzone/blob/main/RELEASE.md#add-new-linux-platforms-and-remove-obsolete-ones)
|
||||
- [ ] Copy the entirety of these instructions onto a new issue and call it **QA and Release version \<VERSION\>**
|
||||
- [ ] [Add new Linux platforms and remove obsolete ones](#add-new-platforms-and-remove-obsolete-ones)
|
||||
- [ ] Bump the Python dependencies using `poetry lock`
|
||||
- [ ] Check for new [WiX releases](https://github.com/wixtoolset/wix/releases) and update it if needed
|
||||
- [ ] [Check for official PySide6 versions](#check-for-official-pyside6-versions)
|
||||
- [ ] Update `version` in `pyproject.toml`
|
||||
- [ ] Update `share/version.txt`
|
||||
- [ ] Update the "Version" field in `install/linux/dangerzone.spec`
|
||||
- [ ] Bump the Debian version by adding a new changelog entry in `debian/changelog`
|
||||
- [ ] [Bump the minimum Docker Desktop versions](https://github.com/freedomofpress/dangerzone/blob/main/RELEASE.md#bump-the-minimum-docker-desktop-version) in `isolation_provider/container.py`
|
||||
- [ ] Bump the dates and versions in the `Dockerfile`
|
||||
- [ ] Update the download links in our `INSTALL.md` page to point to the new version (the download links will be populated after the release)
|
||||
- [ ] Update screenshot in `README.md`, if necessary
|
||||
- [ ] CHANGELOG.md should be updated to include a list of all major changes since the last release
|
||||
- [ ] A draft release should be created. Copy the release notes text from the template at [`docs/templates/release-notes`](https://github.com/freedomofpress/dangerzone/tree/main/docs/templates/)
|
||||
- [ ] Send the release notes to editorial for review
|
||||
- [ ] Do the QA tasks
|
||||
- [ ] Create a PGP-signed git tag for the version, e.g., for dangerzone `v0.1.0`:
|
||||
|
||||
```
|
||||
git tag -s v0.1.0
|
||||
git push origin v0.1.0
|
||||
```
|
||||
**Note**: release candidates are suffixed by `-rcX`.
|
||||
|
||||
## Add new Linux platforms and remove obsolete ones
|
||||
|
||||
|
|
@ -31,7 +30,7 @@ as a special case of Fedora, release-wise). For each of these platforms, we need
|
|||
to check if a new version has been added, or if an existing one is now EOL
|
||||
(https://endoflife.date/ is handy for this purpose).
|
||||
|
||||
In case of a new version (beta, RC, or official release):
|
||||
In case of a new version:
|
||||
|
||||
1. Add it in our CI workflows, to test if that version works.
|
||||
* See `.circleci/config.yml` and `.github/workflows/ci.yml`, as well as
|
||||
|
|
@ -45,17 +44,21 @@ In case of a new version (beta, RC, or official release):
|
|||
`BUILD.md` files where necessary.
|
||||
4. Send a PR with the above changes.
|
||||
|
||||
In case of the removal of a version:
|
||||
In case of an EOL version:
|
||||
|
||||
1. Remove any mention to this version from our repo.
|
||||
* Consult the previous paragraph, but also `grep` your way around.
|
||||
2. Add a notice in our `CHANGELOG.md` about the version removal.
|
||||
|
||||
## Bump the minimum Docker Desktop version
|
||||
## Check for official PySide6 versions
|
||||
|
||||
We embed the minimum docker desktop versions inside Dangerzone, as an incentive for our macOS and Windows users to upgrade to the latests version.
|
||||
PySide6 6.7.0 is available from the Fedora Rawhide repo, and we expect that a
|
||||
similar version will be pushed soon to the rest of the stable releases. Prior to
|
||||
a release, we should check if this has happened already. Once this happens, we
|
||||
should update our CI tests accordingly, and remove this notice.
|
||||
|
||||
You can find the latest version at the time of the release by looking at [their release notes](https://docs.docker.com/desktop/release-notes/)
|
||||
For more info, read:
|
||||
https://github.com/freedomofpress/maint-dangerzone-pyside6/issues/5
|
||||
|
||||
## Large Document Testing
|
||||
|
||||
|
|
@ -65,18 +68,178 @@ Follow the instructions in `docs/developer/TESTING.md` to run the tests.
|
|||
|
||||
These tests will identify any regressions or progression in terms of document coverage.
|
||||
|
||||
## QA
|
||||
|
||||
To ensure that new releases do not introduce regressions, and support existing
|
||||
and newer platforms, we have to do the following:
|
||||
|
||||
- [ ] Make sure that the tip of the `main` branch passes the CI tests.
|
||||
- [ ] Make sure that the Apple account has a valid application password and has
|
||||
agreed to the latest Apple terms (see [macOS release](#macos-release)
|
||||
section).
|
||||
- [ ] Create a test build in Windows and make sure it works:
|
||||
- [ ] Check if the suggested Python version is still supported.
|
||||
- [ ] Create a new development environment with Poetry.
|
||||
- [ ] Build the container image and ensure the development environment uses
|
||||
the new image.
|
||||
- [ ] Run the Dangerzone tests.
|
||||
- [ ] Build and run the Dangerzone .exe
|
||||
- [ ] Test some QA scenarios (see [Scenarios](#Scenarios) below).
|
||||
- [ ] Create a test build in macOS (Intel CPU) and make sure it works:
|
||||
- [ ] Check if the suggested Python version is still supported.
|
||||
- [ ] Create a new development environment with Poetry.
|
||||
- [ ] Build the container image and ensure the development environment uses
|
||||
the new image.
|
||||
- [ ] Run the Dangerzone tests.
|
||||
- [ ] Create and run an app bundle.
|
||||
- [ ] Test some QA scenarios (see [Scenarios](#Scenarios) below).
|
||||
- [ ] Create a test build in macOS (M1/2 CPU) and make sure it works:
|
||||
- [ ] Check if the suggested Python version is still supported.
|
||||
- [ ] Create a new development environment with Poetry.
|
||||
- [ ] Build the container image and ensure the development environment uses
|
||||
the new image.
|
||||
- [ ] Run the Dangerzone tests.
|
||||
- [ ] Create and run an app bundle.
|
||||
- [ ] Test some QA scenarios (see [Scenarios](#Scenarios) below).
|
||||
- [ ] Create a test build in the most recent Ubuntu LTS platform (Ubuntu 24.04
|
||||
as of writing this) and make sure it works:
|
||||
- [ ] Create a new development environment with Poetry.
|
||||
- [ ] Build the container image and ensure the development environment uses
|
||||
the new image.
|
||||
- [ ] Run the Dangerzone tests.
|
||||
- [ ] Create a .deb package and install it system-wide.
|
||||
- [ ] Test some QA scenarios (see [Scenarios](#Scenarios) below).
|
||||
- [ ] Create a test build in the most recent Fedora platform (Fedora 40 as of
|
||||
writing this) and make sure it works:
|
||||
- [ ] Create a new development environment with Poetry.
|
||||
- [ ] Build the container image and ensure the development environment uses
|
||||
the new image.
|
||||
- [ ] Run the Dangerzone tests.
|
||||
- [ ] Create an .rpm package and install it system-wide.
|
||||
- [ ] Test some QA scenarios (see [Scenarios](#Scenarios) below).
|
||||
- [ ] Create a test build in the most recent Qubes Fedora template (Fedora 39 as
|
||||
of writing this) and make sure it works:
|
||||
- [ ] Create a new development environment with Poetry.
|
||||
- [ ] Run the Dangerzone tests.
|
||||
- [ ] Create a Qubes .rpm package and install it system-wide.
|
||||
- [ ] Ensure that the Dangerzone application appears in the "Applications"
|
||||
tab.
|
||||
- [ ] Test some QA scenarios (see [Scenarios](#Scenarios) below) and make sure
|
||||
they spawn disposable qubes.
|
||||
|
||||
### Scenarios
|
||||
|
||||
#### 1. Dangerzone correctly identifies that Docker/Podman is not installed
|
||||
|
||||
_(Only for MacOS / Windows)_
|
||||
|
||||
Temporarily hide the Docker/Podman binaries, e.g., rename the `docker` /
|
||||
`podman` binaries to something else. Then run Dangerzone. Dangerzone should
|
||||
prompt the user to install Docker/Podman.
|
||||
|
||||
#### 2. Dangerzone correctly identifies that Docker is not running
|
||||
|
||||
_(Only for MacOS / Windows)_
|
||||
|
||||
Stop the Docker Desktop application. Then run Dangerzone. Dangerzone should
|
||||
prompt the user to start Docker Desktop.
|
||||
|
||||
|
||||
#### 3. Updating Dangerzone handles external state correctly.
|
||||
|
||||
_(Applies to Windows/MacOS)_
|
||||
|
||||
Install the previous version of Dangerzone, downloaded from the website.
|
||||
|
||||
Open the Dangerzone application and enable some non-default settings.
|
||||
**If there are new settings, make sure to change those as well**.
|
||||
|
||||
Close the Dangerzone application and get the container image for that
|
||||
version. For example:
|
||||
|
||||
```
|
||||
$ docker images dangerzone.rocks/dangerzone:latest
|
||||
REPOSITORY TAG IMAGE ID CREATED SIZE
|
||||
dangerzone.rocks/dangerzone latest <image ID> <date> <size>
|
||||
```
|
||||
|
||||
Then run the version under QA and ensure that the settings remain changed.
|
||||
|
||||
Afterwards check that new docker image was installed by running the same command
|
||||
and seeing the following differences:
|
||||
|
||||
```
|
||||
$ docker images dangerzone.rocks/dangerzone:latest
|
||||
REPOSITORY TAG IMAGE ID CREATED SIZE
|
||||
dangerzone.rocks/dangerzone latest <different ID> <newer date> <different size>
|
||||
```
|
||||
|
||||
#### 4. Dangerzone successfully installs the container image
|
||||
|
||||
_(Only for Linux)_
|
||||
|
||||
Remove the Dangerzone container image from Docker/Podman. Then run Dangerzone.
|
||||
Dangerzone should install the container image successfully.
|
||||
|
||||
#### 5. Dangerzone retains the settings of previous runs
|
||||
|
||||
Run Dangerzone and make some changes in the settings (e.g., change the OCR
|
||||
language, toggle whether to open the document after conversion, etc.). Restart
|
||||
Dangerzone. Dangerzone should show the settings that the user chose.
|
||||
|
||||
#### 6. Dangerzone reports failed conversions
|
||||
|
||||
Run Dangerzone and convert the `tests/test_docs/sample_bad_pdf.pdf` document.
|
||||
Dangerzone should fail gracefully, by reporting that the operation failed, and
|
||||
showing the following error message:
|
||||
|
||||
> The document format is not supported
|
||||
|
||||
#### 7. Dangerzone succeeds in converting multiple documents
|
||||
|
||||
Run Dangerzone against a list of documents, and tick all options. Ensure that:
|
||||
* Conversions take place sequentially.
|
||||
* Attempting to close the window while converting asks the user if they want to
|
||||
abort the conversions.
|
||||
* Conversions are completed successfully.
|
||||
* Conversions show individual progress in real-time (double-check for Qubes).
|
||||
* _(Only for Linux)_ The resulting files open with the PDF viewer of our choice.
|
||||
* OCR seems to have detected characters in the PDF files.
|
||||
* The resulting files have been saved with the proper suffix, in the proper
|
||||
location.
|
||||
* The original files have been saved in the `unsafe/` directory.
|
||||
|
||||
#### 8. Dangerzone CLI succeeds in converting multiple documents
|
||||
|
||||
_(Only for Windows and Linux)_
|
||||
|
||||
Run Dangerzone CLI against a list of documents. Ensure that conversions happen
|
||||
sequentially, are completed successfully, and we see their progress.
|
||||
|
||||
#### 9. Dangerzone can open a document for conversion via right-click -> "Open With"
|
||||
|
||||
_(Only for Windows, MacOS and Qubes)_
|
||||
|
||||
Go to a directory with office documents, right-click on one, and click on "Open
|
||||
With". We should be able to open the file with Dangerzone, and then convert it.
|
||||
|
||||
#### 10. Dangerzone shows helpful errors for setup issues on Qubes
|
||||
|
||||
_(Only for Qubes)_
|
||||
|
||||
Check what errors does Dangerzone throw in the following scenarios. The errors
|
||||
should point the user to the Qubes notifications in the top-right corner:
|
||||
|
||||
1. The `dz-dvm` template does not exist. We can trigger this scenario by
|
||||
temporarily renaming this template.
|
||||
2. The Dangerzone RPC policy does not exist. We can trigger this scenario by
|
||||
temporarily renaming the `dz.Convert` policy.
|
||||
3. The `dz-dvm` disposable Qube cannot start due to insufficient resources. We
|
||||
can trigger this scenario by temporarily increasing the minimum required RAM
|
||||
of the `dz-dvm` template to more than the available amount.
|
||||
|
||||
## Release
|
||||
|
||||
Once we are confident that the release will be out shortly, and doesn't need any more changes:
|
||||
|
||||
- [ ] Create a PGP-signed git tag for the version, e.g., for dangerzone `v0.1.0`:
|
||||
|
||||
```bash
|
||||
git tag -s v0.1.0
|
||||
git push origin v0.1.0
|
||||
```
|
||||
**Note**: release candidates are suffixed by `-rcX`.
|
||||
|
||||
> [!IMPORTANT]
|
||||
> Because we don't have [reproducible builds](https://github.com/freedomofpress/dangerzone/issues/188)
|
||||
> yet, building the Dangerzone container image in various platforms would lead
|
||||
|
|
@ -87,19 +250,7 @@ Once we are confident that the release will be out shortly, and doesn't need any
|
|||
|
||||
### macOS Release
|
||||
|
||||
> [!TIP]
|
||||
> You can automate these steps from your macOS terminal app with:
|
||||
>
|
||||
> ```
|
||||
> export APPLE_ID=<email>
|
||||
> make build-macos-intel # for Intel macOS
|
||||
> make build-macos-arm # for Apple Silicon macOS
|
||||
> ```
|
||||
|
||||
The following needs to happen for both Silicon and Intel chipsets.
|
||||
|
||||
#### Initial Setup
|
||||
|
||||
- Build machine must have:
|
||||
- Apple-trusted `Developer ID Application: Freedom of the Press Foundation (94ZZGGGJ3W)` code-signing certificates installed
|
||||
- Apple account must have:
|
||||
|
|
@ -111,84 +262,36 @@ The following needs to happen for both Silicon and Intel chipsets.
|
|||
https://developer.apple.com and login with the proper Apple ID.
|
||||
|
||||
#### Releasing and Signing
|
||||
- [ ] Verify and install the latest supported Python version from [python.org](https://www.python.org/downloads/macos/)
|
||||
- [ ] Verify and checkout the git tag for this release
|
||||
- [ ] Run `poetry install`
|
||||
- [ ] Run `poetry run ./install/macos/build-app.py`; this will make `dist/Dangerzone.app`
|
||||
- [ ] Run `poetry run ./install/macos/build-app.py --only-codesign`; this will make `dist/Dangerzone.dmg`
|
||||
* You need to run this command as the account that has access to the code signing certificate
|
||||
* You must run this command from the MacOS UI, from a terminal application.
|
||||
- [ ] Notarize it: `xcrun notarytool submit --apple-id "<email>" --keychain-profile "dz-notarytool-release-key" dist/Dangerzone.dmg`
|
||||
* In the end you'll get a `REQUEST_UUID`, which identifies the submission. Keep it to check on its status.
|
||||
* You need to change the `<email>` in the above command with the email
|
||||
associated with the Apple Developer ID.
|
||||
* This command assumes that you have created, and stored in the Keychain, an
|
||||
application password associated with your Apple Developer ID, which will be
|
||||
used specifically for `notarytool`.
|
||||
- [ ] Wait for it to get approved, check status with: `xcrun notarytool info <REQUEST_UUID> --apple-id "<email>" --keychain-profile "dz-notarytool-release-key"`
|
||||
* If it gets rejected, you should be able to see why with the same command
|
||||
(or use the `log` option for a more verbose JSON output)
|
||||
* You will also receive an update in your email.
|
||||
- [ ] After it's approved, staple the ticket: `xcrun stapler staple dist/Dangerzone.dmg`
|
||||
|
||||
Here is what you need to do:
|
||||
This process ends up with the final file:
|
||||
|
||||
- [ ] Verify and install the latest supported Python version from
|
||||
[python.org](https://www.python.org/downloads/macos/) (do not use the one from
|
||||
brew as it is known to [cause issues](https://github.com/freedomofpress/dangerzone/issues/471))
|
||||
```
|
||||
dist/Dangerzone.dmg
|
||||
```
|
||||
|
||||
- [ ] Checkout the dependencies, and clean your local copy:
|
||||
|
||||
```bash
|
||||
|
||||
# In case of a new Python installation or minor version upgrade, e.g., from
|
||||
# 3.11 to 3.12, reinstall Poetry
|
||||
python3 -m pip install poetry
|
||||
|
||||
# You can verify the correct Python version is used
|
||||
poetry debug info
|
||||
|
||||
# Replace with the actual version
|
||||
export DZ_VERSION=$(cat share/version.txt)
|
||||
|
||||
# Verify and checkout the git tag for this release:
|
||||
git checkout -f v$VERSION
|
||||
|
||||
# Clean the git repository
|
||||
git clean -df
|
||||
|
||||
# Clean up the environment
|
||||
poetry env remove --all
|
||||
|
||||
# Install the dependencies
|
||||
poetry sync
|
||||
```
|
||||
|
||||
- [ ] Build the container image and the OCR language data
|
||||
|
||||
```bash
|
||||
poetry run ./install/common/build-image.py
|
||||
poetry run ./install/common/download-tessdata.py
|
||||
|
||||
# Copy the container image to the assets folder
|
||||
cp share/container.tar ~dz/release-assets/$VERSION/dangerzone-$VERSION-arm64.tar
|
||||
cp share/image-id.txt ~dz/release-assets/$VERSION/.
|
||||
```
|
||||
|
||||
- [ ] Build the app bundle
|
||||
|
||||
```bash
|
||||
poetry run ./install/macos/build-app.py
|
||||
```
|
||||
|
||||
- [ ] Sign the application bundle, and notarize it
|
||||
|
||||
You need to run this command as the account that has access to the code signing certificate
|
||||
|
||||
This command assumes that you have created, and stored in the Keychain, an
|
||||
application password associated with your Apple Developer ID, which will be
|
||||
used specifically for `notarytool`.
|
||||
|
||||
```bash
|
||||
# Sign the .App and make it a .dmg
|
||||
poetry run ./install/macos/build-app.py --only-codesign
|
||||
|
||||
# Notarize it. You must run this command from the MacOS UI
|
||||
# from a terminal application.
|
||||
xcrun notarytool submit ./dist/Dangerzone.dmg --apple-id $APPLE_ID --keychain-profile "dz-notarytool-release-key" --wait && xcrun stapler staple dist/Dangerzone.dmg
|
||||
|
||||
# Copy the .dmg to the assets folder
|
||||
ARCH=$(uname -m)
|
||||
if [ "$ARCH" = "x86_64" ]; then
|
||||
ARCH="i686"
|
||||
fi
|
||||
cp dist/Dangerzone.dmg ~dz/release-assets/$VERSION/Dangerzone-$VERSION-$ARCH.dmg
|
||||
```
|
||||
Rename `Dangerzone.dmg` to `Dangerzone-$VERSION.dmg`.
|
||||
|
||||
### Windows Release
|
||||
|
||||
The Windows release is performed in a Windows 11 virtual machine (as opposed to a physical one).
|
||||
The Windows release is performed in a Windows 11 virtual machine as opposed to a physical one.
|
||||
|
||||
#### Initial Setup
|
||||
|
||||
|
|
@ -202,34 +305,11 @@ The Windows release is performed in a Windows 11 virtual machine (as opposed to
|
|||
|
||||
#### Releasing and Signing
|
||||
|
||||
- [ ] Checkout the dependencies, and clean your local copy:
|
||||
```bash
|
||||
# In case of a new Python installation or minor version upgrade, e.g., from
|
||||
# 3.11 to 3.12, reinstall Poetry
|
||||
python3 -m pip install poetry
|
||||
|
||||
# You can verify the correct Python version is used
|
||||
poetry debug info
|
||||
|
||||
# Replace with the actual version
|
||||
export DZ_VERSION=$(cat share/version.txt)
|
||||
|
||||
# Verify and checkout the git tag for this release:
|
||||
git checkout -f v$VERSION
|
||||
|
||||
# Clean the git repository
|
||||
git clean -df
|
||||
|
||||
# Clean up the environment
|
||||
poetry env remove --all
|
||||
|
||||
# Install the dependencies
|
||||
poetry sync
|
||||
```
|
||||
|
||||
- [ ] Verify and checkout the git tag for this release
|
||||
- [ ] Run `poetry install`
|
||||
- [ ] Copy the container image into the VM
|
||||
> [!IMPORTANT]
|
||||
> Instead of running `python .\install\windows\build-image.py` in the VM, run the build image script on the host (making sure to build for `linux/amd64`). Copy `share/container.tar` and `share/image-id.txt` from the host into the `share` folder in the VM.
|
||||
> Instead of running `python .\install\windows\build-image.py` in the VM, run the build image script on the host (making sure to build for `linux/amd64`). Copy `share/container.tar.gz` and `share/image-id.txt` from the host into the `share` folder in the VM
|
||||
- [ ] Run `poetry run .\install\windows\build-app.bat`
|
||||
- [ ] When you're done you will have `dist\Dangerzone.msi`
|
||||
|
||||
|
|
@ -237,17 +317,6 @@ Rename `Dangerzone.msi` to `Dangerzone-$VERSION.msi`.
|
|||
|
||||
### Linux release
|
||||
|
||||
> [!TIP]
|
||||
> You can automate these steps from any Linux distribution with:
|
||||
>
|
||||
> ```
|
||||
> make build-linux
|
||||
> ```
|
||||
>
|
||||
> You can then add the created artifacts to the appropriate APT/YUM repo.
|
||||
|
||||
Below we explain how we build packages for each Linux distribution we support.
|
||||
|
||||
#### Debian/Ubuntu
|
||||
|
||||
Because the Debian packages do not contain compiled Python code for a specific
|
||||
|
|
@ -259,15 +328,21 @@ instructions in our build section](https://github.com/freedomofpress/dangerzone/
|
|||
or create your own locally with:
|
||||
|
||||
```sh
|
||||
# Create and run debian bookworm development environment
|
||||
./dev_scripts/env.py --distro debian --version bookworm build-dev
|
||||
./dev_scripts/env.py --distro debian --version bookworm run --dev bash
|
||||
cd dangerzone
|
||||
```
|
||||
|
||||
# Build the latest container
|
||||
./dev_scripts/env.py --distro debian --version bookworm run --dev bash -c "cd dangerzone && poetry run ./install/common/build-image.py"
|
||||
Build the latest container:
|
||||
|
||||
# Create a .deb
|
||||
./dev_scripts/env.py --distro debian --version bookworm run --dev bash -c "cd dangerzone && ./install/linux/build-deb.py"
|
||||
```sh
|
||||
python3 ./install/common/build-image.py
|
||||
```
|
||||
|
||||
Create a .deb:
|
||||
|
||||
```sh
|
||||
./install/linux/build-deb.py
|
||||
```
|
||||
|
||||
Publish the .deb under `./deb_dist` to the
|
||||
|
|
@ -278,20 +353,28 @@ repo, by sending a PR. Follow the instructions in that repo on how to do so.
|
|||
|
||||
> **NOTE**: This procedure will have to be done for every supported Fedora version.
|
||||
>
|
||||
> In this section, we'll use Fedora 41 as an example.
|
||||
> In this section, we'll use Fedora 39 as an example.
|
||||
|
||||
Create a Fedora development environment. You can [follow the
|
||||
instructions in our build section](https://github.com/freedomofpress/dangerzone/blob/main/BUILD.md#fedora),
|
||||
or create your own locally with:
|
||||
|
||||
```sh
|
||||
./dev_scripts/env.py --distro fedora --version 41 build-dev
|
||||
./dev_scripts/env.py --distro fedora --version 39 build-dev
|
||||
./dev_scripts/env.py --distro fedora --version 39 run --dev bash
|
||||
cd dangerzone
|
||||
```
|
||||
|
||||
# Build the latest container (skip if already built):
|
||||
./dev_scripts/env.py --distro fedora --version 41 run --dev bash -c "cd dangerzone && poetry run ./install/common/build-image.py"
|
||||
Build the latest container:
|
||||
|
||||
# Create a .rpm:
|
||||
./dev_scripts/env.py --distro fedora --version 41 run --dev bash -c "cd dangerzone && ./install/linux/build-rpm.py"
|
||||
```sh
|
||||
python3 ./install/common/build-image.py
|
||||
```
|
||||
|
||||
Create a .rpm:
|
||||
|
||||
```sh
|
||||
./install/linux/build-rpm.py
|
||||
```
|
||||
|
||||
Publish the .rpm under `./dist` to the
|
||||
|
|
@ -302,7 +385,7 @@ Publish the .rpm under `./dist` to the
|
|||
Create a .rpm for Qubes:
|
||||
|
||||
```sh
|
||||
./dev_scripts/env.py --distro fedora --version 41 run --dev bash -c "cd dangerzone && ./install/linux/build-rpm.py --qubes"
|
||||
./install/linux/build-rpm.py --qubes
|
||||
```
|
||||
|
||||
and similarly publish it to the [`freedomofpress/yum-tools-prod`](https://github.com/freedomofpress/yum-tools-prod)
|
||||
|
|
@ -310,39 +393,25 @@ repo.
|
|||
|
||||
## Publishing the Release
|
||||
|
||||
To publish the release, you can follow these steps:
|
||||
|
||||
- [ ] Create an archive of the Dangerzone source in `tar.gz` format:
|
||||
```bash
|
||||
export VERSION=$(cat share/version.txt)
|
||||
git archive --format=tar.gz -o dangerzone-${VERSION:?}.tar.gz --prefix=dangerzone/ v${VERSION:?}
|
||||
```
|
||||
To publish the release:
|
||||
|
||||
- [ ] Run container scan on the produced container images (some time may have passed since the artifacts were built)
|
||||
```bash
|
||||
docker pull anchore/grype:latest
|
||||
docker run --rm -v ./share/container.tar:/container.tar anchore/grype:latest /container.tar
|
||||
```
|
||||
|
||||
- [ ] Collect the assets in a single directory, calculate their SHA-256 hashes, and sign them.
|
||||
There is an `./dev_scripts/sign-assets.py` script to automate this task.
|
||||
* You can use `./dev_scripts/sign-assets.py`, if you want to automate this
|
||||
task.
|
||||
- [ ] Create a new **draft** release on GitHub and upload the macOS and Windows installers.
|
||||
* Copy the release notes text from the template at [`docs/templates/release-notes`](https://github.com/freedomofpress/dangerzone/tree/main/docs/templates/)
|
||||
* You can use `./dev_scripts/upload-asset.py`, if you want to upload an asset
|
||||
using an access token.
|
||||
- [ ] Upload the `container.tar.gz` i686 image that was created in the previous step
|
||||
|
||||
**Important:** Before running the script, make sure that it's the same container images as
|
||||
the ones that are shipped in other platforms (see our [Pre-release](#Pre-release) section)
|
||||
**Important:** Make sure that it's the same container image as the ones that
|
||||
are shipped in other platforms (see our [Pre-release](#Pre-release) section)
|
||||
|
||||
```bash
|
||||
# Sign all the assets
|
||||
./dev_scripts/sign-assets.py ~/release-assets/$VERSION/github --version $VERSION
|
||||
```
|
||||
|
||||
- [ ] Upload all the assets to the draft release on GitHub.
|
||||
```bash
|
||||
find ~/release-assets/$VERSION/github | xargs -n1 ./dev_scripts/upload-asset.py --token ~/token --draft
|
||||
```
|
||||
|
||||
- [ ] Update the [Dangerzone website](https://github.com/freedomofpress/dangerzone.rocks) to link to the new installers.
|
||||
- [ ] Upload the detached signatures (.asc) and checksum file.
|
||||
- [ ] Update the [Dangerzone website](https://github.com/freedomofpress/dangerzone.rocks) to link to the new installers and signatures
|
||||
- [ ] Update the brew cask release of Dangerzone with a [PR like this one](https://github.com/Homebrew/homebrew-cask/pull/116319)
|
||||
- [ ] Update version and links to our installation instructions (`INSTALL.md`) in `README.md`
|
||||
- [ ] Update version and download links in `README.md`
|
||||
|
||||
## Post-release
|
||||
|
||||
|
|
|
|||
|
|
@ -1,14 +0,0 @@
|
|||
This project includes third-party components as follows:
|
||||
|
||||
1. gVisor APT Key
|
||||
- URL: https://gvisor.dev/archive.key
|
||||
- Last updated: 2025-01-21
|
||||
- Description: This is the public key used for verifying packages from the gVisor repository.
|
||||
|
||||
2. Reproducible Containers Helper Script
|
||||
- URL: https://github.com/reproducible-containers/repro-sources-list.sh/blob/d15cf12b26395b857b24fba223b108aff1c91b26/repro-sources-list.sh
|
||||
- Last updated: 2025-01-21
|
||||
- Description: This script is used for building reproducible Debian images.
|
||||
|
||||
Please refer to the respective sources for licensing information and further details regarding the use of these components.
|
||||
|
||||
|
|
@ -1,25 +1,6 @@
|
|||
import logging
|
||||
import os
|
||||
import sys
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Call freeze_support() to avoid passing unknown options to the subprocess.
|
||||
# See https://github.com/freedomofpress/dangerzone/issues/873
|
||||
import multiprocessing
|
||||
|
||||
multiprocessing.freeze_support()
|
||||
|
||||
|
||||
try:
|
||||
from . import vendor # type: ignore [attr-defined]
|
||||
|
||||
vendor_path: str = vendor.__path__[0]
|
||||
logger.debug(f"Using vendored PyMuPDF libraries from '{vendor_path}'")
|
||||
sys.path.insert(0, vendor_path)
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
if "DANGERZONE_MODE" in os.environ:
|
||||
mode = os.environ["DANGERZONE_MODE"]
|
||||
else:
|
||||
|
|
@ -32,4 +13,4 @@ else:
|
|||
if mode == "cli":
|
||||
from .cli import cli_main as main
|
||||
else:
|
||||
from .gui import gui_main as main # noqa: F401
|
||||
from .gui import gui_main as main
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import logging
|
||||
import sys
|
||||
from typing import List, Optional
|
||||
from typing import Any, Callable, List, Optional, TypeVar
|
||||
|
||||
import click
|
||||
from colorama import Back, Fore, Style
|
||||
|
|
@ -11,9 +11,10 @@ from .isolation_provider.container import Container
|
|||
from .isolation_provider.dummy import Dummy
|
||||
from .isolation_provider.qubes import Qubes, is_qubes_native_conversion
|
||||
from .logic import DangerzoneCore
|
||||
from .settings import Settings
|
||||
from .util import get_version, replace_control_chars
|
||||
|
||||
F = TypeVar("F", bound=Callable[..., Any])
|
||||
|
||||
|
||||
def print_header(s: str) -> None:
|
||||
click.echo("")
|
||||
|
|
@ -38,62 +39,30 @@ def print_header(s: str) -> None:
|
|||
)
|
||||
@click.argument(
|
||||
"filenames",
|
||||
required=False,
|
||||
required=True,
|
||||
nargs=-1,
|
||||
type=click.UNPROCESSED,
|
||||
callback=args.validate_input_filenames,
|
||||
)
|
||||
@click.option(
|
||||
"--debug",
|
||||
"debug",
|
||||
flag_value=True,
|
||||
help="Run Dangerzone in debug mode, to get logs from gVisor.",
|
||||
)
|
||||
@click.option(
|
||||
"--set-container-runtime",
|
||||
required=False,
|
||||
help=(
|
||||
"The name or full path of the container runtime you want Dangerzone to use."
|
||||
" You can specify the value 'default' if you want to take back your choice, and"
|
||||
" let Dangerzone use the default runtime for this OS"
|
||||
),
|
||||
)
|
||||
@click.version_option(version=get_version(), message="%(version)s")
|
||||
@errors.handle_document_errors
|
||||
def cli_main(
|
||||
output_filename: Optional[str],
|
||||
ocr_lang: Optional[str],
|
||||
filenames: Optional[List[str]],
|
||||
filenames: List[str],
|
||||
archive: bool,
|
||||
dummy_conversion: bool,
|
||||
debug: bool,
|
||||
set_container_runtime: Optional[str] = None,
|
||||
) -> None:
|
||||
setup_logging()
|
||||
display_banner()
|
||||
if set_container_runtime:
|
||||
settings = Settings()
|
||||
if set_container_runtime == "default":
|
||||
settings.unset_custom_runtime()
|
||||
click.echo(
|
||||
"Instructed Dangerzone to use the default container runtime for this OS"
|
||||
)
|
||||
else:
|
||||
container_runtime = settings.set_custom_runtime(
|
||||
set_container_runtime, autosave=True
|
||||
)
|
||||
click.echo(f"Set the settings container_runtime to {container_runtime}")
|
||||
sys.exit(0)
|
||||
elif not filenames:
|
||||
raise click.UsageError("Missing argument 'FILENAMES...'")
|
||||
|
||||
if getattr(sys, "dangerzone_dev", False) and dummy_conversion:
|
||||
dangerzone = DangerzoneCore(Dummy())
|
||||
elif is_qubes_native_conversion():
|
||||
dangerzone = DangerzoneCore(Qubes())
|
||||
else:
|
||||
dangerzone = DangerzoneCore(Container(debug=debug))
|
||||
dangerzone = DangerzoneCore(Container())
|
||||
|
||||
display_banner()
|
||||
if len(filenames) == 1 and output_filename:
|
||||
dangerzone.add_document_from_filename(filenames[0], output_filename, archive)
|
||||
elif len(filenames) > 1 and output_filename:
|
||||
|
|
@ -328,7 +297,7 @@ def display_banner() -> None:
|
|||
+ Back.BLACK
|
||||
+ Fore.LIGHTWHITE_EX
|
||||
+ Style.BRIGHT
|
||||
+ f"{' ' * left_spaces}Dangerzone v{get_version()}{' ' * right_spaces}"
|
||||
+ f"{' '*left_spaces}Dangerzone v{get_version()}{' '*right_spaces}"
|
||||
+ Fore.YELLOW
|
||||
+ Style.DIM
|
||||
+ "│"
|
||||
|
|
@ -346,10 +315,4 @@ def display_banner() -> None:
|
|||
+ Style.DIM
|
||||
+ "│"
|
||||
)
|
||||
print(
|
||||
Back.BLACK
|
||||
+ Fore.YELLOW
|
||||
+ Style.DIM
|
||||
+ "╰──────────────────────────╯"
|
||||
+ Style.RESET_ALL
|
||||
)
|
||||
print(Back.BLACK + Fore.YELLOW + Style.DIM + "╰──────────────────────────╯")
|
||||
|
|
|
|||
|
|
@ -1,235 +0,0 @@
|
|||
#!/usr/bin/python3
|
||||
|
||||
import json
|
||||
import os
|
||||
import shlex
|
||||
import subprocess
|
||||
import sys
|
||||
import typing
|
||||
|
||||
# This script wraps the command-line arguments passed to it to run as an
|
||||
# unprivileged user in a gVisor sandbox.
|
||||
# Its behavior can be modified with the following environment variables:
|
||||
# RUNSC_DEBUG: If set, print debug messages to stderr, and log all gVisor
|
||||
# output to stderr.
|
||||
# RUNSC_FLAGS: If set, pass these flags to the `runsc` invocation.
|
||||
# These environment variables are not passed on to the sandboxed process.
|
||||
|
||||
|
||||
def log(message: str, *values: typing.Any) -> None:
|
||||
"""Helper function to log messages if RUNSC_DEBUG is set."""
|
||||
if os.environ.get("RUNSC_DEBUG"):
|
||||
print(message.format(*values), file=sys.stderr)
|
||||
|
||||
|
||||
command = sys.argv[1:]
|
||||
if len(command) == 0:
|
||||
log("Invoked without a command; will execute 'sh'.")
|
||||
command = ["sh"]
|
||||
else:
|
||||
log("Invoked with command: {}", " ".join(shlex.quote(s) for s in command))
|
||||
|
||||
# Build and write container OCI config.
|
||||
oci_config: dict[str, typing.Any] = {
|
||||
"ociVersion": "1.0.0",
|
||||
"process": {
|
||||
"user": {
|
||||
# Hardcode the UID/GID of the container image to 1000, since we're in
|
||||
# control of the image creation, and we don't expect it to change.
|
||||
"uid": 1000,
|
||||
"gid": 1000,
|
||||
},
|
||||
"args": command,
|
||||
"env": [
|
||||
"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
|
||||
"PYTHONPATH=/opt/dangerzone",
|
||||
"TERM=xterm",
|
||||
],
|
||||
"cwd": "/",
|
||||
"capabilities": {
|
||||
"bounding": [],
|
||||
"effective": [],
|
||||
"inheritable": [],
|
||||
"permitted": [],
|
||||
},
|
||||
"rlimits": [
|
||||
{"type": "RLIMIT_NOFILE", "hard": 4096, "soft": 4096},
|
||||
],
|
||||
},
|
||||
"root": {"path": "rootfs", "readonly": True},
|
||||
"hostname": "dangerzone",
|
||||
"mounts": [
|
||||
# Mask almost every system directory of the outer container, by mounting tmpfs
|
||||
# on top of them. This is done to avoid leaking any sensitive information,
|
||||
# either mounted by Podman/Docker, or when gVisor runs, since we reuse the same
|
||||
# rootfs. We basically mask everything except for `/usr`, `/bin`, `/lib`,
|
||||
# `/etc`, and `/opt`.
|
||||
#
|
||||
# Note that we set `--root /home/dangerzone/.containers` for the directory where
|
||||
# gVisor will create files at runtime, which means that in principle, we are
|
||||
# covered by the masking of `/home/dangerzone` that follows below.
|
||||
#
|
||||
# Finally, note that the following list has been taken from the dirs in our
|
||||
# container image, and double-checked against the top-level dirs listed in the
|
||||
# Filesystem Hierarchy Standard (FHS) [1]. It would be nice to have an allowlist
|
||||
# approach instead of a denylist, but FHS is such an old standard that we don't
|
||||
# expect any new top-level dirs to pop up any time soon.
|
||||
#
|
||||
# [1] https://en.wikipedia.org/wiki/Filesystem_Hierarchy_Standard
|
||||
{
|
||||
"destination": "/boot",
|
||||
"type": "tmpfs",
|
||||
"source": "tmpfs",
|
||||
"options": ["nosuid", "noexec", "nodev", "ro"],
|
||||
},
|
||||
{
|
||||
"destination": "/dev",
|
||||
"type": "tmpfs",
|
||||
"source": "tmpfs",
|
||||
"options": ["nosuid", "noexec", "nodev"],
|
||||
},
|
||||
{
|
||||
"destination": "/home",
|
||||
"type": "tmpfs",
|
||||
"source": "tmpfs",
|
||||
"options": ["nosuid", "noexec", "nodev", "ro"],
|
||||
},
|
||||
{
|
||||
"destination": "/media",
|
||||
"type": "tmpfs",
|
||||
"source": "tmpfs",
|
||||
"options": ["nosuid", "noexec", "nodev", "ro"],
|
||||
},
|
||||
{
|
||||
"destination": "/mnt",
|
||||
"type": "tmpfs",
|
||||
"source": "tmpfs",
|
||||
"options": ["nosuid", "noexec", "nodev", "ro"],
|
||||
},
|
||||
{
|
||||
"destination": "/proc",
|
||||
"type": "proc",
|
||||
"source": "proc",
|
||||
},
|
||||
{
|
||||
"destination": "/root",
|
||||
"type": "tmpfs",
|
||||
"source": "tmpfs",
|
||||
"options": ["nosuid", "noexec", "nodev", "ro"],
|
||||
},
|
||||
{
|
||||
"destination": "/run",
|
||||
"type": "tmpfs",
|
||||
"source": "tmpfs",
|
||||
"options": ["nosuid", "noexec", "nodev"],
|
||||
},
|
||||
{
|
||||
"destination": "/sbin",
|
||||
"type": "tmpfs",
|
||||
"source": "tmpfs",
|
||||
"options": ["nosuid", "noexec", "nodev", "ro"],
|
||||
},
|
||||
{
|
||||
"destination": "/srv",
|
||||
"type": "tmpfs",
|
||||
"source": "tmpfs",
|
||||
"options": ["nosuid", "noexec", "nodev", "ro"],
|
||||
},
|
||||
{
|
||||
"destination": "/sys",
|
||||
"type": "tmpfs",
|
||||
"source": "tmpfs",
|
||||
"options": ["nosuid", "noexec", "nodev", "ro"],
|
||||
},
|
||||
{
|
||||
"destination": "/tmp",
|
||||
"type": "tmpfs",
|
||||
"source": "tmpfs",
|
||||
"options": ["nosuid", "noexec", "nodev"],
|
||||
},
|
||||
{
|
||||
"destination": "/var",
|
||||
"type": "tmpfs",
|
||||
"source": "tmpfs",
|
||||
"options": ["nosuid", "noexec", "nodev"],
|
||||
},
|
||||
# LibreOffice needs a writable home directory, so just mount a tmpfs
|
||||
# over it.
|
||||
{
|
||||
"destination": "/home/dangerzone",
|
||||
"type": "tmpfs",
|
||||
"source": "tmpfs",
|
||||
"options": ["nosuid", "noexec", "nodev"],
|
||||
},
|
||||
# Used for LibreOffice extensions, which are only conditionally
|
||||
# installed depending on which file is being converted.
|
||||
{
|
||||
"destination": "/usr/lib/libreoffice/share/extensions/",
|
||||
"type": "tmpfs",
|
||||
"source": "tmpfs",
|
||||
"options": ["nosuid", "noexec", "nodev"],
|
||||
},
|
||||
],
|
||||
"linux": {
|
||||
"namespaces": [
|
||||
{"type": "pid"},
|
||||
{"type": "network"},
|
||||
{"type": "ipc"},
|
||||
{"type": "uts"},
|
||||
{"type": "mount"},
|
||||
],
|
||||
},
|
||||
}
|
||||
not_forwarded_env = set(
|
||||
(
|
||||
"PATH",
|
||||
"HOME",
|
||||
"SHLVL",
|
||||
"HOSTNAME",
|
||||
"TERM",
|
||||
"PWD",
|
||||
"RUNSC_FLAGS",
|
||||
"RUNSC_DEBUG",
|
||||
)
|
||||
)
|
||||
for key_val in oci_config["process"]["env"]:
|
||||
not_forwarded_env.add(key_val[: key_val.index("=")])
|
||||
for key, val in os.environ.items():
|
||||
if key in not_forwarded_env:
|
||||
continue
|
||||
oci_config["process"]["env"].append("%s=%s" % (key, val))
|
||||
if os.environ.get("RUNSC_DEBUG"):
|
||||
log("Command inside gVisor sandbox: {}", command)
|
||||
log("OCI config:")
|
||||
json.dump(oci_config, sys.stderr, indent=2, sort_keys=True)
|
||||
# json.dump doesn't print a trailing newline, so print one here:
|
||||
log("")
|
||||
with open("/home/dangerzone/dangerzone-image/config.json", "w") as oci_config_out:
|
||||
json.dump(oci_config, oci_config_out, indent=2, sort_keys=True)
|
||||
|
||||
# Run gVisor.
|
||||
runsc_argv = [
|
||||
"/usr/bin/runsc",
|
||||
"--rootless=true",
|
||||
"--network=none",
|
||||
"--root=/home/dangerzone/.containers",
|
||||
# Disable DirectFS for to make the seccomp filter even stricter,
|
||||
# at some performance cost.
|
||||
"--directfs=false",
|
||||
]
|
||||
if os.environ.get("RUNSC_DEBUG"):
|
||||
runsc_argv += ["--debug=true", "--alsologtostderr=true"]
|
||||
if os.environ.get("RUNSC_FLAGS"):
|
||||
runsc_argv += [x for x in shlex.split(os.environ.get("RUNSC_FLAGS", "")) if x]
|
||||
runsc_argv += ["run", "--bundle=/home/dangerzone/dangerzone-image", "dangerzone"]
|
||||
log(
|
||||
"Running gVisor with command line: {}", " ".join(shlex.quote(s) for s in runsc_argv)
|
||||
)
|
||||
runsc_process = subprocess.run(
|
||||
runsc_argv,
|
||||
check=False,
|
||||
)
|
||||
log("gVisor quit with exit code: {}", runsc_process.returncode)
|
||||
|
||||
# We're done.
|
||||
sys.exit(runsc_process.returncode)
|
||||
|
|
@ -1,29 +0,0 @@
|
|||
-----BEGIN PGP PUBLIC KEY BLOCK-----
|
||||
|
||||
mQINBF0meAYBEACcBYPOSBiKtid+qTQlbgKGPxUYt0cNZiQqWXylhYUT4PuNlNx5
|
||||
s+sBLFvNTpdTrXMmZ8NkekyjD1HardWvebvJT4u+Ho/9jUr4rP71cNwNtocz/w8G
|
||||
DsUXSLgH8SDkq6xw0L+5eGc78BBg9cOeBeFBm3UPgxTBXS9Zevoi2w1lzSxkXvjx
|
||||
cGzltzMZfPXERljgLzp9AAfhg/2ouqVQm37fY+P/NDzFMJ1XHPIIp9KJl/prBVud
|
||||
jJJteFZ5sgL6MwjBQq2kw+q2Jb8Zfjl0BeXDgGMN5M5lGhX2wTfiMbfo7KWyzRnB
|
||||
RpSP3BxlLqYeQUuLG5Yx8z3oA3uBkuKaFOKvXtiScxmGM/+Ri2YM3m66imwDhtmP
|
||||
AKwTPI3Re4gWWOffglMVSv2sUAY32XZ74yXjY1VhK3bN3WFUPGrgQx4X7GP0A1Te
|
||||
lzqkT3VSMXieImTASosK5L5Q8rryvgCeI9tQLn9EpYFCtU3LXvVgTreGNEEjMOnL
|
||||
dR7yOU+Fs775stn6ucqmdYarx7CvKUrNAhgEeHMonLe1cjYScF7NfLO1GIrQKJR2
|
||||
DE0f+uJZ52inOkO8ufh3WVQJSYszuS3HCY7w5oj1aP38k/y9zZdZvVvwAWZaiqBQ
|
||||
iwjVs6Kub76VVZZhRDf4iYs8k1Zh64nXdfQt250d8U5yMPF3wIJ+c1yhxwARAQAB
|
||||
tCpUaGUgZ1Zpc29yIEF1dGhvcnMgPGd2aXNvci1ib3RAZ29vZ2xlLmNvbT6JAk4E
|
||||
EwEKADgCGwMFCwkIBwIGFQoJCAsCBBYCAwECHgECF4AWIQRvHfheOnHCSRjnJ9Vv
|
||||
xtVU4yvZQwUCYO4TxQAKCRBvxtVU4yvZQ9UoEACLPV7CnEA2bjCPi0NCWB/Mo1WL
|
||||
evqv7Wv7vmXzI1K9DrqOhxuamQW75SVXg1df0hTJWbKFmDAip6NEC2Rg5P+A8hHj
|
||||
nW/VG+q4ZFT662jDhnXQiO9L7EZzjyqNF4yWYzzgnqEu/SmGkDLDYiUCcGBqS2oE
|
||||
EQfk7RHJSLMJXAnNDH7OUDgrirSssg/dlQ5uAHA9Au80VvC5fsTKza8b3Aydw3SV
|
||||
iB8/Yuikbl8wKbpSGiXtR4viElXjNips0+mBqaUk2xpqSBrsfN+FezcInVXaXFeq
|
||||
xtpq2/3M3DYbqCRjqeyd9wNi92FHdOusNrK4MYe0pAYbGjc65BwH+F0T4oJ8ZSJV
|
||||
lIt+FZ0MqM1T97XadybYFsJh8qvajQpZEPL+zzNncc4f1d80e7+lwIZV/al0FZWW
|
||||
Zlp7TpbeO/uW+lHs5W14YKwaQVh1whapKXTrATipNOOSCw2hnfrT8V7Hy55QWaGZ
|
||||
f4/kfy929EeCP16d/LqOClv0j0RBr6NhRBQ0l/BE/mXjJwIk6nKwi+Yi4ek1ARi6
|
||||
AlCMLn9AZF7aTGpvCiftzIrlyDfVZT5IX03TayxRHZ4b1Rj8eyJaHcjI49u83gkr
|
||||
4LGX08lEawn9nxFSx4RCg2swGiYw5F436wwwAIozqJuDASeTa3QND3au5v0oYWnl
|
||||
umDySUl5wPaAaALgzA==
|
||||
=5/8T
|
||||
-----END PGP PUBLIC KEY BLOCK-----
|
||||
|
|
@ -1,103 +0,0 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Copyright The repro-sources-list.sh Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# repro-sources-list.sh:
|
||||
# configures /etc/apt/sources.list and similar files for installing packages from a snapshot.
|
||||
#
|
||||
# This script is expected to be executed inside Dockerfile.
|
||||
#
|
||||
# The following distributions are supported:
|
||||
# - debian:11 (/etc/apt/sources.list)
|
||||
# - debian:12 (/etc/apt/sources.list.d/debian.sources)
|
||||
# - ubuntu:22.04 (/etc/apt/sources.list)
|
||||
# - ubuntu:24.04 (/etc/apt/sources.listd/ubuntu.sources)
|
||||
# - archlinux (/etc/pacman.d/mirrorlist)
|
||||
#
|
||||
# For the further information, see https://github.com/reproducible-containers/repro-sources-list.sh
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
set -eux -o pipefail
|
||||
|
||||
. /etc/os-release
|
||||
|
||||
: "${KEEP_CACHE:=1}"
|
||||
|
||||
keep_apt_cache() {
|
||||
rm -f /etc/apt/apt.conf.d/docker-clean
|
||||
echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' >/etc/apt/apt.conf.d/keep-cache
|
||||
}
|
||||
|
||||
case "${ID}" in
|
||||
"debian")
|
||||
: "${SNAPSHOT_ARCHIVE_BASE:=http://snapshot.debian.org/archive/}"
|
||||
: "${BACKPORTS:=}"
|
||||
if [ -e /etc/apt/sources.list.d/debian.sources ]; then
|
||||
: "${SOURCE_DATE_EPOCH:=$(stat --format=%Y /etc/apt/sources.list.d/debian.sources)}"
|
||||
rm -f /etc/apt/sources.list.d/debian.sources
|
||||
else
|
||||
: "${SOURCE_DATE_EPOCH:=$(stat --format=%Y /etc/apt/sources.list)}"
|
||||
fi
|
||||
snapshot="$(printf "%(%Y%m%dT%H%M%SZ)T\n" "${SOURCE_DATE_EPOCH}")"
|
||||
# TODO: use the new format for Debian >= 12
|
||||
echo "deb [check-valid-until=no] ${SNAPSHOT_ARCHIVE_BASE}debian/${snapshot} ${VERSION_CODENAME} main" >/etc/apt/sources.list
|
||||
echo "deb [check-valid-until=no] ${SNAPSHOT_ARCHIVE_BASE}debian-security/${snapshot} ${VERSION_CODENAME}-security main" >>/etc/apt/sources.list
|
||||
echo "deb [check-valid-until=no] ${SNAPSHOT_ARCHIVE_BASE}debian/${snapshot} ${VERSION_CODENAME}-updates main" >>/etc/apt/sources.list
|
||||
if [ "${BACKPORTS}" = 1 ]; then echo "deb [check-valid-until=no] ${SNAPSHOT_ARCHIVE_BASE}debian/${snapshot} ${VERSION_CODENAME}-backports main" >>/etc/apt/sources.list; fi
|
||||
if [ "${KEEP_CACHE}" = 1 ]; then keep_apt_cache; fi
|
||||
;;
|
||||
"ubuntu")
|
||||
: "${SNAPSHOT_ARCHIVE_BASE:=http://snapshot.ubuntu.com/}"
|
||||
if [ -e /etc/apt/sources.list.d/ubuntu.sources ]; then
|
||||
: "${SOURCE_DATE_EPOCH:=$(stat --format=%Y /etc/apt/sources.list.d/ubuntu.sources)}"
|
||||
rm -f /etc/apt/sources.list.d/ubuntu.sources
|
||||
else
|
||||
: "${SOURCE_DATE_EPOCH:=$(stat --format=%Y /etc/apt/sources.list)}"
|
||||
fi
|
||||
snapshot="$(printf "%(%Y%m%dT%H%M%SZ)T\n" "${SOURCE_DATE_EPOCH}")"
|
||||
# TODO: use the new format for Ubuntu >= 24.04
|
||||
echo "deb [check-valid-until=no] ${SNAPSHOT_ARCHIVE_BASE}ubuntu/${snapshot} ${VERSION_CODENAME} main restricted" >/etc/apt/sources.list
|
||||
echo "deb [check-valid-until=no] ${SNAPSHOT_ARCHIVE_BASE}ubuntu/${snapshot} ${VERSION_CODENAME}-updates main restricted" >>/etc/apt/sources.list
|
||||
echo "deb [check-valid-until=no] ${SNAPSHOT_ARCHIVE_BASE}ubuntu/${snapshot} ${VERSION_CODENAME} universe" >>/etc/apt/sources.list
|
||||
echo "deb [check-valid-until=no] ${SNAPSHOT_ARCHIVE_BASE}ubuntu/${snapshot} ${VERSION_CODENAME}-updates universe" >>/etc/apt/sources.list
|
||||
echo "deb [check-valid-until=no] ${SNAPSHOT_ARCHIVE_BASE}ubuntu/${snapshot} ${VERSION_CODENAME} multiverse" >>/etc/apt/sources.list
|
||||
echo "deb [check-valid-until=no] ${SNAPSHOT_ARCHIVE_BASE}ubuntu/${snapshot} ${VERSION_CODENAME}-updates multiverse" >>/etc/apt/sources.list
|
||||
echo "deb [check-valid-until=no] ${SNAPSHOT_ARCHIVE_BASE}ubuntu/${snapshot} ${VERSION_CODENAME}-backports main restricted universe multiverse" >>/etc/apt/sources.list
|
||||
echo "deb [check-valid-until=no] ${SNAPSHOT_ARCHIVE_BASE}ubuntu/${snapshot} ${VERSION_CODENAME}-security main restricted" >>/etc/apt/sources.list
|
||||
echo "deb [check-valid-until=no] ${SNAPSHOT_ARCHIVE_BASE}ubuntu/${snapshot} ${VERSION_CODENAME}-security universe" >>/etc/apt/sources.list
|
||||
echo "deb [check-valid-until=no] ${SNAPSHOT_ARCHIVE_BASE}ubuntu/${snapshot} ${VERSION_CODENAME}-security multiverse" >>/etc/apt/sources.list
|
||||
if [ "${KEEP_CACHE}" = 1 ]; then keep_apt_cache; fi
|
||||
# http://snapshot.ubuntu.com is redirected to https, so we have to install ca-certificates
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
apt-get -o Acquire::https::Verify-Peer=false update >&2
|
||||
apt-get -o Acquire::https::Verify-Peer=false install -y ca-certificates >&2
|
||||
;;
|
||||
"arch")
|
||||
: "${SNAPSHOT_ARCHIVE_BASE:=http://archive.archlinux.org/}"
|
||||
: "${SOURCE_DATE_EPOCH:=$(stat --format=%Y /var/log/pacman.log)}"
|
||||
export SOURCE_DATE_EPOCH
|
||||
# shellcheck disable=SC2016
|
||||
date -d "@${SOURCE_DATE_EPOCH}" "+Server = ${SNAPSHOT_ARCHIVE_BASE}repos/%Y/%m/%d/\$repo/os/\$arch" >/etc/pacman.d/mirrorlist
|
||||
;;
|
||||
*)
|
||||
echo >&2 "Unsupported distribution: ${ID}"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
: "${WRITE_SOURCE_DATE_EPOCH:=/dev/null}"
|
||||
echo "${SOURCE_DATE_EPOCH}" >"${WRITE_SOURCE_DATE_EPOCH}"
|
||||
echo "SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH}"
|
||||
|
|
@ -1,201 +0,0 @@
|
|||
import logging
|
||||
import os
|
||||
import platform
|
||||
import shutil
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from typing import List, Optional, Tuple
|
||||
|
||||
from . import errors
|
||||
from .settings import Settings
|
||||
from .util import get_resource_path, get_subprocess_startupinfo
|
||||
|
||||
CONTAINER_NAME = "dangerzone.rocks/dangerzone"
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Runtime(object):
|
||||
"""Represents the container runtime to use.
|
||||
|
||||
- It can be specified via the settings, using the "container_runtime" key,
|
||||
which should point to the full path of the runtime;
|
||||
- If the runtime is not specified via the settings, it defaults
|
||||
to "podman" on Linux and "docker" on macOS and Windows.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
settings = Settings()
|
||||
|
||||
if settings.custom_runtime_specified():
|
||||
self.path = Path(settings.get("container_runtime"))
|
||||
if not self.path.exists():
|
||||
raise errors.UnsupportedContainerRuntime(self.path)
|
||||
self.name = self.path.stem
|
||||
else:
|
||||
self.name = self.get_default_runtime_name()
|
||||
self.path = Runtime.path_from_name(self.name)
|
||||
|
||||
if self.name not in ("podman", "docker"):
|
||||
raise errors.UnsupportedContainerRuntime(self.name)
|
||||
|
||||
@staticmethod
|
||||
def path_from_name(name: str) -> Path:
|
||||
name_path = Path(name)
|
||||
if name_path.is_file():
|
||||
return name_path
|
||||
else:
|
||||
runtime = shutil.which(name_path)
|
||||
if runtime is None:
|
||||
raise errors.NoContainerTechException(name)
|
||||
return Path(runtime)
|
||||
|
||||
@staticmethod
|
||||
def get_default_runtime_name() -> str:
|
||||
return "podman" if platform.system() == "Linux" else "docker"
|
||||
|
||||
|
||||
def get_runtime_version(runtime: Optional[Runtime] = None) -> Tuple[int, int]:
|
||||
"""Get the major/minor parts of the Docker/Podman version.
|
||||
|
||||
Some of the operations we perform in this module rely on some Podman features
|
||||
that are not available across all of our platforms. In order to have a proper
|
||||
fallback, we need to know the Podman version. More specifically, we're fine with
|
||||
just knowing the major and minor version, since writing/installing a full-blown
|
||||
semver parser is an overkill.
|
||||
"""
|
||||
runtime = runtime or Runtime()
|
||||
|
||||
# Get the Docker/Podman version, using a Go template.
|
||||
if runtime.name == "podman":
|
||||
query = "{{.Client.Version}}"
|
||||
else:
|
||||
query = "{{.Server.Version}}"
|
||||
|
||||
cmd = [str(runtime.path), "version", "-f", query]
|
||||
try:
|
||||
version = subprocess.run(
|
||||
cmd,
|
||||
startupinfo=get_subprocess_startupinfo(),
|
||||
capture_output=True,
|
||||
check=True,
|
||||
).stdout.decode()
|
||||
except Exception as e:
|
||||
msg = f"Could not get the version of the {runtime.name.capitalize()} tool: {e}"
|
||||
raise RuntimeError(msg) from e
|
||||
|
||||
# Parse this version and return the major/minor parts, since we don't need the
|
||||
# rest.
|
||||
try:
|
||||
major, minor, _ = version.split(".", 3)
|
||||
return (int(major), int(minor))
|
||||
except Exception as e:
|
||||
msg = (
|
||||
f"Could not parse the version of the {runtime.name.capitalize()} tool"
|
||||
f" (found: '{version}') due to the following error: {e}"
|
||||
)
|
||||
raise RuntimeError(msg)
|
||||
|
||||
|
||||
def list_image_tags() -> List[str]:
|
||||
"""Get the tags of all loaded Dangerzone images.
|
||||
|
||||
This method returns a mapping of image tags to image IDs, for all Dangerzone
|
||||
images. This can be useful when we want to find which are the local image tags,
|
||||
and which image ID does the "latest" tag point to.
|
||||
"""
|
||||
runtime = Runtime()
|
||||
return (
|
||||
subprocess.check_output(
|
||||
[
|
||||
str(runtime.path),
|
||||
"image",
|
||||
"list",
|
||||
"--format",
|
||||
"{{ .Tag }}",
|
||||
CONTAINER_NAME,
|
||||
],
|
||||
text=True,
|
||||
startupinfo=get_subprocess_startupinfo(),
|
||||
)
|
||||
.strip()
|
||||
.split()
|
||||
)
|
||||
|
||||
|
||||
def add_image_tag(image_id: str, new_tag: str) -> None:
|
||||
"""Add a tag to the Dangerzone image."""
|
||||
runtime = Runtime()
|
||||
log.debug(f"Adding tag '{new_tag}' to image '{image_id}'")
|
||||
subprocess.check_output(
|
||||
[str(runtime.path), "tag", image_id, new_tag],
|
||||
startupinfo=get_subprocess_startupinfo(),
|
||||
)
|
||||
|
||||
|
||||
def delete_image_tag(tag: str) -> None:
|
||||
"""Delete a Dangerzone image tag."""
|
||||
runtime = Runtime()
|
||||
log.warning(f"Deleting old container image: {tag}")
|
||||
try:
|
||||
subprocess.check_output(
|
||||
[str(runtime.name), "rmi", "--force", tag],
|
||||
startupinfo=get_subprocess_startupinfo(),
|
||||
)
|
||||
except Exception as e:
|
||||
log.warning(
|
||||
f"Couldn't delete old container image '{tag}', so leaving it there."
|
||||
f" Original error: {e}"
|
||||
)
|
||||
|
||||
|
||||
def get_expected_tag() -> str:
|
||||
"""Get the tag of the Dangerzone image tarball from the image-id.txt file."""
|
||||
with get_resource_path("image-id.txt").open() as f:
|
||||
return f.read().strip()
|
||||
|
||||
|
||||
def load_image_tarball() -> None:
|
||||
runtime = Runtime()
|
||||
log.info("Installing Dangerzone container image...")
|
||||
tarball_path = get_resource_path("container.tar")
|
||||
try:
|
||||
res = subprocess.run(
|
||||
[str(runtime.path), "load", "-i", str(tarball_path)],
|
||||
startupinfo=get_subprocess_startupinfo(),
|
||||
capture_output=True,
|
||||
check=True,
|
||||
)
|
||||
except subprocess.CalledProcessError as e:
|
||||
if e.stderr:
|
||||
error = e.stderr.decode()
|
||||
else:
|
||||
error = "No output"
|
||||
raise errors.ImageInstallationException(
|
||||
f"Could not install container image: {error}"
|
||||
)
|
||||
|
||||
# Loading an image built with Buildkit in Podman 3.4 messes up its name. The tag
|
||||
# somehow becomes the name of the loaded image [1].
|
||||
#
|
||||
# We know that older Podman versions are not generally affected, since Podman v3.0.1
|
||||
# on Debian Bullseye works properly. Also, Podman v4.0 is not affected, so it makes
|
||||
# sense to target only Podman v3.4 for a fix.
|
||||
#
|
||||
# The fix is simple, tag the image properly based on the expected tag from
|
||||
# `share/image-id.txt` and delete the incorrect tag.
|
||||
#
|
||||
# [1] https://github.com/containers/podman/issues/16490
|
||||
if runtime.name == "podman" and get_runtime_version(runtime) == (3, 4):
|
||||
expected_tag = get_expected_tag()
|
||||
bad_tag = f"localhost/{expected_tag}:latest"
|
||||
good_tag = f"{CONTAINER_NAME}:{expected_tag}"
|
||||
|
||||
log.debug(
|
||||
f"Dangerzone images loaded in Podman v3.4 usually have an invalid tag."
|
||||
" Fixing it..."
|
||||
)
|
||||
add_image_tag(bad_tag, good_tag)
|
||||
delete_image_tag(bad_tag)
|
||||
|
||||
log.info("Successfully installed container image")
|
||||
|
|
@ -1,8 +1,16 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
import asyncio
|
||||
import glob
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from abc import abstractmethod
|
||||
from typing import Callable, List, Optional, TextIO, Tuple
|
||||
from typing import Callable, Dict, List, Optional, TextIO, Tuple, Union
|
||||
|
||||
DEFAULT_DPI = 150 # Pixels per inch
|
||||
INT_BYTES = 2
|
||||
|
|
@ -13,6 +21,13 @@ def running_on_qubes() -> bool:
|
|||
return os.path.exists("/usr/share/qubes/marker-vm")
|
||||
|
||||
|
||||
def get_tessdata_dir() -> str:
|
||||
if running_on_qubes():
|
||||
return "/usr/share/tesseract/tessdata/"
|
||||
else:
|
||||
return "/usr/share/tessdata/"
|
||||
|
||||
|
||||
class DangerzoneConverter:
|
||||
def __init__(self, progress_callback: Optional[Callable] = None) -> None:
|
||||
self.percentage: float = 0.0
|
||||
|
|
|
|||
|
|
@ -1,18 +1,11 @@
|
|||
#!/usr/bin/env python3
|
||||
import asyncio
|
||||
import glob
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
from typing import Dict, Optional
|
||||
|
||||
# XXX: PyMUPDF logs to stdout by default [1]. The PyMuPDF devs provide a way [2] to log to
|
||||
# stderr, but it's based on environment variables. These envvars are consulted at import
|
||||
# time [3], so we have to set them here, before we import `fitz`.
|
||||
#
|
||||
# [1] https://github.com/freedomofpress/dangerzone/issues/877
|
||||
# [2] https://github.com/pymupdf/PyMuPDF/issues/3135#issuecomment-1992625724
|
||||
# [3] https://github.com/pymupdf/PyMuPDF/blob/9717935eeb2d50d15440d62575878214226795f9/src/__init__.py#L62-L63
|
||||
os.environ["PYMUPDF_MESSAGE"] = "fd:2"
|
||||
os.environ["PYMUPDF_LOG"] = "fd:2"
|
||||
|
||||
from typing import Dict, List, Optional, TextIO
|
||||
|
||||
import fitz
|
||||
import magic
|
||||
|
|
@ -129,10 +122,6 @@ class DocumentToPixels(DangerzoneConverter):
|
|||
# At least .odt, .docx, .odg, .odp, .ods, and .pptx
|
||||
"application/zip": {
|
||||
"type": "libreoffice",
|
||||
# NOTE: `file` command < 5.45 cannot detect hwpx files properly, so we
|
||||
# enable the extension in any case. See also:
|
||||
# https://github.com/freedomofpress/dangerzone/pull/460#issuecomment-1654166465
|
||||
"libreoffice_ext": "h2orestart.oxt",
|
||||
},
|
||||
# At least .doc, .docx, .odg, .odp, .odt, .pdf, .ppt, .pptx, .xls, and .xlsx
|
||||
"application/octet-stream": {
|
||||
|
|
@ -232,6 +221,7 @@ class DocumentToPixels(DangerzoneConverter):
|
|||
raise errors.MaxPagesException()
|
||||
await self.write_page_count(doc.page_count)
|
||||
|
||||
page_base = "/tmp/page"
|
||||
for page in doc.pages():
|
||||
# TODO check if page.number is doc-controlled
|
||||
page_num = page.number + 1 # pages start in 1
|
||||
|
|
@ -253,7 +243,7 @@ class DocumentToPixels(DangerzoneConverter):
|
|||
"unzip",
|
||||
"-d",
|
||||
f"/usr/lib/libreoffice/share/extensions/{libreoffice_ext}/",
|
||||
f"/opt/libreoffice_ext/{libreoffice_ext}",
|
||||
f"/libreoffice_ext/{libreoffice_ext}",
|
||||
]
|
||||
await self.run_command(
|
||||
unzip_args,
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import subprocess
|
||||
from typing import List, Optional, Type, Union
|
||||
|
||||
# XXX: errors start at 128 for conversion-related issues
|
||||
|
|
@ -78,12 +79,12 @@ class MaxPagesException(PagesException):
|
|||
|
||||
class MaxPageWidthException(PagesException):
|
||||
error_code = ERROR_SHIFT + 44
|
||||
error_message = "A page exceeded the maximum width."
|
||||
error_message = f"A page exceeded the maximum width."
|
||||
|
||||
|
||||
class MaxPageHeightException(PagesException):
|
||||
error_code = ERROR_SHIFT + 45
|
||||
error_message = "A page exceeded the maximum height."
|
||||
error_message = f"A page exceeded the maximum height."
|
||||
|
||||
|
||||
class PageCountMismatch(PagesException):
|
||||
|
|
|
|||
153
dangerzone/conversion/pixels_to_pdf.py
Normal file
153
dangerzone/conversion/pixels_to_pdf.py
Normal file
|
|
@ -0,0 +1,153 @@
|
|||
#!/usr/bin/env python3
|
||||
"""
|
||||
Here are the steps, with progress bar percentages:
|
||||
|
||||
- 50%-95%: Convert each page of pixels into a PDF (each page takes 45/n%, where n is the number of pages)
|
||||
- 95%-100%: Compress the final PDF
|
||||
"""
|
||||
import asyncio
|
||||
import contextlib
|
||||
import glob
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
from typing import Optional
|
||||
|
||||
from .common import DEFAULT_DPI, DangerzoneConverter, get_tessdata_dir, running_on_qubes
|
||||
|
||||
|
||||
class PixelsToPDF(DangerzoneConverter):
|
||||
async def convert(
|
||||
self, ocr_lang: Optional[str] = None, tempdir: Optional[str] = None
|
||||
) -> None:
|
||||
self.percentage = 50.0
|
||||
if tempdir is None:
|
||||
tempdir = "/safezone"
|
||||
|
||||
# XXX lazy loading of fitz module to avoid import issues on non-Qubes systems
|
||||
import fitz
|
||||
|
||||
num_pages = len(glob.glob(f"{tempdir}/pixels/page-*.rgb"))
|
||||
total_size = 0.0
|
||||
|
||||
safe_doc = fitz.Document()
|
||||
|
||||
# Convert RGB files to PDF files
|
||||
percentage_per_page = 45.0 / num_pages
|
||||
for page_num in range(1, num_pages + 1):
|
||||
filename_base = f"{tempdir}/pixels/page-{page_num}"
|
||||
rgb_filename = f"{filename_base}.rgb"
|
||||
width_filename = f"{filename_base}.width"
|
||||
height_filename = f"{filename_base}.height"
|
||||
|
||||
with open(width_filename) as f:
|
||||
width = int(f.read().strip())
|
||||
with open(height_filename) as f:
|
||||
height = int(f.read().strip())
|
||||
with open(rgb_filename, "rb") as rgb_f:
|
||||
untrusted_rgb_data = rgb_f.read()
|
||||
# The first few operations happen on a per-page basis.
|
||||
page_size = len(untrusted_rgb_data)
|
||||
total_size += page_size
|
||||
with contextlib.redirect_stdout(io.StringIO()):
|
||||
pixmap = fitz.Pixmap(
|
||||
fitz.Colorspace(fitz.CS_RGB),
|
||||
width,
|
||||
height,
|
||||
untrusted_rgb_data,
|
||||
False,
|
||||
)
|
||||
pixmap.set_dpi(DEFAULT_DPI, DEFAULT_DPI)
|
||||
if ocr_lang: # OCR the document
|
||||
self.update_progress(
|
||||
f"Converting page {page_num}/{num_pages} from pixels to searchable PDF"
|
||||
)
|
||||
if int(fitz.version[2]) >= 20230621000001:
|
||||
page_pdf_bytes = pixmap.pdfocr_tobytes(
|
||||
compress=True,
|
||||
language=ocr_lang,
|
||||
tessdata=get_tessdata_dir(),
|
||||
)
|
||||
else:
|
||||
# XXX: In PyMuPDF v1.22.5, the function signature of
|
||||
# `pdfocr_tobytes()` / `pdfocr_save()` was extended with an argument
|
||||
# to explicitly set the Tesseract data dir [1].
|
||||
#
|
||||
# In earlier versions, the PyMuPDF developers recommend setting this
|
||||
# path via the TESSDATA_PREFIX environment variable. In practice,
|
||||
# this environment variable is read at import time, so subsequent
|
||||
# changes to the environment variable are not tracked [2].
|
||||
#
|
||||
# To make things worse, any attempt to alter the internal attribute
|
||||
# (`fitz.TESSDATA_PREFIX`) makes no difference as well, when using
|
||||
# the OCR functions. That's due to the way imports work in `fitz`,
|
||||
# where somehow the internal `fitz.fitz` module is shadowed.
|
||||
#
|
||||
# A hacky solution is to grab the `fitz.fitz` module from
|
||||
# `sys.modules`, and set there the TESSDATA_PREFIX variable. We can
|
||||
# get away with this hack because we have a proper solution for
|
||||
# subsequent PyMuPDF versions, and we know that nothing will change
|
||||
# in older versions.
|
||||
#
|
||||
# TODO: Remove after oldest distro has PyMuPDF >= v1.22.5
|
||||
#
|
||||
# [1]: https://pymupdf.readthedocs.io/en/latest/pixmap.html#Pixmap.pdfocr_save
|
||||
# [2]: https://github.com/pymupdf/PyMuPDF/blob/0368e56cfa6afb55bcf6c726e7f51a2a16a5ccba/fitz/fitz.i#L308
|
||||
sys.modules["fitz.fitz"].TESSDATA_PREFIX = get_tessdata_dir() # type: ignore [attr-defined]
|
||||
|
||||
page_pdf_bytes = pixmap.pdfocr_tobytes(
|
||||
compress=True,
|
||||
language=ocr_lang,
|
||||
)
|
||||
ocr_pdf = fitz.open("pdf", page_pdf_bytes)
|
||||
else: # Don't OCR
|
||||
self.update_progress(
|
||||
f"Converting page {page_num}/{num_pages} from pixels to PDF"
|
||||
)
|
||||
page_doc = fitz.Document()
|
||||
page_doc.insert_file(pixmap)
|
||||
page_pdf_bytes = page_doc.tobytes(deflate_images=True)
|
||||
|
||||
safe_doc.insert_pdf(fitz.open("pdf", page_pdf_bytes))
|
||||
self.percentage += percentage_per_page
|
||||
|
||||
self.percentage = 100.0
|
||||
self.update_progress("Safe PDF created")
|
||||
|
||||
# Move converted files into /safezone
|
||||
if running_on_qubes():
|
||||
safe_pdf_path = f"{tempdir}/safe-output-compressed.pdf"
|
||||
else:
|
||||
safe_pdf_path = f"/safezone/safe-output-compressed.pdf"
|
||||
|
||||
safe_doc.save(safe_pdf_path, deflate_images=True)
|
||||
|
||||
def update_progress(self, text: str, *, error: bool = False) -> None:
|
||||
if running_on_qubes():
|
||||
if self.progress_callback:
|
||||
self.progress_callback(error, text, self.percentage)
|
||||
else:
|
||||
print(
|
||||
json.dumps(
|
||||
{"error": error, "text": text, "percentage": self.percentage}
|
||||
)
|
||||
)
|
||||
sys.stdout.flush()
|
||||
|
||||
|
||||
async def main() -> int:
|
||||
ocr_lang = os.environ.get("OCR_LANGUAGE") if os.environ.get("OCR") == "1" else None
|
||||
converter = PixelsToPDF()
|
||||
|
||||
try:
|
||||
await converter.convert(ocr_lang)
|
||||
return 0
|
||||
except (RuntimeError, ValueError) as e:
|
||||
converter.update_progress(str(e), error=True)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(asyncio.run(main()))
|
||||
|
|
@ -2,11 +2,14 @@ import enum
|
|||
import logging
|
||||
import os
|
||||
import platform
|
||||
import re
|
||||
import secrets
|
||||
from pathlib import Path, PurePosixPath, PureWindowsPath
|
||||
import stat
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
import appdirs
|
||||
|
||||
from . import errors, util
|
||||
|
||||
SAFE_EXTENSION = "-safe.pdf"
|
||||
|
|
@ -70,20 +73,6 @@ class Document:
|
|||
def validate_output_filename(filename: str) -> None:
|
||||
if not filename.endswith(".pdf"):
|
||||
raise errors.NonPDFOutputFileException()
|
||||
|
||||
if platform.system() == "Windows":
|
||||
final_filename = PureWindowsPath(filename).name
|
||||
illegal_chars_regex = re.compile(r"[\"*/:<>?\\|]")
|
||||
else:
|
||||
final_filename = PurePosixPath(filename).name
|
||||
illegal_chars_regex = re.compile(r"[\\]")
|
||||
|
||||
if platform.system() in ("Windows", "Darwin"):
|
||||
match = illegal_chars_regex.search(final_filename)
|
||||
if match:
|
||||
# filename contains illegal characters
|
||||
raise errors.IllegalOutputFilenameException(match.group(0))
|
||||
|
||||
if not os.access(Path(filename).parent, os.W_OK):
|
||||
# in unwriteable directory
|
||||
raise errors.UnwriteableOutputDirException()
|
||||
|
|
@ -123,10 +112,6 @@ class Document:
|
|||
self.validate_output_filename(filename)
|
||||
self._output_filename = filename
|
||||
|
||||
@property
|
||||
def sanitized_output_filename(self) -> str:
|
||||
return util.replace_control_chars(self.output_filename)
|
||||
|
||||
@property
|
||||
def suffix(self) -> str:
|
||||
return self._suffix
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import functools
|
||||
import logging
|
||||
import sys
|
||||
from typing import Any, Callable, TypeVar, cast
|
||||
from typing import Any, Callable, Sequence, TypeVar, cast
|
||||
|
||||
import click
|
||||
|
||||
|
|
@ -42,13 +42,6 @@ class NonPDFOutputFileException(DocumentFilenameException):
|
|||
super().__init__("Safe PDF filename must end in '.pdf'")
|
||||
|
||||
|
||||
class IllegalOutputFilenameException(DocumentFilenameException):
|
||||
"""Exception for when the output file contains illegal characters."""
|
||||
|
||||
def __init__(self, char: str) -> None:
|
||||
super().__init__(f"Illegal character: {char}")
|
||||
|
||||
|
||||
class UnwriteableOutputDirException(DocumentFilenameException):
|
||||
"""Exception for when the output file is not writeable."""
|
||||
|
||||
|
|
@ -102,7 +95,7 @@ class SuffixNotApplicableException(DocumentFilenameException):
|
|||
|
||||
|
||||
def handle_document_errors(func: F) -> F:
|
||||
"""Decorator to log document-related errors and exit gracefully."""
|
||||
"""Log document-related errors and exit gracefully."""
|
||||
|
||||
@functools.wraps(func)
|
||||
def wrapper(*args, **kwargs): # type: ignore
|
||||
|
|
@ -117,30 +110,3 @@ def handle_document_errors(func: F) -> F:
|
|||
sys.exit(1)
|
||||
|
||||
return cast(F, wrapper)
|
||||
|
||||
|
||||
#### Container-related errors
|
||||
|
||||
|
||||
class ImageNotPresentException(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class ImageInstallationException(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class NoContainerTechException(Exception):
|
||||
def __init__(self, container_tech: str) -> None:
|
||||
super().__init__(f"{container_tech} is not installed")
|
||||
|
||||
|
||||
class NotAvailableContainerTechException(Exception):
|
||||
def __init__(self, container_tech: str, error: str) -> None:
|
||||
self.error = error
|
||||
self.container_tech = container_tech
|
||||
super().__init__(f"{container_tech} is not available")
|
||||
|
||||
|
||||
class UnsupportedContainerRuntime(Exception):
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -1,11 +1,13 @@
|
|||
import enum
|
||||
import functools
|
||||
import logging
|
||||
import os
|
||||
import platform
|
||||
import signal
|
||||
import sys
|
||||
import typing
|
||||
from typing import List, Optional
|
||||
import uuid
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
import click
|
||||
import colorama
|
||||
|
|
@ -51,7 +53,7 @@ class Application(QtWidgets.QApplication):
|
|||
def __init__(self, *args: typing.Any, **kwargs: typing.Any) -> None:
|
||||
super(Application, self).__init__(*args, **kwargs)
|
||||
self.setQuitOnLastWindowClosed(False)
|
||||
with get_resource_path("dangerzone.css").open("r") as f:
|
||||
with open(get_resource_path("dangerzone.css"), "r") as f:
|
||||
style = f.read()
|
||||
self.setStyleSheet(style)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,14 +1,11 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import platform
|
||||
import shlex
|
||||
import subprocess
|
||||
import typing
|
||||
from collections import OrderedDict
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
from typing import Dict, Optional
|
||||
|
||||
from colorama import Fore
|
||||
|
||||
|
|
@ -24,10 +21,11 @@ else:
|
|||
from PySide2 import QtCore, QtGui, QtWidgets
|
||||
|
||||
if platform.system() == "Linux":
|
||||
from xdg.DesktopEntry import DesktopEntry, ParsingError
|
||||
from xdg.DesktopEntry import DesktopEntry
|
||||
|
||||
from ..isolation_provider.base import IsolationProvider
|
||||
from ..logic import DangerzoneCore
|
||||
from ..settings import Settings
|
||||
from ..util import get_resource_path, replace_control_chars
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
|
@ -52,7 +50,7 @@ class DangerzoneGui(DangerzoneCore):
|
|||
# Preload font
|
||||
self.fixed_font = QtGui.QFontDatabase.systemFont(QtGui.QFontDatabase.FixedFont)
|
||||
|
||||
# Preload ordered list of PDF viewers on computer, starting with default
|
||||
# Preload list of PDF viewers on computer
|
||||
self.pdf_viewers = self._find_pdf_viewers()
|
||||
|
||||
# Are we done waiting (for Docker Desktop to be installed, or for container to install)
|
||||
|
|
@ -63,7 +61,7 @@ class DangerzoneGui(DangerzoneCore):
|
|||
path = get_resource_path("dangerzone.ico")
|
||||
else:
|
||||
path = get_resource_path("icon.png")
|
||||
return QtGui.QIcon(str(path))
|
||||
return QtGui.QIcon(path)
|
||||
|
||||
def open_pdf_viewer(self, filename: str) -> None:
|
||||
if platform.system() == "Darwin":
|
||||
|
|
@ -96,22 +94,9 @@ class DangerzoneGui(DangerzoneCore):
|
|||
log.info(Fore.YELLOW + "> " + Fore.CYAN + args_str)
|
||||
subprocess.Popen(args)
|
||||
|
||||
def _find_pdf_viewers(self) -> OrderedDict[str, str]:
|
||||
pdf_viewers: OrderedDict[str, str] = OrderedDict()
|
||||
def _find_pdf_viewers(self) -> Dict[str, str]:
|
||||
pdf_viewers: Dict[str, str] = {}
|
||||
if platform.system() == "Linux":
|
||||
# Opportunistically query for default pdf handler
|
||||
default_pdf_viewer = None
|
||||
try:
|
||||
default_pdf_viewer = subprocess.check_output(
|
||||
["xdg-mime", "query", "default", "application/pdf"]
|
||||
).decode()
|
||||
except (FileNotFoundError, subprocess.CalledProcessError) as e:
|
||||
# Log it and continue
|
||||
log.info(
|
||||
"xdg-mime query failed, default PDF handler could not be found."
|
||||
)
|
||||
log.debug(f"xdg-mime query failed: {e}")
|
||||
|
||||
# Find all .desktop files
|
||||
for search_path in [
|
||||
"/usr/share/applications",
|
||||
|
|
@ -123,37 +108,15 @@ class DangerzoneGui(DangerzoneCore):
|
|||
full_filename = os.path.join(search_path, filename)
|
||||
if os.path.splitext(filename)[1] == ".desktop":
|
||||
# See which ones can open PDFs
|
||||
try:
|
||||
desktop_entry = DesktopEntry(full_filename)
|
||||
except ParsingError:
|
||||
# Do not stop when encountering malformed desktop entries
|
||||
continue
|
||||
except Exception:
|
||||
log.exception(
|
||||
"Encountered the following exception while processing desktop entry %s",
|
||||
full_filename,
|
||||
desktop_entry = DesktopEntry(full_filename)
|
||||
if (
|
||||
"application/pdf" in desktop_entry.getMimeTypes()
|
||||
and "dangerzone" not in desktop_entry.getName().lower()
|
||||
):
|
||||
pdf_viewers[desktop_entry.getName()] = (
|
||||
desktop_entry.getExec()
|
||||
)
|
||||
else:
|
||||
desktop_entry_name = desktop_entry.getName()
|
||||
if (
|
||||
"application/pdf" in desktop_entry.getMimeTypes()
|
||||
and "dangerzone" not in desktop_entry_name.lower()
|
||||
):
|
||||
pdf_viewers[desktop_entry_name] = (
|
||||
desktop_entry.getExec()
|
||||
)
|
||||
|
||||
# Put the default entry first
|
||||
if filename == default_pdf_viewer:
|
||||
try:
|
||||
pdf_viewers.move_to_end(
|
||||
desktop_entry_name, last=False
|
||||
)
|
||||
except KeyError as e:
|
||||
# Should be unreachable
|
||||
log.error(
|
||||
f"Problem reordering applications: {e}"
|
||||
)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
|
||||
|
|
@ -235,7 +198,7 @@ class Dialog(QtWidgets.QDialog):
|
|||
self.done(int(QtWidgets.QDialog.Rejected))
|
||||
|
||||
def launch(self) -> int:
|
||||
return self.exec()
|
||||
return self.exec_()
|
||||
|
||||
|
||||
class Alert(Dialog):
|
||||
|
|
@ -252,7 +215,7 @@ class Alert(Dialog):
|
|||
def create_layout(self) -> QtWidgets.QBoxLayout:
|
||||
logo = QtWidgets.QLabel()
|
||||
logo.setPixmap(
|
||||
QtGui.QPixmap.fromImage(QtGui.QImage(str(get_resource_path("icon.png"))))
|
||||
QtGui.QPixmap.fromImage(QtGui.QImage(get_resource_path("icon.png")))
|
||||
)
|
||||
|
||||
label = QtWidgets.QLabel()
|
||||
|
|
|
|||
|
|
@ -1,32 +1,31 @@
|
|||
import json
|
||||
import logging
|
||||
import os
|
||||
import platform
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
import typing
|
||||
from multiprocessing.pool import ThreadPool
|
||||
from pathlib import Path
|
||||
from typing import List, Optional
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
from colorama import Fore, Style
|
||||
|
||||
# FIXME: See https://github.com/freedomofpress/dangerzone/issues/320 for more details.
|
||||
if typing.TYPE_CHECKING:
|
||||
from PySide2 import QtCore, QtGui, QtSvg, QtWidgets
|
||||
from PySide2.QtCore import Qt
|
||||
from PySide2.QtWidgets import QAction, QTextEdit
|
||||
else:
|
||||
try:
|
||||
from PySide6 import QtCore, QtGui, QtSvg, QtWidgets
|
||||
from PySide6.QtCore import Qt
|
||||
from PySide6.QtGui import QAction
|
||||
from PySide6.QtWidgets import QTextEdit
|
||||
except ImportError:
|
||||
from PySide2 import QtCore, QtGui, QtSvg, QtWidgets
|
||||
from PySide2.QtCore import Qt
|
||||
from PySide2.QtWidgets import QAction, QTextEdit
|
||||
|
||||
from .. import errors
|
||||
from ..document import SAFE_EXTENSION, Document
|
||||
from ..isolation_provider.qubes import is_qubes_native_conversion
|
||||
from ..util import format_exception, get_resource_path, get_version
|
||||
from ..isolation_provider.container import Container, NoContainerTechException
|
||||
from ..isolation_provider.dummy import Dummy
|
||||
from ..isolation_provider.qubes import Qubes, is_qubes_native_conversion
|
||||
from ..util import get_resource_path, get_subprocess_startupinfo, get_version
|
||||
from .logic import Alert, CollapsibleBox, DangerzoneGui, UpdateDialog
|
||||
from .updater import UpdateReport
|
||||
|
||||
|
|
@ -55,60 +54,6 @@ about updates.</p>
|
|||
HAMBURGER_MENU_SIZE = 30
|
||||
|
||||
|
||||
def load_svg_image(filename: str, width: int, height: int) -> QtGui.QPixmap:
|
||||
"""Load an SVG image from a filename.
|
||||
|
||||
This answer is basically taken from: https://stackoverflow.com/a/25689790
|
||||
"""
|
||||
path = get_resource_path(filename)
|
||||
svg_renderer = QtSvg.QSvgRenderer(str(path))
|
||||
image = QtGui.QImage(width, height, QtGui.QImage.Format_ARGB32)
|
||||
# Set the ARGB to 0 to prevent rendering artifacts
|
||||
image.fill(0x00000000)
|
||||
svg_renderer.render(QtGui.QPainter(image))
|
||||
pixmap = QtGui.QPixmap.fromImage(image)
|
||||
return pixmap
|
||||
|
||||
|
||||
def get_supported_extensions() -> List[str]:
|
||||
supported_ext = [
|
||||
".pdf",
|
||||
".docx",
|
||||
".doc",
|
||||
".docm",
|
||||
".xlsx",
|
||||
".xls",
|
||||
".pptx",
|
||||
".ppt",
|
||||
".odt",
|
||||
".odg",
|
||||
".odp",
|
||||
".ods",
|
||||
".epub",
|
||||
".jpg",
|
||||
".jpeg",
|
||||
".gif",
|
||||
".png",
|
||||
".tif",
|
||||
".tiff",
|
||||
".bmp",
|
||||
".pnm",
|
||||
".pbm",
|
||||
".ppm",
|
||||
".svg",
|
||||
]
|
||||
|
||||
# XXX: We disable loading HWP/HWPX files on Qubes, because H2ORestart does not work there.
|
||||
# See:
|
||||
#
|
||||
# https://github.com/freedomofpress/dangerzone/issues/494
|
||||
hwp_filters = [".hwp", ".hwpx"]
|
||||
if is_qubes_native_conversion():
|
||||
supported_ext += hwp_filters
|
||||
|
||||
return supported_ext
|
||||
|
||||
|
||||
class MainWindow(QtWidgets.QMainWindow):
|
||||
def __init__(self, dangerzone: DangerzoneGui) -> None:
|
||||
super(MainWindow, self).__init__()
|
||||
|
|
@ -117,7 +62,6 @@ class MainWindow(QtWidgets.QMainWindow):
|
|||
|
||||
self.setWindowTitle("Dangerzone")
|
||||
self.setWindowIcon(self.dangerzone.get_window_icon())
|
||||
self.alert: Optional[Alert] = None
|
||||
|
||||
self.setMinimumWidth(600)
|
||||
if platform.system() == "Darwin":
|
||||
|
|
@ -129,8 +73,9 @@ class MainWindow(QtWidgets.QMainWindow):
|
|||
|
||||
# Header
|
||||
logo = QtWidgets.QLabel()
|
||||
icon_path = str(get_resource_path("icon.png"))
|
||||
logo.setPixmap(QtGui.QPixmap.fromImage(QtGui.QImage(icon_path)))
|
||||
logo.setPixmap(
|
||||
QtGui.QPixmap.fromImage(QtGui.QImage(get_resource_path("icon.png")))
|
||||
)
|
||||
header_label = QtWidgets.QLabel("Dangerzone")
|
||||
header_label.setFont(self.dangerzone.fixed_font)
|
||||
header_label.setStyleSheet("QLabel { font-weight: bold; font-size: 50px; }")
|
||||
|
|
@ -143,7 +88,7 @@ class MainWindow(QtWidgets.QMainWindow):
|
|||
self.hamburger_button = QtWidgets.QToolButton()
|
||||
self.hamburger_button.setPopupMode(QtWidgets.QToolButton.InstantPopup)
|
||||
self.hamburger_button.setIcon(
|
||||
QtGui.QIcon(load_svg_image("hamburger_menu.svg", width=64, height=64))
|
||||
QtGui.QIcon(self.load_svg_image("hamburger_menu.svg"))
|
||||
)
|
||||
self.hamburger_button.setFixedSize(HAMBURGER_MENU_SIZE, HAMBURGER_MENU_SIZE)
|
||||
self.hamburger_button.setIconSize(
|
||||
|
|
@ -184,18 +129,21 @@ class MainWindow(QtWidgets.QMainWindow):
|
|||
header_layout.addWidget(self.hamburger_button)
|
||||
header_layout.addSpacing(15)
|
||||
|
||||
# Content widget, contains all the window content except waiting widget
|
||||
self.content_widget = ContentWidget(self.dangerzone)
|
||||
|
||||
if self.dangerzone.isolation_provider.should_wait_install():
|
||||
if isinstance(self.dangerzone.isolation_provider, Container):
|
||||
# Waiting widget replaces content widget while container runtime isn't available
|
||||
self.waiting_widget: WaitingWidget = WaitingWidgetContainer(self.dangerzone)
|
||||
self.waiting_widget.finished.connect(self.waiting_finished)
|
||||
else:
|
||||
|
||||
elif isinstance(self.dangerzone.isolation_provider, Dummy) or isinstance(
|
||||
self.dangerzone.isolation_provider, Qubes
|
||||
):
|
||||
# Don't wait with dummy converter and on Qubes.
|
||||
self.waiting_widget = WaitingWidget()
|
||||
self.dangerzone.is_waiting_finished = True
|
||||
|
||||
# Content widget, contains all the window content except waiting widget
|
||||
self.content_widget = ContentWidget(self.dangerzone)
|
||||
|
||||
# Only use the waiting widget if container runtime isn't available
|
||||
if self.dangerzone.is_waiting_finished:
|
||||
self.waiting_widget.hide()
|
||||
|
|
@ -219,20 +167,22 @@ class MainWindow(QtWidgets.QMainWindow):
|
|||
# This allows us to make QSS rules conditional on the OS color mode.
|
||||
self.setProperty("OSColorMode", self.dangerzone.app.os_color_mode.value)
|
||||
|
||||
if hasattr(self.dangerzone.isolation_provider, "check_docker_desktop_version"):
|
||||
try:
|
||||
is_version_valid, version = (
|
||||
self.dangerzone.isolation_provider.check_docker_desktop_version()
|
||||
)
|
||||
if not is_version_valid:
|
||||
self.handle_docker_desktop_version_check(is_version_valid, version)
|
||||
except errors.UnsupportedContainerRuntime as e:
|
||||
pass # It's caught later in the flow.
|
||||
except errors.NoContainerTechException as e:
|
||||
pass # It's caught later in the flow.
|
||||
|
||||
self.show()
|
||||
|
||||
def load_svg_image(self, filename: str) -> QtGui.QPixmap:
|
||||
"""Load an SVG image from a filename.
|
||||
|
||||
This answer is basically taken from: https://stackoverflow.com/a/25689790
|
||||
"""
|
||||
path = get_resource_path(filename)
|
||||
svg_renderer = QtSvg.QSvgRenderer(path)
|
||||
image = QtGui.QImage(64, 64, QtGui.QImage.Format_ARGB32)
|
||||
# Set the ARGB to 0 to prevent rendering artifacts
|
||||
image.fill(0x00000000)
|
||||
svg_renderer.render(QtGui.QPainter(image))
|
||||
pixmap = QtGui.QPixmap.fromImage(image)
|
||||
return pixmap
|
||||
|
||||
def show_update_success(self) -> None:
|
||||
"""Inform the user about a new Dangerzone release."""
|
||||
version = self.dangerzone.settings.get("updater_latest_version")
|
||||
|
|
@ -255,7 +205,7 @@ class MainWindow(QtWidgets.QMainWindow):
|
|||
ok_text="Ok",
|
||||
has_cancel=False,
|
||||
)
|
||||
update_widget.launch()
|
||||
update_widget.exec_()
|
||||
|
||||
def show_update_error(self) -> None:
|
||||
"""Inform the user about an error during update checks"""
|
||||
|
|
@ -276,7 +226,7 @@ class MainWindow(QtWidgets.QMainWindow):
|
|||
ok_text="Close",
|
||||
has_cancel=False,
|
||||
)
|
||||
update_widget.launch()
|
||||
update_widget.exec_()
|
||||
|
||||
def toggle_updates_triggered(self) -> None:
|
||||
"""Change the underlying update check settings based on the user's choice."""
|
||||
|
|
@ -284,46 +234,6 @@ class MainWindow(QtWidgets.QMainWindow):
|
|||
self.dangerzone.settings.set("updater_check", check)
|
||||
self.dangerzone.settings.save()
|
||||
|
||||
def handle_docker_desktop_version_check(
|
||||
self, is_version_valid: bool, version: str
|
||||
) -> None:
|
||||
hamburger_menu = self.hamburger_button.menu()
|
||||
sep = hamburger_menu.insertSeparator(hamburger_menu.actions()[0])
|
||||
upgrade_action = QAction("Docker Desktop should be upgraded", hamburger_menu)
|
||||
upgrade_action.setIcon(
|
||||
QtGui.QIcon(
|
||||
load_svg_image(
|
||||
"hamburger_menu_update_dot_error.svg", width=64, height=64
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
message = """
|
||||
<p>A new version of Docker Desktop is available. Please upgrade your system.</p>
|
||||
<p>Visit the <a href="https://www.docker.com/products/docker-desktop">Docker Desktop website</a> to download the latest version.</p>
|
||||
<em>Keeping Docker Desktop up to date allows you to have more confidence that your documents are processed safely.</em>
|
||||
"""
|
||||
self.alert = Alert(
|
||||
self.dangerzone,
|
||||
title="Upgrade Docker Desktop",
|
||||
message=message,
|
||||
ok_text="Ok",
|
||||
has_cancel=False,
|
||||
)
|
||||
|
||||
def _launch_alert() -> None:
|
||||
if self.alert:
|
||||
self.alert.launch()
|
||||
|
||||
upgrade_action.triggered.connect(_launch_alert)
|
||||
hamburger_menu.insertAction(sep, upgrade_action)
|
||||
|
||||
self.hamburger_button.setIcon(
|
||||
QtGui.QIcon(
|
||||
load_svg_image("hamburger_menu_update_error.svg", width=64, height=64)
|
||||
)
|
||||
)
|
||||
|
||||
def handle_updates(self, report: UpdateReport) -> None:
|
||||
"""Handle update reports from the update checker thread.
|
||||
|
||||
|
|
@ -353,21 +263,13 @@ class MainWindow(QtWidgets.QMainWindow):
|
|||
return
|
||||
|
||||
self.hamburger_button.setIcon(
|
||||
QtGui.QIcon(
|
||||
load_svg_image(
|
||||
"hamburger_menu_update_error.svg", width=64, height=64
|
||||
)
|
||||
)
|
||||
QtGui.QIcon(self.load_svg_image("hamburger_menu_update_error.svg"))
|
||||
)
|
||||
sep = hamburger_menu.insertSeparator(hamburger_menu.actions()[0])
|
||||
# FIXME: Add red bubble next to the text.
|
||||
error_action = QAction("Update error", hamburger_menu)
|
||||
error_action = QtGui.QAction("Update error", hamburger_menu) # type: ignore [attr-defined]
|
||||
error_action.setIcon(
|
||||
QtGui.QIcon(
|
||||
load_svg_image(
|
||||
"hamburger_menu_update_dot_error.svg", width=64, height=64
|
||||
)
|
||||
)
|
||||
QtGui.QIcon(self.load_svg_image("hamburger_menu_update_dot_error.svg"))
|
||||
)
|
||||
error_action.triggered.connect(self.show_update_error)
|
||||
hamburger_menu.insertAction(sep, error_action)
|
||||
|
|
@ -382,20 +284,14 @@ class MainWindow(QtWidgets.QMainWindow):
|
|||
self.dangerzone.settings.save()
|
||||
|
||||
self.hamburger_button.setIcon(
|
||||
QtGui.QIcon(
|
||||
load_svg_image(
|
||||
"hamburger_menu_update_success.svg", width=64, height=64
|
||||
)
|
||||
)
|
||||
QtGui.QIcon(self.load_svg_image("hamburger_menu_update_success.svg"))
|
||||
)
|
||||
|
||||
sep = hamburger_menu.insertSeparator(hamburger_menu.actions()[0])
|
||||
success_action = QAction("New version available", hamburger_menu)
|
||||
success_action = QtGui.QAction("New version available", hamburger_menu) # type: ignore [attr-defined]
|
||||
success_action.setIcon(
|
||||
QtGui.QIcon(
|
||||
load_svg_image(
|
||||
"hamburger_menu_update_dot_available.svg", width=64, height=64
|
||||
)
|
||||
self.load_svg_image("hamburger_menu_update_dot_available.svg")
|
||||
)
|
||||
)
|
||||
success_action.triggered.connect(self.show_update_success)
|
||||
|
|
@ -410,7 +306,7 @@ class MainWindow(QtWidgets.QMainWindow):
|
|||
self.content_widget.show()
|
||||
|
||||
def closeEvent(self, e: QtGui.QCloseEvent) -> None:
|
||||
self.alert = Alert(
|
||||
alert_widget = Alert(
|
||||
self.dangerzone,
|
||||
message="Some documents are still being converted.\n Are you sure you want to quit?",
|
||||
ok_text="Abort conversions",
|
||||
|
|
@ -424,7 +320,7 @@ class MainWindow(QtWidgets.QMainWindow):
|
|||
else:
|
||||
self.dangerzone.app.exit(0)
|
||||
else:
|
||||
accept_exit = self.alert.launch()
|
||||
accept_exit = alert_widget.exec_()
|
||||
if not accept_exit:
|
||||
e.ignore()
|
||||
return
|
||||
|
|
@ -435,24 +331,15 @@ class MainWindow(QtWidgets.QMainWindow):
|
|||
|
||||
|
||||
class InstallContainerThread(QtCore.QThread):
|
||||
finished = QtCore.Signal(str)
|
||||
finished = QtCore.Signal()
|
||||
|
||||
def __init__(self, dangerzone: DangerzoneGui) -> None:
|
||||
super(InstallContainerThread, self).__init__()
|
||||
self.dangerzone = dangerzone
|
||||
|
||||
def run(self) -> None:
|
||||
error = None
|
||||
try:
|
||||
installed = self.dangerzone.isolation_provider.install()
|
||||
except Exception as e:
|
||||
log.error("Container installation problem")
|
||||
error = format_exception(e)
|
||||
else:
|
||||
if not installed:
|
||||
error = "The image cannot be found. This can be caused by a faulty container image."
|
||||
finally:
|
||||
self.finished.emit(error)
|
||||
self.dangerzone.isolation_provider.install()
|
||||
self.finished.emit()
|
||||
|
||||
|
||||
class WaitingWidget(QtWidgets.QWidget):
|
||||
|
|
@ -462,29 +349,6 @@ class WaitingWidget(QtWidgets.QWidget):
|
|||
super(WaitingWidget, self).__init__()
|
||||
|
||||
|
||||
class TracebackWidget(QTextEdit):
|
||||
"""Reusable component to present tracebacks to the user.
|
||||
|
||||
By default, the widget is initialized but does not appear.
|
||||
You need to call `.set_content("traceback")` on it so the
|
||||
traceback is displayed.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
super(TracebackWidget, self).__init__()
|
||||
# Error
|
||||
self.setReadOnly(True)
|
||||
self.setVisible(False)
|
||||
self.setProperty("style", "traceback")
|
||||
# Enable copying
|
||||
self.setTextInteractionFlags(Qt.TextSelectableByMouse)
|
||||
|
||||
def set_content(self, error: Optional[str] = None) -> None:
|
||||
if error:
|
||||
self.setPlainText(error)
|
||||
self.setVisible(True)
|
||||
|
||||
|
||||
class WaitingWidgetContainer(WaitingWidget):
|
||||
# These are the possible states that the WaitingWidget can show.
|
||||
#
|
||||
|
|
@ -495,6 +359,7 @@ class WaitingWidgetContainer(WaitingWidget):
|
|||
#
|
||||
# Linux states
|
||||
# - "install_container"
|
||||
finished = QtCore.Signal()
|
||||
|
||||
def __init__(self, dangerzone: DangerzoneGui) -> None:
|
||||
super(WaitingWidgetContainer, self).__init__()
|
||||
|
|
@ -516,13 +381,10 @@ class WaitingWidgetContainer(WaitingWidget):
|
|||
self.buttons = QtWidgets.QWidget()
|
||||
self.buttons.setLayout(buttons_layout)
|
||||
|
||||
self.traceback = TracebackWidget()
|
||||
|
||||
# Layout
|
||||
layout = QtWidgets.QVBoxLayout()
|
||||
layout.addStretch()
|
||||
layout.addWidget(self.label)
|
||||
layout.addWidget(self.traceback)
|
||||
layout.addStretch()
|
||||
layout.addWidget(self.buttons)
|
||||
layout.addStretch()
|
||||
|
|
@ -533,98 +395,53 @@ class WaitingWidgetContainer(WaitingWidget):
|
|||
|
||||
def check_state(self) -> None:
|
||||
state: Optional[str] = None
|
||||
error: Optional[str] = None
|
||||
|
||||
try:
|
||||
self.dangerzone.isolation_provider.is_available()
|
||||
except errors.NoContainerTechException as e:
|
||||
if isinstance( # Sanity check
|
||||
self.dangerzone.isolation_provider, Container
|
||||
):
|
||||
container_runtime = self.dangerzone.isolation_provider.get_runtime()
|
||||
except NoContainerTechException as e:
|
||||
log.error(str(e))
|
||||
state = "not_installed"
|
||||
except errors.NotAvailableContainerTechException as e:
|
||||
log.error(str(e))
|
||||
state = "not_running"
|
||||
error = e.error
|
||||
except Exception as e:
|
||||
log.error(str(e))
|
||||
state = "not_running"
|
||||
error = format_exception(e)
|
||||
|
||||
else:
|
||||
state = "install_container"
|
||||
# Can we run `docker image ls` without an error
|
||||
with subprocess.Popen(
|
||||
[container_runtime, "image", "ls"],
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
startupinfo=get_subprocess_startupinfo(),
|
||||
) as p:
|
||||
p.communicate()
|
||||
if p.returncode != 0:
|
||||
log.error("Docker is not running")
|
||||
state = "not_running"
|
||||
else:
|
||||
# Always try installing the container
|
||||
state = "install_container"
|
||||
|
||||
# Update the state
|
||||
self.state_change(state, error)
|
||||
|
||||
def show_error(self, msg: str, details: Optional[str] = None) -> None:
|
||||
self.label.setText(msg)
|
||||
show_traceback = details is not None
|
||||
if show_traceback:
|
||||
self.traceback.set_content(details)
|
||||
self.traceback.setVisible(show_traceback)
|
||||
self.buttons.show()
|
||||
|
||||
def show_message(self, msg: str) -> None:
|
||||
self.label.setText(msg)
|
||||
self.traceback.setVisible(False)
|
||||
self.buttons.hide()
|
||||
|
||||
def installation_finished(self, error: Optional[str] = None) -> None:
|
||||
if error:
|
||||
msg = (
|
||||
"During installation of the dangerzone image, <br>"
|
||||
"the following error occured:"
|
||||
)
|
||||
self.show_error(msg, error)
|
||||
else:
|
||||
self.finished.emit()
|
||||
|
||||
def state_change(self, state: str, error: Optional[str] = None) -> None:
|
||||
custom_runtime = self.dangerzone.settings.custom_runtime_specified()
|
||||
self.state_change(state)
|
||||
|
||||
def state_change(self, state: str) -> None:
|
||||
if state == "not_installed":
|
||||
if custom_runtime:
|
||||
self.show_error(
|
||||
"<strong>We could not find the container runtime defined in your settings</strong><br><br>"
|
||||
"Please check your settings, install it if needed, and retry."
|
||||
)
|
||||
elif platform.system() == "Linux":
|
||||
self.show_error(
|
||||
"<strong>Dangerzone requires Podman</strong><br><br>"
|
||||
"Install it and retry."
|
||||
)
|
||||
else:
|
||||
self.show_error(
|
||||
"<strong>Dangerzone requires Docker Desktop</strong><br><br>"
|
||||
"<a href='https://www.docker.com/products/docker-desktop'>Download Docker Desktop</a>"
|
||||
", install it, and open it."
|
||||
)
|
||||
|
||||
elif state == "not_running":
|
||||
if custom_runtime:
|
||||
self.show_error(
|
||||
"<strong>We were unable to start the container runtime defined in your settings</strong><br><br>"
|
||||
"Please check your settings, install it if needed, and retry."
|
||||
)
|
||||
elif platform.system() == "Linux":
|
||||
# "not_running" here means that the `podman image ls` command failed.
|
||||
self.show_error(
|
||||
"<strong>Dangerzone requires Podman</strong><br><br>"
|
||||
"Podman is installed but cannot run properly. See errors below",
|
||||
error,
|
||||
)
|
||||
else:
|
||||
self.show_error(
|
||||
"<strong>Dangerzone requires Docker Desktop</strong><br><br>"
|
||||
"Docker is installed but isn't running.<br><br>"
|
||||
"Open Docker and make sure it's running in the background.",
|
||||
error,
|
||||
)
|
||||
else:
|
||||
self.show_message(
|
||||
"Installing the Dangerzone container image.<br><br>"
|
||||
"This might take a few minutes..."
|
||||
self.label.setText(
|
||||
"<strong>Dangerzone Requires Docker Desktop</strong><br><br><a href='https://www.docker.com/products/docker-desktop'>Download Docker Desktop</a>, install it, and open it."
|
||||
)
|
||||
self.buttons.show()
|
||||
elif state == "not_running":
|
||||
self.label.setText(
|
||||
"<strong>Dangerzone Requires Docker Desktop</strong><br><br>Docker is installed but isn't running.<br><br>Open Docker and make sure it's running in the background."
|
||||
)
|
||||
self.buttons.show()
|
||||
else:
|
||||
self.label.setText(
|
||||
"Installing the Dangerzone container image.<br><br>This might take a few minutes..."
|
||||
)
|
||||
self.buttons.hide()
|
||||
self.install_container_t = InstallContainerThread(self.dangerzone)
|
||||
self.install_container_t.finished.connect(self.installation_finished)
|
||||
self.install_container_t.finished.connect(self.finished)
|
||||
self.install_container_t.start()
|
||||
|
||||
|
||||
|
|
@ -639,10 +456,6 @@ class ContentWidget(QtWidgets.QWidget):
|
|||
# Doc selection widget
|
||||
self.doc_selection_widget = DocSelectionWidget(self.dangerzone)
|
||||
self.doc_selection_widget.documents_selected.connect(self.documents_selected)
|
||||
self.doc_selection_wrapper = DocSelectionDropFrame(
|
||||
self.dangerzone, self.doc_selection_widget
|
||||
)
|
||||
self.doc_selection_wrapper.documents_selected.connect(self.documents_selected)
|
||||
|
||||
# Settings
|
||||
self.settings_widget = SettingsWidget(self.dangerzone)
|
||||
|
|
@ -663,26 +476,26 @@ class ContentWidget(QtWidgets.QWidget):
|
|||
layout = QtWidgets.QVBoxLayout()
|
||||
layout.addWidget(self.settings_widget, stretch=1)
|
||||
layout.addWidget(self.documents_list, stretch=1)
|
||||
layout.addWidget(self.doc_selection_wrapper, stretch=1)
|
||||
layout.addWidget(self.doc_selection_widget, stretch=1)
|
||||
self.setLayout(layout)
|
||||
|
||||
def documents_selected(self, docs: List[Document]) -> None:
|
||||
if self.conversion_started:
|
||||
self.alert = Alert(
|
||||
Alert(
|
||||
self.dangerzone,
|
||||
message="Dangerzone does not support adding documents after the conversion has started.",
|
||||
has_cancel=False,
|
||||
).launch()
|
||||
).exec_()
|
||||
return
|
||||
|
||||
# Ensure all files in batch are in the same directory
|
||||
dirnames = {os.path.dirname(doc.input_filename) for doc in docs}
|
||||
if len(dirnames) > 1:
|
||||
self.alert = Alert(
|
||||
Alert(
|
||||
self.dangerzone,
|
||||
message="Dangerzone does not support adding documents from multiple locations.\n\n The newly added documents were ignored.",
|
||||
has_cancel=False,
|
||||
).launch()
|
||||
).exec_()
|
||||
return
|
||||
|
||||
# Clear previously selected documents
|
||||
|
|
@ -693,7 +506,7 @@ class ContentWidget(QtWidgets.QWidget):
|
|||
for doc in docs:
|
||||
self.dangerzone.add_document(doc)
|
||||
|
||||
self.doc_selection_wrapper.hide()
|
||||
self.doc_selection_widget.hide()
|
||||
self.settings_widget.show()
|
||||
|
||||
if len(docs) > 0:
|
||||
|
|
@ -739,8 +552,20 @@ class DocSelectionWidget(QtWidgets.QWidget):
|
|||
self.file_dialog = QtWidgets.QFileDialog()
|
||||
self.file_dialog.setWindowTitle("Open Documents")
|
||||
self.file_dialog.setFileMode(QtWidgets.QFileDialog.ExistingFiles)
|
||||
|
||||
# XXX: We disable loading HWP/HWPX files on Qubes, because H2ORestart does not work there.
|
||||
# See:
|
||||
#
|
||||
# https://github.com/freedomofpress/dangerzone/issues/494
|
||||
hwp_filters = "*.hwp *.hwpx"
|
||||
if is_qubes_native_conversion():
|
||||
hwp_filters = ""
|
||||
self.file_dialog.setNameFilters(
|
||||
["Documents (*" + " *".join(get_supported_extensions()) + ")"]
|
||||
[
|
||||
"Documents (*.pdf *.docx *.doc *.docm *.xlsx *.xls *.pptx *.ppt *.odt"
|
||||
f" *.odg *.odp *.ods {hwp_filters} *.epub *.jpg *.jpeg *.gif *.png"
|
||||
" *.tif *.tiff *.bmp *.pnm *.pbm *.ppm *.svg)"
|
||||
]
|
||||
)
|
||||
|
||||
def dangerous_doc_button_clicked(self) -> None:
|
||||
|
|
@ -760,103 +585,6 @@ class DocSelectionWidget(QtWidgets.QWidget):
|
|||
pass
|
||||
|
||||
|
||||
class DocSelectionDropFrame(QtWidgets.QFrame):
|
||||
"""
|
||||
HACK Docs selecting widget "drag-n-drop" border widget
|
||||
The border frame doesn't show around the whole widget
|
||||
unless there is another widget wrapping it
|
||||
"""
|
||||
|
||||
documents_selected = QtCore.Signal(list)
|
||||
|
||||
def __init__(
|
||||
self, dangerzone: DangerzoneGui, docs_selection_widget: DocSelectionWidget
|
||||
) -> None:
|
||||
super().__init__()
|
||||
|
||||
self.dangerzone = dangerzone
|
||||
self.docs_selection_widget = docs_selection_widget
|
||||
|
||||
# Drag and drop functionality
|
||||
self.setAcceptDrops(True)
|
||||
|
||||
self.document_image_text = QtWidgets.QLabel(
|
||||
"Drag and drop\ndocuments here\n\nor"
|
||||
)
|
||||
self.document_image_text.setAlignment(QtCore.Qt.AlignCenter)
|
||||
self.document_image = QtWidgets.QLabel()
|
||||
self.document_image.setAlignment(QtCore.Qt.AlignCenter)
|
||||
self.document_image.setPixmap(
|
||||
load_svg_image("document.svg", width=20, height=24)
|
||||
)
|
||||
|
||||
self.center_layout = QtWidgets.QVBoxLayout()
|
||||
self.center_layout.addWidget(self.document_image)
|
||||
self.center_layout.addWidget(self.document_image_text)
|
||||
self.center_layout.addWidget(self.docs_selection_widget)
|
||||
|
||||
self.drop_layout = QtWidgets.QVBoxLayout()
|
||||
self.drop_layout.addStretch()
|
||||
self.drop_layout.addLayout(self.center_layout)
|
||||
self.drop_layout.addStretch()
|
||||
|
||||
self.setLayout(self.drop_layout)
|
||||
|
||||
def dragEnterEvent(self, ev: QtGui.QDragEnterEvent) -> None:
|
||||
ev.accept()
|
||||
|
||||
def dragLeaveEvent(self, ev: QtGui.QDragLeaveEvent) -> None:
|
||||
ev.accept()
|
||||
|
||||
def dropEvent(self, ev: QtGui.QDropEvent) -> None:
|
||||
ev.setDropAction(QtCore.Qt.CopyAction)
|
||||
documents = []
|
||||
supported_exts = get_supported_extensions()
|
||||
for url_path in ev.mimeData().urls():
|
||||
doc_path = url_path.toLocalFile()
|
||||
doc_ext = os.path.splitext(doc_path)[1]
|
||||
if doc_ext in supported_exts:
|
||||
documents += [Document(doc_path)]
|
||||
|
||||
# Ignore anything dropped that's not a file (e.g. text)
|
||||
if len(documents) == 0:
|
||||
return
|
||||
|
||||
# Ignore when all dropped files are unsupported
|
||||
total_dragged_docs = len(ev.mimeData().urls())
|
||||
num_unsupported_docs = total_dragged_docs - len(documents)
|
||||
|
||||
if num_unsupported_docs == total_dragged_docs:
|
||||
return
|
||||
|
||||
# Confirm with user when _some_ docs were ignored
|
||||
if num_unsupported_docs > 0:
|
||||
if not self.prompt_continue_without(num_unsupported_docs):
|
||||
return
|
||||
self.documents_selected.emit(documents)
|
||||
|
||||
def prompt_continue_without(self, num_unsupported_docs: int) -> int:
|
||||
"""
|
||||
Prompt the user if they want to convert even though some files are not
|
||||
supported.
|
||||
"""
|
||||
if num_unsupported_docs == 1:
|
||||
text = "1 file is not supported."
|
||||
ok_text = "Continue without this file"
|
||||
else: # plural
|
||||
text = f"{num_unsupported_docs} files are not supported."
|
||||
ok_text = "Continue without these files"
|
||||
|
||||
self.alert = Alert(
|
||||
self.dangerzone,
|
||||
message=f"{text}\nThe supported extensions are: "
|
||||
+ ", ".join(get_supported_extensions()),
|
||||
ok_text=ok_text,
|
||||
)
|
||||
|
||||
return self.alert.exec_()
|
||||
|
||||
|
||||
class SettingsWidget(QtWidgets.QWidget):
|
||||
start_clicked = QtCore.Signal()
|
||||
change_docs_clicked = QtCore.Signal()
|
||||
|
|
@ -884,23 +612,23 @@ class SettingsWidget(QtWidgets.QWidget):
|
|||
self.safe_extension = QtWidgets.QLineEdit()
|
||||
self.safe_extension.setStyleSheet("margin-left: -6px;") # no left margin
|
||||
self.safe_extension.textChanged.connect(self.update_ui)
|
||||
self.safe_extension_invalid = QtWidgets.QLabel("")
|
||||
self.safe_extension_invalid = QtWidgets.QLabel("(must end in .pdf)")
|
||||
self.safe_extension_invalid.setStyleSheet("color: red")
|
||||
self.safe_extension_invalid.hide()
|
||||
self.safe_extension_name_layout = QtWidgets.QHBoxLayout()
|
||||
self.safe_extension_name_layout.setSpacing(0)
|
||||
self.safe_extension_name_layout.addWidget(self.safe_extension_filename)
|
||||
self.safe_extension_name_layout.addWidget(self.safe_extension)
|
||||
self.dot_pdf_validator = QtGui.QRegularExpressionValidator(
|
||||
QtCore.QRegularExpression(r".*\.[Pp][Dd][Ff]")
|
||||
)
|
||||
if platform.system() == "Linux":
|
||||
illegal_chars_regex = r"[/]"
|
||||
elif platform.system() == "Darwin":
|
||||
illegal_chars_regex = r"[\\]"
|
||||
|
||||
# FIXME: Workaround for https://github.com/freedomofpress/dangerzone/issues/339.
|
||||
# We should drop this once we drop Ubuntu Focal support.
|
||||
if hasattr(QtGui, "QRegularExpressionValidator"):
|
||||
dot_pdf_regex = QtCore.QRegularExpression(r".*\.[Pp][Dd][Ff]")
|
||||
validator = QtGui.QRegularExpressionValidator(dot_pdf_regex)
|
||||
else:
|
||||
illegal_chars_regex = r"[\"*/:<>?\\|]"
|
||||
self.illegal_chars_regex = QtCore.QRegularExpression(illegal_chars_regex)
|
||||
dot_pdf_regex = QtCore.QRegExp(r".*\.[Pp][Dd][Ff]") # type: ignore [assignment]
|
||||
validator = QtGui.QRegExpValidator(dot_pdf_regex) # type: ignore [call-overload]
|
||||
self.safe_extension.setValidator(validator)
|
||||
self.safe_extension_layout = QtWidgets.QHBoxLayout()
|
||||
self.safe_extension_layout.addWidget(self.save_checkbox)
|
||||
self.safe_extension_layout.addWidget(self.safe_extension_label)
|
||||
|
|
@ -1043,32 +771,14 @@ class SettingsWidget(QtWidgets.QWidget):
|
|||
# ignore validity if not saving file
|
||||
self.safe_extension_invalid.hide()
|
||||
return True
|
||||
return (
|
||||
self.check_safe_extension_illegal_chars()
|
||||
and self.check_safe_extension_dot_pdf()
|
||||
)
|
||||
|
||||
def check_safe_extension_illegal_chars(self) -> bool:
|
||||
match = self.illegal_chars_regex.match(self.safe_extension.text())
|
||||
if match.hasMatch():
|
||||
self.set_safe_extension_invalid_label(
|
||||
f"illegal character: {match.captured()}"
|
||||
)
|
||||
if self.safe_extension.hasAcceptableInput():
|
||||
self.safe_extension_invalid.hide()
|
||||
return True
|
||||
else:
|
||||
# prevent starting conversion until correct
|
||||
self.safe_extension_invalid.show()
|
||||
return False
|
||||
self.safe_extension_invalid.hide()
|
||||
return True
|
||||
|
||||
def check_safe_extension_dot_pdf(self) -> bool:
|
||||
self.safe_extension.setValidator(self.dot_pdf_validator)
|
||||
if not self.safe_extension.hasAcceptableInput():
|
||||
self.set_safe_extension_invalid_label("must end in .pdf")
|
||||
return False
|
||||
self.safe_extension_invalid.hide()
|
||||
return True
|
||||
|
||||
def set_safe_extension_invalid_label(self, string: str) -> None:
|
||||
self.safe_extension_invalid.setText(string)
|
||||
self.safe_extension_invalid.show()
|
||||
|
||||
def check_either_save_or_open(self) -> bool:
|
||||
return (
|
||||
|
|
@ -1115,7 +825,7 @@ class SettingsWidget(QtWidgets.QWidget):
|
|||
|
||||
if n_docs == 1:
|
||||
self.start_button.setText("Convert to Safe Document")
|
||||
self.docs_selected_label.setText("1 document selected")
|
||||
self.docs_selected_label.setText(f"1 document selected")
|
||||
else:
|
||||
self.start_button.setText("Convert to Safe Documents")
|
||||
self.docs_selected_label.setText(f"{n_docs} documents selected")
|
||||
|
|
@ -1131,7 +841,7 @@ class SettingsWidget(QtWidgets.QWidget):
|
|||
dialog.setFileMode(QtWidgets.QFileDialog.Directory)
|
||||
dialog.setOption(QtWidgets.QFileDialog.ShowDirsOnly, True)
|
||||
|
||||
if dialog.exec() == QtWidgets.QFileDialog.Accepted:
|
||||
if dialog.exec_() == QtWidgets.QFileDialog.Accepted:
|
||||
selected_dir = dialog.selectedFiles()[0]
|
||||
if selected_dir is not None:
|
||||
self.dangerzone.output_dir = str(selected_dir)
|
||||
|
|
@ -1319,7 +1029,7 @@ class DocumentWidget(QtWidgets.QWidget):
|
|||
|
||||
def load_status_image(self, filename: str) -> QtGui.QPixmap:
|
||||
path = get_resource_path(filename)
|
||||
img = QtGui.QImage(str(path))
|
||||
img = QtGui.QImage(path)
|
||||
image = QtGui.QPixmap.fromImage(img)
|
||||
return image.scaled(QtCore.QSize(15, 15))
|
||||
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ import platform
|
|||
import sys
|
||||
import time
|
||||
import typing
|
||||
from typing import Optional
|
||||
from typing import Any, Optional
|
||||
|
||||
from packaging import version
|
||||
|
||||
|
|
@ -20,7 +20,7 @@ else:
|
|||
|
||||
# XXX implict import for "markdown" module required for Cx_Freeze to build on Windows
|
||||
# See https://github.com/freedomofpress/dangerzone/issues/501
|
||||
import html.parser # noqa: F401
|
||||
import html.parser
|
||||
|
||||
import markdown
|
||||
import requests
|
||||
|
|
@ -206,7 +206,7 @@ class UpdaterThread(QtCore.QThread):
|
|||
current_time = self._get_now_timestamp()
|
||||
last_check = self.dangerzone.settings.get("updater_last_check")
|
||||
if current_time < last_check + UPDATE_CHECK_COOLDOWN_SECS:
|
||||
log.debug("Cooling down update checks")
|
||||
log.debug(f"Cooling down update checks")
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
|
@ -232,13 +232,13 @@ class UpdaterThread(QtCore.QThread):
|
|||
|
||||
try:
|
||||
info = res.json()
|
||||
except json.JSONDecodeError:
|
||||
except json.JSONDecodeError as e:
|
||||
raise ValueError(f"Received a non-JSON response from {self.GH_RELEASE_URL}")
|
||||
|
||||
try:
|
||||
version = info["tag_name"].lstrip("v")
|
||||
changelog = markdown.markdown(info["body"])
|
||||
except KeyError:
|
||||
except KeyError as e:
|
||||
raise ValueError(
|
||||
f"Missing required fields in JSON response from {self.GH_RELEASE_URL}"
|
||||
)
|
||||
|
|
@ -255,10 +255,10 @@ class UpdaterThread(QtCore.QThread):
|
|||
previous run.
|
||||
2. In GitHub, by hitting the latest releases API.
|
||||
"""
|
||||
log.debug("Checking for Dangerzone updates")
|
||||
log.debug(f"Checking for Dangerzone updates")
|
||||
latest_version = self.dangerzone.settings.get("updater_latest_version")
|
||||
if version.parse(get_version()) < version.parse(latest_version):
|
||||
log.debug("Determined that there is an update due to cached results")
|
||||
log.debug(f"Determined that there is an update due to cached results")
|
||||
return UpdateReport(
|
||||
version=latest_version,
|
||||
changelog=self.dangerzone.settings.get("updater_latest_changelog"),
|
||||
|
|
@ -275,7 +275,7 @@ class UpdaterThread(QtCore.QThread):
|
|||
"updater_last_check", self._get_now_timestamp(), autosave=True
|
||||
)
|
||||
|
||||
log.debug("Checking the latest GitHub release")
|
||||
log.debug(f"Checking the latest GitHub release")
|
||||
report = self.get_latest_info()
|
||||
log.debug(f"Latest version in GitHub is {report.version}")
|
||||
if report.version and self.can_update(latest_version, report.version):
|
||||
|
|
@ -285,7 +285,7 @@ class UpdaterThread(QtCore.QThread):
|
|||
)
|
||||
return report
|
||||
|
||||
log.debug("No need to update")
|
||||
log.debug(f"No need to update")
|
||||
return UpdateReport()
|
||||
|
||||
##################
|
||||
|
|
|
|||
|
|
@ -1,62 +1,33 @@
|
|||
import contextlib
|
||||
import logging
|
||||
import os
|
||||
import platform
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import tempfile
|
||||
from abc import ABC, abstractmethod
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
from typing import IO, Callable, Iterator, Optional
|
||||
|
||||
import fitz
|
||||
from colorama import Fore, Style
|
||||
|
||||
from ..conversion import errors
|
||||
from ..conversion.common import DEFAULT_DPI, INT_BYTES
|
||||
from ..conversion.common import INT_BYTES
|
||||
from ..document import Document
|
||||
from ..util import get_tessdata_dir, replace_control_chars
|
||||
from ..util import replace_control_chars
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
MAX_CONVERSION_LOG_CHARS = 150 * 50 # up to ~150 lines of 50 characters
|
||||
DOC_TO_PIXELS_LOG_START = "----- DOC TO PIXELS LOG START -----"
|
||||
DOC_TO_PIXELS_LOG_END = "----- DOC TO PIXELS LOG END -----"
|
||||
PIXELS_TO_PDF_LOG_START = "----- PIXELS TO PDF LOG START -----"
|
||||
PIXELS_TO_PDF_LOG_END = "----- PIXELS TO PDF LOG END -----"
|
||||
|
||||
TIMEOUT_EXCEPTION = 15
|
||||
TIMEOUT_GRACE = 15
|
||||
TIMEOUT_FORCE = 5
|
||||
|
||||
|
||||
def _signal_process_group(p: subprocess.Popen, signo: int) -> None:
|
||||
"""Send a signal to a process group."""
|
||||
try:
|
||||
os.killpg(os.getpgid(p.pid), signo)
|
||||
except (ProcessLookupError, PermissionError):
|
||||
# If the process no longer exists, we may encounter the above errors, either
|
||||
# when looking for the process group (ProcessLookupError), or when trying to
|
||||
# kill a process group that no longer exists (PermissionError)
|
||||
return
|
||||
except Exception:
|
||||
log.exception(
|
||||
f"Unexpected error while sending signal {signo} to the"
|
||||
f"document-to-pixels process group (PID: {p.pid})"
|
||||
)
|
||||
|
||||
|
||||
def terminate_process_group(p: subprocess.Popen) -> None:
|
||||
"""Terminate a process group."""
|
||||
if platform.system() == "Windows":
|
||||
p.terminate()
|
||||
else:
|
||||
_signal_process_group(p, signal.SIGTERM)
|
||||
|
||||
|
||||
def kill_process_group(p: subprocess.Popen) -> None:
|
||||
"""Forcefully kill a process group."""
|
||||
if platform.system() == "Windows":
|
||||
p.kill()
|
||||
else:
|
||||
_signal_process_group(p, signal.SIGKILL)
|
||||
|
||||
|
||||
def read_bytes(f: IO[bytes], size: int, exact: bool = True) -> bytes:
|
||||
"""Read bytes from a file-like object."""
|
||||
buf = f.read(size)
|
||||
|
|
@ -73,9 +44,9 @@ def read_int(f: IO[bytes]) -> int:
|
|||
return int.from_bytes(untrusted_int, "big", signed=False)
|
||||
|
||||
|
||||
def sanitize_debug_text(text: bytes) -> str:
|
||||
"""Read all the buffer and return a sanitized version"""
|
||||
untrusted_text = text.decode("ascii", errors="replace")
|
||||
def read_debug_text(f: IO[bytes], size: int) -> str:
|
||||
"""Read arbitrarily long text (for debug purposes), and sanitize it."""
|
||||
untrusted_text = f.read(size).decode("ascii", errors="replace")
|
||||
return replace_control_chars(untrusted_text, keep_newlines=True)
|
||||
|
||||
|
||||
|
|
@ -84,16 +55,12 @@ class IsolationProvider(ABC):
|
|||
Abstracts an isolation provider
|
||||
"""
|
||||
|
||||
def __init__(self, debug: bool = False) -> None:
|
||||
self.debug = debug
|
||||
if self.should_capture_stderr():
|
||||
def __init__(self) -> None:
|
||||
if getattr(sys, "dangerzone_dev", False) == True:
|
||||
self.proc_stderr = subprocess.PIPE
|
||||
else:
|
||||
self.proc_stderr = subprocess.DEVNULL
|
||||
|
||||
def should_capture_stderr(self) -> bool:
|
||||
return self.debug or getattr(sys, "dangerzone_dev", False)
|
||||
|
||||
@abstractmethod
|
||||
def install(self) -> bool:
|
||||
pass
|
||||
|
|
@ -107,8 +74,11 @@ class IsolationProvider(ABC):
|
|||
self.progress_callback = progress_callback
|
||||
document.mark_as_converting()
|
||||
try:
|
||||
with self.doc_to_pixels_proc(document) as conversion_proc:
|
||||
self.convert_with_proc(document, ocr_lang, conversion_proc)
|
||||
with tempfile.TemporaryDirectory() as t:
|
||||
Path(f"{t}/pixels").mkdir()
|
||||
with self.doc_to_pixels_proc(document) as conversion_proc:
|
||||
self.doc_to_pixels(document, t, conversion_proc)
|
||||
self.pixels_to_pdf(document, t, ocr_lang)
|
||||
document.mark_as_safe()
|
||||
if document.archive_after_conversion:
|
||||
document.archive()
|
||||
|
|
@ -122,45 +92,8 @@ class IsolationProvider(ABC):
|
|||
self.print_progress(document, True, str(e), 0)
|
||||
document.mark_as_failed()
|
||||
|
||||
def ocr_page(self, pixmap: fitz.Pixmap, ocr_lang: str) -> bytes:
|
||||
"""Get a single page as pixels, OCR it, and return a PDF as bytes."""
|
||||
return pixmap.pdfocr_tobytes(
|
||||
compress=True,
|
||||
language=ocr_lang,
|
||||
tessdata=str(get_tessdata_dir()),
|
||||
)
|
||||
|
||||
def pixels_to_pdf_page(
|
||||
self,
|
||||
untrusted_data: bytes,
|
||||
untrusted_width: int,
|
||||
untrusted_height: int,
|
||||
ocr_lang: Optional[str],
|
||||
) -> fitz.Document:
|
||||
"""Convert a byte array of RGB pixels into a PDF page, optionally with OCR."""
|
||||
pixmap = fitz.Pixmap(
|
||||
fitz.Colorspace(fitz.CS_RGB),
|
||||
untrusted_width,
|
||||
untrusted_height,
|
||||
untrusted_data,
|
||||
False,
|
||||
)
|
||||
pixmap.set_dpi(DEFAULT_DPI, DEFAULT_DPI)
|
||||
|
||||
if ocr_lang: # OCR the document
|
||||
page_pdf_bytes = self.ocr_page(pixmap, ocr_lang)
|
||||
else: # Don't OCR
|
||||
page_doc = fitz.Document()
|
||||
page_doc.insert_file(pixmap)
|
||||
page_pdf_bytes = page_doc.tobytes(deflate_images=True)
|
||||
|
||||
return fitz.open("pdf", page_pdf_bytes)
|
||||
|
||||
def convert_with_proc(
|
||||
self,
|
||||
document: Document,
|
||||
ocr_lang: Optional[str],
|
||||
p: subprocess.Popen,
|
||||
def doc_to_pixels(
|
||||
self, document: Document, tempdir: str, p: subprocess.Popen
|
||||
) -> None:
|
||||
percentage = 0.0
|
||||
with open(document.input_filename, "rb") as f:
|
||||
|
|
@ -168,22 +101,17 @@ class IsolationProvider(ABC):
|
|||
assert p.stdin is not None
|
||||
p.stdin.write(f.read())
|
||||
p.stdin.close()
|
||||
except BrokenPipeError:
|
||||
except BrokenPipeError as e:
|
||||
raise errors.ConverterProcException()
|
||||
|
||||
assert p.stdout
|
||||
n_pages = read_int(p.stdout)
|
||||
if n_pages == 0 or n_pages > errors.MAX_PAGES:
|
||||
raise errors.MaxPagesException()
|
||||
step = 100 / n_pages
|
||||
|
||||
safe_doc = fitz.Document()
|
||||
percentage_per_page = 49.0 / n_pages
|
||||
|
||||
for page in range(1, n_pages + 1):
|
||||
searchable = "searchable " if ocr_lang else ""
|
||||
text = (
|
||||
f"Converting page {page}/{n_pages} from pixels to {searchable}PDF"
|
||||
)
|
||||
text = f"Converting page {page}/{n_pages} to pixels"
|
||||
self.print_progress(document, False, text, percentage)
|
||||
|
||||
width = read_int(p.stdout)
|
||||
|
|
@ -199,27 +127,39 @@ class IsolationProvider(ABC):
|
|||
num_pixels,
|
||||
)
|
||||
|
||||
page_pdf = self.pixels_to_pdf_page(
|
||||
untrusted_pixels,
|
||||
width,
|
||||
height,
|
||||
ocr_lang,
|
||||
)
|
||||
safe_doc.insert_pdf(page_pdf)
|
||||
# Wrapper code
|
||||
with open(f"{tempdir}/pixels/page-{page}.width", "w") as f_width:
|
||||
f_width.write(str(width))
|
||||
with open(f"{tempdir}/pixels/page-{page}.height", "w") as f_height:
|
||||
f_height.write(str(height))
|
||||
with open(f"{tempdir}/pixels/page-{page}.rgb", "wb") as f_rgb:
|
||||
f_rgb.write(untrusted_pixels)
|
||||
|
||||
percentage += step
|
||||
percentage += percentage_per_page
|
||||
|
||||
# Ensure nothing else is read after all bitmaps are obtained
|
||||
p.stdout.close()
|
||||
|
||||
# Saving it with a different name first, because PyMuPDF cannot handle
|
||||
# non-Unicode chars.
|
||||
safe_doc.save(document.sanitized_output_filename)
|
||||
os.replace(document.sanitized_output_filename, document.output_filename)
|
||||
|
||||
# TODO handle leftover code input
|
||||
text = "Successfully converted document"
|
||||
self.print_progress(document, False, text, 100)
|
||||
text = "Converted document to pixels"
|
||||
self.print_progress(document, False, text, percentage)
|
||||
|
||||
if getattr(sys, "dangerzone_dev", False):
|
||||
assert p.stderr
|
||||
debug_log = read_debug_text(p.stderr, MAX_CONVERSION_LOG_CHARS)
|
||||
p.stderr.close()
|
||||
log.info(
|
||||
"Conversion output (doc to pixels)\n"
|
||||
f"{DOC_TO_PIXELS_LOG_START}\n"
|
||||
f"{debug_log}" # no need for an extra newline here
|
||||
f"{DOC_TO_PIXELS_LOG_END}"
|
||||
)
|
||||
|
||||
@abstractmethod
|
||||
def pixels_to_pdf(
|
||||
self, document: Document, tempdir: str, ocr_lang: Optional[str]
|
||||
) -> None:
|
||||
pass
|
||||
|
||||
def print_progress(
|
||||
self, document: Document, error: bool, text: str, percentage: float
|
||||
|
|
@ -255,16 +195,6 @@ class IsolationProvider(ABC):
|
|||
)
|
||||
return errors.exception_from_error_code(error_code)
|
||||
|
||||
@abstractmethod
|
||||
def should_wait_install(self) -> bool:
|
||||
"""Whether this isolation provider takes a lot of time to install."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def is_available(self) -> bool:
|
||||
"""Whether the backing implementation of the isolation provider is available."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def get_max_parallel_conversions(self) -> int:
|
||||
pass
|
||||
|
|
@ -311,7 +241,7 @@ class IsolationProvider(ABC):
|
|||
)
|
||||
|
||||
# Forcefully kill the running process.
|
||||
kill_process_group(p)
|
||||
p.kill()
|
||||
try:
|
||||
p.wait(timeout_force)
|
||||
except subprocess.TimeoutExpired:
|
||||
|
|
@ -329,16 +259,7 @@ class IsolationProvider(ABC):
|
|||
timeout_force: int = TIMEOUT_FORCE,
|
||||
) -> Iterator[subprocess.Popen]:
|
||||
"""Start a conversion process, pass it to the caller, and then clean it up."""
|
||||
# Store the proc stderr in memory
|
||||
stderr = BytesIO()
|
||||
p = self.start_doc_to_pixels_proc(document)
|
||||
stderr_thread = self.start_stderr_thread(p, stderr)
|
||||
|
||||
if platform.system() != "Windows":
|
||||
assert os.getpgid(p.pid) != os.getpgid(os.getpid()), (
|
||||
"Parent shares same PGID with child"
|
||||
)
|
||||
|
||||
try:
|
||||
yield p
|
||||
except errors.ConverterProcException as e:
|
||||
|
|
@ -349,40 +270,74 @@ class IsolationProvider(ABC):
|
|||
document, p, timeout_grace=timeout_grace, timeout_force=timeout_force
|
||||
)
|
||||
|
||||
if stderr_thread:
|
||||
# Wait for the thread to complete. If it's still alive, mention it in the debug log.
|
||||
stderr_thread.join(timeout=1)
|
||||
|
||||
debug_bytes = stderr.getvalue()
|
||||
debug_log = sanitize_debug_text(debug_bytes)
|
||||
# From global_common:
|
||||
|
||||
incomplete = "(incomplete) " if stderr_thread.is_alive() else ""
|
||||
# def validate_convert_to_pixel_output(self, common, output):
|
||||
# """
|
||||
# Take the output from the convert to pixels tasks and validate it. Returns
|
||||
# a tuple like: (success (boolean), error_message (str))
|
||||
# """
|
||||
# max_image_width = 10000
|
||||
# max_image_height = 10000
|
||||
|
||||
log.info(
|
||||
"Conversion output (doc to pixels)\n"
|
||||
f"----- DOC TO PIXELS LOG START {incomplete}-----\n"
|
||||
f"{debug_log}" # no need for an extra newline here
|
||||
"----- DOC TO PIXELS LOG END -----"
|
||||
)
|
||||
# # Did we hit an error?
|
||||
# for line in output.split("\n"):
|
||||
# if (
|
||||
# "failed:" in line
|
||||
# or "The document format is not supported" in line
|
||||
# or "Error" in line
|
||||
# ):
|
||||
# return False, output
|
||||
|
||||
def start_stderr_thread(
|
||||
self, process: subprocess.Popen, stderr: IO[bytes]
|
||||
) -> Optional[threading.Thread]:
|
||||
"""Start a thread to read stderr from the process"""
|
||||
# # How many pages was that?
|
||||
# num_pages = None
|
||||
# for line in output.split("\n"):
|
||||
# if line.startswith("Document has "):
|
||||
# num_pages = line.split(" ")[2]
|
||||
# break
|
||||
# if not num_pages or not num_pages.isdigit() or int(num_pages) <= 0:
|
||||
# return False, "Invalid number of pages returned"
|
||||
# num_pages = int(num_pages)
|
||||
|
||||
def _stream_stderr(process_stderr: IO[bytes]) -> None:
|
||||
try:
|
||||
for line in process_stderr:
|
||||
stderr.write(line)
|
||||
except (ValueError, IOError) as e:
|
||||
log.debug(f"Stderr stream closed: {e}")
|
||||
# # Make sure we have the files we expect
|
||||
# expected_filenames = []
|
||||
# for i in range(1, num_pages + 1):
|
||||
# expected_filenames += [
|
||||
# f"page-{i}.rgb",
|
||||
# f"page-{i}.width",
|
||||
# f"page-{i}.height",
|
||||
# ]
|
||||
# expected_filenames.sort()
|
||||
# actual_filenames = os.listdir(common.pixel_dir.name)
|
||||
# actual_filenames.sort()
|
||||
|
||||
if process.stderr:
|
||||
stderr_thread = threading.Thread(
|
||||
target=_stream_stderr,
|
||||
args=(process.stderr,),
|
||||
daemon=True,
|
||||
)
|
||||
stderr_thread.start()
|
||||
return stderr_thread
|
||||
return None
|
||||
# if expected_filenames != actual_filenames:
|
||||
# return (
|
||||
# False,
|
||||
# f"We expected these files:\n{expected_filenames}\n\nBut we got these files:\n{actual_filenames}",
|
||||
# )
|
||||
|
||||
# # Make sure the files are the correct sizes
|
||||
# for i in range(1, num_pages + 1):
|
||||
# with open(f"{common.pixel_dir.name}/page-{i}.width") as f:
|
||||
# w_str = f.read().strip()
|
||||
# with open(f"{common.pixel_dir.name}/page-{i}.height") as f:
|
||||
# h_str = f.read().strip()
|
||||
# w = int(w_str)
|
||||
# h = int(h_str)
|
||||
# if (
|
||||
# not w_str.isdigit()
|
||||
# or not h_str.isdigit()
|
||||
# or w <= 0
|
||||
# or w > max_image_width
|
||||
# or h <= 0
|
||||
# or h > max_image_height
|
||||
# ):
|
||||
# return False, f"Page {i} has invalid geometry"
|
||||
|
||||
# # Make sure the RGB file is the correct size
|
||||
# if os.path.getsize(f"{common.pixel_dir.name}/page-{i}.rgb") != w * h * 3:
|
||||
# return False, f"Page {i} has an invalid RGB file size"
|
||||
|
||||
# return True, True
|
||||
|
|
|
|||
|
|
@ -1,21 +1,18 @@
|
|||
import gzip
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import platform
|
||||
import shlex
|
||||
import shutil
|
||||
import subprocess
|
||||
from typing import List, Tuple
|
||||
import sys
|
||||
from typing import Any, List, Optional
|
||||
|
||||
from .. import container_utils, errors
|
||||
from ..container_utils import Runtime
|
||||
from ..conversion import errors
|
||||
from ..document import Document
|
||||
from ..util import get_resource_path, get_subprocess_startupinfo
|
||||
from .base import IsolationProvider, terminate_process_group
|
||||
|
||||
TIMEOUT_KILL = 5 # Timeout in seconds until the kill command returns.
|
||||
MINIMUM_DOCKER_DESKTOP = {
|
||||
"Darwin": "4.40.0",
|
||||
"Windows": "4.40.0",
|
||||
}
|
||||
from ..util import get_resource_path, get_subprocess_startupinfo, get_tmp_dir
|
||||
from .base import PIXELS_TO_PDF_LOG_END, PIXELS_TO_PDF_LOG_START, IsolationProvider
|
||||
|
||||
# Define startupinfo for subprocesses
|
||||
if platform.system() == "Windows":
|
||||
|
|
@ -28,159 +25,109 @@ else:
|
|||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class NoContainerTechException(Exception):
|
||||
def __init__(self, container_tech: str) -> None:
|
||||
super().__init__(f"{container_tech} is not installed")
|
||||
|
||||
|
||||
class Container(IsolationProvider):
|
||||
# Name of the dangerzone container
|
||||
CONTAINER_NAME = "dangerzone.rocks/dangerzone"
|
||||
|
||||
@staticmethod
|
||||
def get_runtime_security_args() -> List[str]:
|
||||
"""Security options applicable to the outer Dangerzone container.
|
||||
|
||||
Our security precautions for the outer Dangerzone container are the following:
|
||||
* Do not let the container assume new privileges.
|
||||
* Drop all capabilities, except for CAP_SYS_CHROOT, which is necessary for
|
||||
running gVisor.
|
||||
* Do not allow access to the network stack.
|
||||
* Run the container as the unprivileged `dangerzone` user.
|
||||
* Set the `container_engine_t` SELinux label, which allows gVisor to work on
|
||||
SELinux-enforcing systems
|
||||
(see https://github.com/freedomofpress/dangerzone/issues/880).
|
||||
* Set a custom seccomp policy for every container engine, since the `ptrace(2)`
|
||||
system call is forbidden by some.
|
||||
|
||||
For Podman specifically, where applicable, we also add the following:
|
||||
* Do not log the container's output.
|
||||
* Do not map the host user to the container, with `--userns nomap` (available
|
||||
from Podman 4.1 onwards)
|
||||
"""
|
||||
runtime = Runtime()
|
||||
if runtime.name == "podman":
|
||||
security_args = ["--log-driver", "none"]
|
||||
security_args += ["--security-opt", "no-new-privileges"]
|
||||
if container_utils.get_runtime_version() >= (4, 1):
|
||||
# We perform a platform check to avoid the following Podman Desktop
|
||||
# error on Windows:
|
||||
#
|
||||
# Error: nomap is only supported in rootless mode
|
||||
#
|
||||
# See also: https://github.com/freedomofpress/dangerzone/issues/1127
|
||||
if platform.system() != "Windows":
|
||||
security_args += ["--userns", "nomap"]
|
||||
def get_runtime_name() -> str:
|
||||
if platform.system() == "Linux":
|
||||
runtime_name = "podman"
|
||||
else:
|
||||
security_args = ["--security-opt=no-new-privileges:true"]
|
||||
# Windows, Darwin, and unknown use docker for now, dangerzone-vm eventually
|
||||
runtime_name = "docker"
|
||||
return runtime_name
|
||||
|
||||
# We specify a custom seccomp policy uniformly, because on certain container
|
||||
# engines the default policy might not allow the `ptrace(2)` syscall [1]. Our
|
||||
# custom seccomp policy has been copied as is [2] from the official Podman repo.
|
||||
#
|
||||
# [1] https://github.com/freedomofpress/dangerzone/issues/846
|
||||
# [2] https://github.com/containers/common/blob/d3283f8401eeeb21f3c59a425b5461f069e199a7/pkg/seccomp/seccomp.json
|
||||
seccomp_json_path = str(get_resource_path("seccomp.gvisor.json"))
|
||||
# We perform a platform check to avoid the following Podman Desktop
|
||||
# error on Windows:
|
||||
#
|
||||
# Error: opening seccomp profile failed: open
|
||||
# C:\[...]\dangerzone\share\seccomp.gvisor.json: no such file or directory
|
||||
#
|
||||
# See also: https://github.com/freedomofpress/dangerzone/issues/1127
|
||||
if runtime.name == "podman" and platform.system() != "Windows":
|
||||
security_args += ["--security-opt", f"seccomp={seccomp_json_path}"]
|
||||
|
||||
security_args += ["--cap-drop", "all"]
|
||||
security_args += ["--cap-add", "SYS_CHROOT"]
|
||||
security_args += ["--security-opt", "label=type:container_engine_t"]
|
||||
|
||||
security_args += ["--network=none"]
|
||||
security_args += ["-u", "dangerzone"]
|
||||
|
||||
return security_args
|
||||
@staticmethod
|
||||
def get_runtime() -> str:
|
||||
container_tech = Container.get_runtime_name()
|
||||
runtime = shutil.which(container_tech)
|
||||
if runtime is None:
|
||||
raise NoContainerTechException(container_tech)
|
||||
return runtime
|
||||
|
||||
@staticmethod
|
||||
def install() -> bool:
|
||||
"""Install the container image tarball, or verify that it's already installed.
|
||||
|
||||
Perform the following actions:
|
||||
1. Get the tags of any locally available images that match Dangerzone's image
|
||||
name.
|
||||
2. Get the expected image tag from the image-id.txt file.
|
||||
- If this tag is present in the local images, then we can return.
|
||||
- Else, prune the older container images and continue.
|
||||
3. Load the image tarball and make sure it matches the expected tag.
|
||||
"""
|
||||
old_tags = container_utils.list_image_tags()
|
||||
expected_tag = container_utils.get_expected_tag()
|
||||
|
||||
if expected_tag not in old_tags:
|
||||
# Prune older container images.
|
||||
log.info(
|
||||
f"Could not find a Dangerzone container image with tag '{expected_tag}'"
|
||||
)
|
||||
for tag in old_tags:
|
||||
tag = container_utils.CONTAINER_NAME + ":" + tag
|
||||
container_utils.delete_image_tag(tag)
|
||||
else:
|
||||
Make sure the podman container is installed. Linux only.
|
||||
"""
|
||||
if Container.is_container_installed():
|
||||
return True
|
||||
|
||||
# Load the image tarball into the container runtime.
|
||||
container_utils.load_image_tarball()
|
||||
# Load the container into podman
|
||||
log.info("Installing Dangerzone container image...")
|
||||
|
||||
# Check that the container image has the expected image tag.
|
||||
# See https://github.com/freedomofpress/dangerzone/issues/988 for an example
|
||||
# where this was not the case.
|
||||
new_tags = container_utils.list_image_tags()
|
||||
if expected_tag not in new_tags:
|
||||
raise errors.ImageNotPresentException(
|
||||
f"Could not find expected tag '{expected_tag}' after loading the"
|
||||
" container image tarball"
|
||||
)
|
||||
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def should_wait_install() -> bool:
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def is_available() -> bool:
|
||||
runtime = Runtime()
|
||||
|
||||
# Can we run `docker/podman image ls` without an error
|
||||
with subprocess.Popen(
|
||||
[str(runtime.path), "image", "ls"],
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.PIPE,
|
||||
p = subprocess.Popen(
|
||||
[Container.get_runtime(), "load"],
|
||||
stdin=subprocess.PIPE,
|
||||
startupinfo=get_subprocess_startupinfo(),
|
||||
) as p:
|
||||
_, stderr = p.communicate()
|
||||
if p.returncode != 0:
|
||||
raise errors.NotAvailableContainerTechException(
|
||||
runtime.name, stderr.decode()
|
||||
)
|
||||
|
||||
chunk_size = 10240
|
||||
compressed_container_path = get_resource_path("container.tar.gz")
|
||||
with gzip.open(compressed_container_path) as f:
|
||||
while True:
|
||||
chunk = f.read(chunk_size)
|
||||
if len(chunk) > 0:
|
||||
if p.stdin:
|
||||
p.stdin.write(chunk)
|
||||
else:
|
||||
break
|
||||
p.communicate()
|
||||
|
||||
if not Container.is_container_installed():
|
||||
log.error("Failed to install the container image")
|
||||
return False
|
||||
|
||||
log.info("Container image installed")
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def is_container_installed() -> bool:
|
||||
"""
|
||||
See if the podman container is installed. Linux only.
|
||||
"""
|
||||
# Get the image id
|
||||
with open(get_resource_path("image-id.txt")) as f:
|
||||
expected_image_id = f.read().strip()
|
||||
|
||||
# See if this image is already installed
|
||||
installed = False
|
||||
found_image_id = subprocess.check_output(
|
||||
[
|
||||
Container.get_runtime(),
|
||||
"image",
|
||||
"list",
|
||||
"--format",
|
||||
"{{.ID}}",
|
||||
Container.CONTAINER_NAME,
|
||||
],
|
||||
text=True,
|
||||
startupinfo=get_subprocess_startupinfo(),
|
||||
)
|
||||
found_image_id = found_image_id.strip()
|
||||
|
||||
if found_image_id == expected_image_id:
|
||||
installed = True
|
||||
elif found_image_id == "":
|
||||
pass
|
||||
else:
|
||||
log.info("Deleting old dangerzone container image")
|
||||
|
||||
try:
|
||||
subprocess.check_output(
|
||||
[Container.get_runtime(), "rmi", "--force", found_image_id],
|
||||
startupinfo=get_subprocess_startupinfo(),
|
||||
)
|
||||
return True
|
||||
except:
|
||||
log.warning("Couldn't delete old container image, so leaving it there")
|
||||
|
||||
def check_docker_desktop_version(self) -> Tuple[bool, str]:
|
||||
# On windows and darwin, check that the minimum version is met
|
||||
version = ""
|
||||
runtime = Runtime()
|
||||
runtime_is_docker = runtime.name == "docker"
|
||||
platform_is_not_linux = platform.system() != "Linux"
|
||||
|
||||
if runtime_is_docker and platform_is_not_linux:
|
||||
with subprocess.Popen(
|
||||
["docker", "version", "--format", "{{.Server.Platform.Name}}"],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
startupinfo=get_subprocess_startupinfo(),
|
||||
) as p:
|
||||
stdout, stderr = p.communicate()
|
||||
if p.returncode != 0:
|
||||
# When an error occurs, consider that the check went
|
||||
# through, as we're checking for installation compatibiliy
|
||||
# somewhere else already
|
||||
return True, version
|
||||
# The output is like "Docker Desktop 4.35.1 (173168)"
|
||||
version = stdout.decode().replace("Docker Desktop", "").split()[0]
|
||||
if version < MINIMUM_DOCKER_DESKTOP[platform.system()]:
|
||||
return False, version
|
||||
return True, version
|
||||
return installed
|
||||
|
||||
def doc_to_pixels_container_name(self, document: Document) -> str:
|
||||
"""Unique container name for the doc-to-pixels phase."""
|
||||
|
|
@ -190,6 +137,31 @@ class Container(IsolationProvider):
|
|||
"""Unique container name for the pixels-to-pdf phase."""
|
||||
return f"dangerzone-pixels-to-pdf-{document.id}"
|
||||
|
||||
def assert_field_type(self, val: Any, _type: object) -> None:
|
||||
# XXX: Use a stricter check than isinstance because `bool` is a subclass of
|
||||
# `int`.
|
||||
#
|
||||
# See https://stackoverflow.com/a/37888668
|
||||
if not type(val) == _type:
|
||||
raise ValueError("Status field has incorrect type")
|
||||
|
||||
def parse_progress_trusted(self, document: Document, line: str) -> None:
|
||||
"""
|
||||
Parses a line returned by the container.
|
||||
"""
|
||||
try:
|
||||
status = json.loads(line)
|
||||
text = status["text"]
|
||||
self.assert_field_type(text, str)
|
||||
error = status["error"]
|
||||
self.assert_field_type(error, bool)
|
||||
percentage = status["percentage"]
|
||||
self.assert_field_type(percentage, float)
|
||||
self.print_progress(document, error, text, percentage)
|
||||
except Exception:
|
||||
error_message = f"Invalid JSON returned from container:\n\n\t {line}"
|
||||
self.print_progress(document, True, error_message, -1)
|
||||
|
||||
def exec(
|
||||
self,
|
||||
args: List[str],
|
||||
|
|
@ -203,39 +175,45 @@ class Container(IsolationProvider):
|
|||
stdout=subprocess.PIPE,
|
||||
stderr=self.proc_stderr,
|
||||
startupinfo=startupinfo,
|
||||
# Start the conversion process in a new session, so that we can later on
|
||||
# kill the process group, without killing the controlling script.
|
||||
start_new_session=True,
|
||||
)
|
||||
|
||||
def exec_container(
|
||||
self,
|
||||
command: List[str],
|
||||
name: str,
|
||||
extra_args: List[str] = [],
|
||||
) -> subprocess.Popen:
|
||||
runtime = Runtime()
|
||||
security_args = self.get_runtime_security_args()
|
||||
debug_args = []
|
||||
if self.debug:
|
||||
debug_args += ["-e", "RUNSC_DEBUG=1"]
|
||||
container_runtime = self.get_runtime()
|
||||
|
||||
if self.get_runtime_name() == "podman":
|
||||
security_args = ["--log-driver", "none"]
|
||||
security_args += ["--security-opt", "no-new-privileges"]
|
||||
security_args += ["--userns", "keep-id"]
|
||||
else:
|
||||
security_args = ["--security-opt=no-new-privileges:true"]
|
||||
|
||||
# drop all linux kernel capabilities
|
||||
security_args += ["--cap-drop", "all"]
|
||||
user_args = ["-u", "dangerzone"]
|
||||
enable_stdin = ["-i"]
|
||||
set_name = ["--name", name]
|
||||
|
||||
prevent_leakage_args = ["--rm"]
|
||||
image_name = [
|
||||
container_utils.CONTAINER_NAME + ":" + container_utils.get_expected_tag()
|
||||
]
|
||||
|
||||
args = (
|
||||
["run"]
|
||||
["run", "--network", "none"]
|
||||
+ user_args
|
||||
+ security_args
|
||||
+ debug_args
|
||||
+ prevent_leakage_args
|
||||
+ enable_stdin
|
||||
+ set_name
|
||||
+ image_name
|
||||
+ extra_args
|
||||
+ [self.CONTAINER_NAME]
|
||||
+ command
|
||||
)
|
||||
return self.exec([str(runtime.path)] + args)
|
||||
|
||||
args = [container_runtime] + args
|
||||
return self.exec(args)
|
||||
|
||||
def kill_container(self, name: str) -> None:
|
||||
"""Terminate a spawned container.
|
||||
|
|
@ -247,32 +225,69 @@ class Container(IsolationProvider):
|
|||
connected to the Docker daemon, and killing it will just close the associated
|
||||
standard streams.
|
||||
"""
|
||||
runtime = Runtime()
|
||||
cmd = [str(runtime.path), "kill", name]
|
||||
container_runtime = self.get_runtime()
|
||||
cmd = [container_runtime, "kill", name]
|
||||
try:
|
||||
# We do not check the exit code of the process here, since the container may
|
||||
# have stopped right before invoking this command. In that case, the
|
||||
# command's output will contain some error messages, so we capture them in
|
||||
# order to silence them.
|
||||
#
|
||||
# NOTE: We specify a timeout for this command, since we've seen it hang
|
||||
# indefinitely for specific files. See:
|
||||
# https://github.com/freedomofpress/dangerzone/issues/854
|
||||
subprocess.run(
|
||||
cmd,
|
||||
capture_output=True,
|
||||
startupinfo=get_subprocess_startupinfo(),
|
||||
timeout=TIMEOUT_KILL,
|
||||
)
|
||||
except subprocess.TimeoutExpired:
|
||||
log.warning(
|
||||
f"Could not kill container '{name}' within {TIMEOUT_KILL} seconds"
|
||||
cmd, capture_output=True, startupinfo=get_subprocess_startupinfo()
|
||||
)
|
||||
except Exception as e:
|
||||
log.exception(
|
||||
f"Unexpected error occurred while killing container '{name}': {str(e)}"
|
||||
)
|
||||
|
||||
def pixels_to_pdf(
|
||||
self, document: Document, tempdir: str, ocr_lang: Optional[str]
|
||||
) -> None:
|
||||
# Convert pixels to safe PDF
|
||||
command = [
|
||||
"/usr/bin/python3",
|
||||
"-m",
|
||||
"dangerzone.conversion.pixels_to_pdf",
|
||||
]
|
||||
extra_args = [
|
||||
"-v",
|
||||
f"{tempdir}:/safezone:Z",
|
||||
"-e",
|
||||
f"OCR={0 if ocr_lang is None else 1}",
|
||||
"-e",
|
||||
f"OCR_LANGUAGE={ocr_lang}",
|
||||
]
|
||||
|
||||
name = self.pixels_to_pdf_container_name(document)
|
||||
pixels_to_pdf_proc = self.exec_container(command, name, extra_args)
|
||||
if pixels_to_pdf_proc.stdout:
|
||||
for line in pixels_to_pdf_proc.stdout:
|
||||
self.parse_progress_trusted(document, line.decode())
|
||||
error_code = pixels_to_pdf_proc.wait()
|
||||
|
||||
# In case of a dev run, log everything from the second container.
|
||||
if getattr(sys, "dangerzone_dev", False):
|
||||
assert pixels_to_pdf_proc.stderr
|
||||
out = pixels_to_pdf_proc.stderr.read().decode()
|
||||
text = (
|
||||
f"Conversion output: (pixels to PDF)\n"
|
||||
f"{PIXELS_TO_PDF_LOG_START}\n{out}\n{PIXELS_TO_PDF_LOG_END}"
|
||||
)
|
||||
log.info(text)
|
||||
|
||||
if error_code != 0:
|
||||
log.error("pixels-to-pdf failed")
|
||||
raise errors.exception_from_error_code(error_code)
|
||||
else:
|
||||
# Move the final file to the right place
|
||||
if os.path.exists(document.output_filename):
|
||||
os.remove(document.output_filename)
|
||||
|
||||
container_output_filename = os.path.join(
|
||||
tempdir, "safe-output-compressed.pdf"
|
||||
)
|
||||
shutil.move(container_output_filename, document.output_filename)
|
||||
|
||||
def start_doc_to_pixels_proc(self, document: Document) -> subprocess.Popen:
|
||||
# Convert document to pixels
|
||||
command = [
|
||||
|
|
@ -292,7 +307,7 @@ class Container(IsolationProvider):
|
|||
#
|
||||
# See also https://github.com/freedomofpress/dangerzone/issues/791
|
||||
self.kill_container(self.doc_to_pixels_container_name(document))
|
||||
terminate_process_group(p)
|
||||
p.terminate()
|
||||
|
||||
def ensure_stop_doc_to_pixels_proc( # type: ignore [no-untyped-def]
|
||||
self, document: Document, *args, **kwargs
|
||||
|
|
@ -304,10 +319,10 @@ class Container(IsolationProvider):
|
|||
# after a podman kill / docker kill invocation, this will likely be the case,
|
||||
# else the container runtime (Docker/Podman) has experienced a problem, and we
|
||||
# should report it.
|
||||
runtime = Runtime()
|
||||
container_runtime = self.get_runtime()
|
||||
name = self.doc_to_pixels_container_name(document)
|
||||
all_containers = subprocess.run(
|
||||
[str(runtime.path), "ps", "-a"],
|
||||
[container_runtime, "ps", "-a"],
|
||||
capture_output=True,
|
||||
startupinfo=get_subprocess_startupinfo(),
|
||||
)
|
||||
|
|
@ -318,20 +333,19 @@ class Container(IsolationProvider):
|
|||
# FIXME hardcoded 1 until length conversions are better handled
|
||||
# https://github.com/freedomofpress/dangerzone/issues/257
|
||||
return 1
|
||||
runtime = Runtime() # type: ignore [unreachable]
|
||||
|
||||
n_cpu = 1
|
||||
n_cpu = 1 # type: ignore [unreachable]
|
||||
if platform.system() == "Linux":
|
||||
# if on linux containers run natively
|
||||
cpu_count = os.cpu_count()
|
||||
if cpu_count is not None:
|
||||
n_cpu = cpu_count
|
||||
|
||||
elif runtime.name == "docker":
|
||||
elif self.get_runtime_name() == "docker":
|
||||
# For Windows and MacOS containers run in VM
|
||||
# So we obtain the CPU count for the VM
|
||||
n_cpu_str = subprocess.check_output(
|
||||
[str(runtime.path), "info", "--format", "{{.NCPU}}"],
|
||||
[self.get_runtime(), "info", "--format", "{{.NCPU}}"],
|
||||
text=True,
|
||||
startupinfo=get_subprocess_startupinfo(),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,25 +1,19 @@
|
|||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Callable, Optional
|
||||
|
||||
from ..conversion.common import DangerzoneConverter
|
||||
from ..document import Document
|
||||
from .base import IsolationProvider, terminate_process_group
|
||||
from ..util import get_resource_path
|
||||
from .base import IsolationProvider
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def dummy_script() -> None:
|
||||
sys.stdin.buffer.read()
|
||||
pages = 2
|
||||
width = height = 9
|
||||
DangerzoneConverter._write_int(pages)
|
||||
for page in range(pages):
|
||||
DangerzoneConverter._write_int(width)
|
||||
DangerzoneConverter._write_int(height)
|
||||
DangerzoneConverter._write_bytes(width * height * 3 * b"A")
|
||||
|
||||
|
||||
class Dummy(IsolationProvider):
|
||||
"""Dummy Isolation Provider (FOR TESTING ONLY)
|
||||
|
||||
|
|
@ -34,38 +28,59 @@ class Dummy(IsolationProvider):
|
|||
"Dummy isolation provider is UNSAFE and should never be "
|
||||
+ "called in a non-testing system."
|
||||
)
|
||||
super().__init__()
|
||||
|
||||
def install(self) -> bool:
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def is_available() -> bool:
|
||||
return True
|
||||
def convert(
|
||||
self,
|
||||
document: Document,
|
||||
ocr_lang: Optional[str],
|
||||
progress_callback: Optional[Callable] = None,
|
||||
) -> None:
|
||||
self.progress_callback = None
|
||||
log.debug("Dummy converter started:")
|
||||
log.debug(
|
||||
f" - document: {os.path.basename(document.input_filename)} ({document.id})"
|
||||
)
|
||||
log.debug(f" - ocr : {ocr_lang}")
|
||||
log.debug("\n(simulating conversion)")
|
||||
success = True
|
||||
progress = [
|
||||
[False, "Converting to PDF using GraphicsMagick", 0.0],
|
||||
[False, "Separating document into pages", 3.0],
|
||||
[False, "Converting page 1/1 to pixels", 5.0],
|
||||
[False, "Converted document to pixels", 50.0],
|
||||
[False, "Converting page 1/1 from pixels to PDF", 50.0],
|
||||
[False, "Merging 1 pages into a single PDF", 95.0],
|
||||
[False, "Compressing PDF", 97.0],
|
||||
[False, "Safe PDF created", 100.0],
|
||||
]
|
||||
for error, text, percentage in progress:
|
||||
self.print_progress(document, error, text, percentage) # type: ignore [arg-type]
|
||||
if error:
|
||||
success = False
|
||||
time.sleep(0.2)
|
||||
if success:
|
||||
shutil.copy(
|
||||
get_resource_path("dummy_document.pdf"), document.output_filename
|
||||
)
|
||||
document.mark_as_safe()
|
||||
if document.archive_after_conversion:
|
||||
document.archive()
|
||||
|
||||
@staticmethod
|
||||
def should_wait_install() -> bool:
|
||||
return False
|
||||
def pixels_to_pdf(
|
||||
self, document: Document, tempdir: str, ocr_lang: Optional[str]
|
||||
) -> None:
|
||||
pass
|
||||
|
||||
def start_doc_to_pixels_proc(self, document: Document) -> subprocess.Popen:
|
||||
cmd = [
|
||||
sys.executable,
|
||||
"-c",
|
||||
"from dangerzone.isolation_provider.dummy import dummy_script;"
|
||||
" dummy_script()",
|
||||
]
|
||||
return subprocess.Popen(
|
||||
cmd,
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=self.proc_stderr,
|
||||
start_new_session=True,
|
||||
)
|
||||
return subprocess.Popen("True")
|
||||
|
||||
def terminate_doc_to_pixels_proc(
|
||||
self, document: Document, p: subprocess.Popen
|
||||
) -> None:
|
||||
terminate_process_group(p)
|
||||
pass
|
||||
|
||||
def get_max_parallel_conversions(self) -> int:
|
||||
return 1
|
||||
|
|
|
|||
|
|
@ -1,16 +1,21 @@
|
|||
import asyncio
|
||||
import inspect
|
||||
import io
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
from typing import IO
|
||||
from typing import IO, Callable, Optional
|
||||
|
||||
from ..conversion import errors
|
||||
from ..conversion.common import running_on_qubes
|
||||
from ..conversion.pixels_to_pdf import PixelsToPDF
|
||||
from ..document import Document
|
||||
from ..util import get_resource_path
|
||||
from .base import IsolationProvider
|
||||
from .base import PIXELS_TO_PDF_LOG_END, PIXELS_TO_PDF_LOG_START, IsolationProvider
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -21,19 +26,33 @@ class Qubes(IsolationProvider):
|
|||
def install(self) -> bool:
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def is_available() -> bool:
|
||||
return True
|
||||
def pixels_to_pdf(
|
||||
self, document: Document, tempdir: str, ocr_lang: Optional[str]
|
||||
) -> None:
|
||||
def print_progress_wrapper(error: bool, text: str, percentage: float) -> None:
|
||||
self.print_progress(document, error, text, percentage)
|
||||
|
||||
@staticmethod
|
||||
def should_wait_install() -> bool:
|
||||
return False
|
||||
converter = PixelsToPDF(progress_callback=print_progress_wrapper)
|
||||
try:
|
||||
asyncio.run(converter.convert(ocr_lang, tempdir))
|
||||
except (RuntimeError, ValueError) as e:
|
||||
raise errors.UnexpectedConversionError(str(e))
|
||||
finally:
|
||||
if getattr(sys, "dangerzone_dev", False):
|
||||
out = converter.captured_output.decode()
|
||||
text = (
|
||||
f"Conversion output: (pixels to PDF)\n"
|
||||
f"{PIXELS_TO_PDF_LOG_START}\n{out}{PIXELS_TO_PDF_LOG_END}"
|
||||
)
|
||||
log.info(text)
|
||||
|
||||
shutil.move(f"{tempdir}/safe-output-compressed.pdf", document.output_filename)
|
||||
|
||||
def get_max_parallel_conversions(self) -> int:
|
||||
return 1
|
||||
|
||||
def start_doc_to_pixels_proc(self, document: Document) -> subprocess.Popen:
|
||||
dev_mode = getattr(sys, "dangerzone_dev", False) is True
|
||||
dev_mode = getattr(sys, "dangerzone_dev", False) == True
|
||||
if dev_mode:
|
||||
# Use dz.ConvertDev RPC call instead, if we are in development mode.
|
||||
# Basically, the change is that we also transfer the necessary Python
|
||||
|
|
@ -49,9 +68,6 @@ class Qubes(IsolationProvider):
|
|||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=stderr,
|
||||
# Start the conversion process in a new session, so that we can later on
|
||||
# kill the process group, without killing the controlling script.
|
||||
start_new_session=True,
|
||||
)
|
||||
|
||||
if dev_mode:
|
||||
|
|
@ -78,18 +94,14 @@ class Qubes(IsolationProvider):
|
|||
standard streams explicitly, so that we can afterwards use `Popen.wait()` to
|
||||
learn if the qube terminated.
|
||||
|
||||
Note that we don't close the stderr stream because we want to read debug logs
|
||||
from it. In the rare case where a qube cannot terminate because it's stuck
|
||||
writing at stderr (this is not the expected behavior), we expect that the
|
||||
process will still be forcefully killed after the soft termination timeout
|
||||
expires.
|
||||
|
||||
[1]: https://github.com/freedomofpress/dangerzone/issues/563#issuecomment-2034803232
|
||||
"""
|
||||
if p.stdin:
|
||||
p.stdin.close()
|
||||
if p.stdout:
|
||||
p.stdout.close()
|
||||
if p.stderr:
|
||||
p.stderr.close()
|
||||
|
||||
def teleport_dz_module(self, wpipe: IO[bytes]) -> None:
|
||||
"""Send the dangerzone module to another qube, as a zipfile."""
|
||||
|
|
@ -103,6 +115,9 @@ class Qubes(IsolationProvider):
|
|||
with zipfile.ZipFile(temp_file, "w") as z:
|
||||
z.mkdir("dangerzone/")
|
||||
z.writestr("dangerzone/__init__.py", "")
|
||||
import dangerzone.conversion
|
||||
|
||||
conv_path = Path(dangerzone.conversion.__file__).parent
|
||||
for root, _, files in os.walk(_conv_path):
|
||||
for file in files:
|
||||
if file.endswith(".py"):
|
||||
|
|
@ -130,6 +145,7 @@ def is_qubes_native_conversion() -> bool:
|
|||
# This disambiguates if it is running a Qubes targetted build or not
|
||||
# (Qubes-specific builds don't ship the container image)
|
||||
|
||||
return not get_resource_path("container.tar").exists()
|
||||
compressed_container_path = get_resource_path("container.tar.gz")
|
||||
return not os.path.exists(compressed_container_path)
|
||||
else:
|
||||
return False
|
||||
|
|
|
|||
|
|
@ -1,6 +1,12 @@
|
|||
import concurrent.futures
|
||||
import gzip
|
||||
import json
|
||||
import logging
|
||||
import pathlib
|
||||
import platform
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from typing import Callable, List, Optional
|
||||
|
||||
import colorama
|
||||
|
|
@ -23,14 +29,19 @@ class DangerzoneCore(object):
|
|||
# Initialize terminal colors
|
||||
colorama.init(autoreset=True)
|
||||
|
||||
# App data folder
|
||||
self.appdata_path = util.get_config_dir()
|
||||
|
||||
# Languages supported by tesseract
|
||||
with get_resource_path("ocr-languages.json").open("r") as f:
|
||||
with open(get_resource_path("ocr-languages.json"), "r") as f:
|
||||
unsorted_ocr_languages = json.load(f)
|
||||
self.ocr_languages = dict(sorted(unsorted_ocr_languages.items()))
|
||||
|
||||
# Load settings
|
||||
self.settings = Settings()
|
||||
self.settings = Settings(self)
|
||||
|
||||
self.documents: List[Document] = []
|
||||
|
||||
self.isolation_provider = isolation_provider
|
||||
|
||||
def add_document_from_filename(
|
||||
|
|
@ -62,18 +73,11 @@ class DangerzoneCore(object):
|
|||
self, ocr_lang: Optional[str], stdout_callback: Optional[Callable] = None
|
||||
) -> None:
|
||||
def convert_doc(document: Document) -> None:
|
||||
try:
|
||||
self.isolation_provider.convert(
|
||||
document,
|
||||
ocr_lang,
|
||||
stdout_callback,
|
||||
)
|
||||
|
||||
except Exception:
|
||||
log.exception(
|
||||
f"Unexpected error occurred while converting '{document}'"
|
||||
)
|
||||
document.mark_as_failed()
|
||||
self.isolation_provider.convert(
|
||||
document,
|
||||
ocr_lang,
|
||||
stdout_callback,
|
||||
)
|
||||
|
||||
max_jobs = self.isolation_provider.get_max_parallel_conversions()
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=max_jobs) as executor:
|
||||
|
|
|
|||
|
|
@ -1,24 +1,29 @@
|
|||
import json
|
||||
import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any, Dict
|
||||
from typing import TYPE_CHECKING, Any, Dict, Optional
|
||||
|
||||
from packaging import version
|
||||
|
||||
from .document import SAFE_EXTENSION
|
||||
from .util import get_config_dir, get_version
|
||||
from .util import get_version
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .logic import DangerzoneCore
|
||||
|
||||
SETTINGS_FILENAME: str = "settings.json"
|
||||
|
||||
|
||||
class Settings:
|
||||
settings: Dict[str, Any]
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.settings_filename = get_config_dir() / SETTINGS_FILENAME
|
||||
def __init__(self, dangerzone: "DangerzoneCore") -> None:
|
||||
self.dangerzone = dangerzone
|
||||
self.settings_filename = os.path.join(
|
||||
self.dangerzone.appdata_path, SETTINGS_FILENAME
|
||||
)
|
||||
self.default_settings: Dict[str, Any] = self.generate_default_settings()
|
||||
self.load()
|
||||
|
||||
|
|
@ -40,22 +45,6 @@ class Settings:
|
|||
"updater_errors": 0,
|
||||
}
|
||||
|
||||
def custom_runtime_specified(self) -> bool:
|
||||
return "container_runtime" in self.settings
|
||||
|
||||
def set_custom_runtime(self, runtime: str, autosave: bool = False) -> Path:
|
||||
from .container_utils import Runtime # Avoid circular import
|
||||
|
||||
container_runtime = Runtime.path_from_name(runtime)
|
||||
self.settings["container_runtime"] = str(container_runtime)
|
||||
if autosave:
|
||||
self.save()
|
||||
return container_runtime
|
||||
|
||||
def unset_custom_runtime(self) -> None:
|
||||
self.settings.pop("container_runtime")
|
||||
self.save()
|
||||
|
||||
def get(self, key: str) -> Any:
|
||||
return self.settings[key]
|
||||
|
||||
|
|
@ -90,7 +79,7 @@ class Settings:
|
|||
if version.parse(get_version()) > version.parse(self.get(key)):
|
||||
self.set(key, get_version())
|
||||
|
||||
except Exception:
|
||||
except:
|
||||
log.error("Error loading settings, falling back to default")
|
||||
self.settings = self.default_settings
|
||||
|
||||
|
|
@ -102,6 +91,6 @@ class Settings:
|
|||
self.save()
|
||||
|
||||
def save(self) -> None:
|
||||
self.settings_filename.parent.mkdir(parents=True, exist_ok=True)
|
||||
with self.settings_filename.open("w") as settings_file:
|
||||
os.makedirs(self.dangerzone.appdata_path, exist_ok=True)
|
||||
with open(self.settings_filename, "w") as settings_file:
|
||||
json.dump(self.settings, settings_file, indent=4)
|
||||
|
|
|
|||
|
|
@ -1,76 +1,54 @@
|
|||
import pathlib
|
||||
import platform
|
||||
import string
|
||||
import subprocess
|
||||
import sys
|
||||
import traceback
|
||||
import unicodedata
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
try:
|
||||
import platformdirs
|
||||
except ImportError:
|
||||
import appdirs as platformdirs
|
||||
import appdirs
|
||||
|
||||
|
||||
def get_config_dir() -> Path:
|
||||
return Path(platformdirs.user_config_dir("dangerzone"))
|
||||
def get_config_dir() -> str:
|
||||
return appdirs.user_config_dir("dangerzone")
|
||||
|
||||
|
||||
def get_resource_path(filename: str) -> Path:
|
||||
def get_tmp_dir() -> Optional[str]:
|
||||
"""Get the parent dir for the Dangerzone temporary dirs.
|
||||
|
||||
This function returns the parent directory where Dangerzone will store its temporary
|
||||
directories. The default behavior is to let Python choose for us (e.g., in `/tmp`
|
||||
for Linux), which is why we return None. However, we still need to define this
|
||||
function in order to be able to set this dir via mocking in our tests.
|
||||
"""
|
||||
return None
|
||||
|
||||
|
||||
def get_resource_path(filename: str) -> str:
|
||||
if getattr(sys, "dangerzone_dev", False):
|
||||
# Look for resources directory relative to python file
|
||||
project_root = Path(__file__).parent.parent
|
||||
prefix = project_root / "share"
|
||||
project_root = pathlib.Path(__file__).parent.parent
|
||||
prefix = project_root.joinpath("share")
|
||||
else:
|
||||
if platform.system() == "Darwin":
|
||||
bin_path = Path(sys.executable)
|
||||
bin_path = pathlib.Path(sys.executable)
|
||||
app_path = bin_path.parent.parent
|
||||
prefix = app_path / "Resources" / "share"
|
||||
prefix = app_path.joinpath("Resources", "share")
|
||||
elif platform.system() == "Linux":
|
||||
prefix = Path(sys.prefix) / "share" / "dangerzone"
|
||||
prefix = pathlib.Path(sys.prefix).joinpath("share", "dangerzone")
|
||||
elif platform.system() == "Windows":
|
||||
exe_path = Path(sys.executable)
|
||||
exe_path = pathlib.Path(sys.executable)
|
||||
dz_install_path = exe_path.parent
|
||||
prefix = dz_install_path / "share"
|
||||
prefix = dz_install_path.joinpath("share")
|
||||
else:
|
||||
raise NotImplementedError(f"Unsupported system {platform.system()}")
|
||||
return prefix / filename
|
||||
|
||||
|
||||
def get_tessdata_dir() -> Path:
|
||||
if getattr(sys, "dangerzone_dev", False) or platform.system() in (
|
||||
"Windows",
|
||||
"Darwin",
|
||||
):
|
||||
# Always use the tessdata path from the Dangerzone ./share directory, for
|
||||
# development builds, or in Windows/macOS platforms.
|
||||
return get_resource_path("tessdata")
|
||||
|
||||
# In case of Linux systems, grab the Tesseract data from any of the following
|
||||
# locations. We have found some of the locations through trial and error, whereas
|
||||
# others are taken from the docs:
|
||||
#
|
||||
# [...] Possibilities are /usr/share/tesseract-ocr/tessdata or
|
||||
# /usr/share/tessdata or /usr/share/tesseract-ocr/4.00/tessdata. [1]
|
||||
#
|
||||
# [1] https://tesseract-ocr.github.io/tessdoc/Installation.html
|
||||
tessdata_dirs = [
|
||||
Path("/usr/share/tessdata/"), # on some Debian
|
||||
Path("/usr/share/tesseract/tessdata/"), # on Fedora
|
||||
Path("/usr/share/tesseract-ocr/tessdata/"), # ? (documented)
|
||||
Path("/usr/share/tesseract-ocr/4.00/tessdata/"), # on Debian Bullseye
|
||||
Path("/usr/share/tesseract-ocr/5/tessdata/"), # on Debian Trixie
|
||||
]
|
||||
|
||||
for dir in tessdata_dirs:
|
||||
if dir.is_dir():
|
||||
return dir
|
||||
|
||||
raise RuntimeError("Tesseract language data are not installed in the system")
|
||||
resource_path = prefix.joinpath(filename)
|
||||
return str(resource_path)
|
||||
|
||||
|
||||
def get_version() -> str:
|
||||
try:
|
||||
with get_resource_path("version.txt").open() as f:
|
||||
with open(get_resource_path("version.txt")) as f:
|
||||
version = f.read().strip()
|
||||
except FileNotFoundError:
|
||||
# In dev mode, in Windows, get_resource_path doesn't work properly for the container, but luckily
|
||||
|
|
@ -120,13 +98,3 @@ def replace_control_chars(untrusted_str: str, keep_newlines: bool = False) -> st
|
|||
else:
|
||||
sanitized_str += "<EFBFBD>"
|
||||
return sanitized_str
|
||||
|
||||
|
||||
def format_exception(e: Exception) -> str:
|
||||
# The signature of traceback.format_exception has changed in python 3.10
|
||||
if sys.version_info < (3, 10):
|
||||
output = traceback.format_exception(*sys.exc_info())
|
||||
else:
|
||||
output = traceback.format_exception(e)
|
||||
|
||||
return "".join(output)
|
||||
|
|
|
|||
29
debian/changelog
vendored
29
debian/changelog
vendored
|
|
@ -1,29 +0,0 @@
|
|||
dangerzone (0.9.0) unstable; urgency=low
|
||||
|
||||
* Released Dangerzone 0.9.0
|
||||
|
||||
-- Freedom of the Press Foundation <info@freedom.press> Mon, 31 Mar 2025 15:57:18 +0300
|
||||
|
||||
dangerzone (0.8.1) unstable; urgency=low
|
||||
|
||||
* Released Dangerzone 0.8.1
|
||||
|
||||
-- Freedom of the Press Foundation <info@freedom.press> Tue, 22 Dec 2024 22:03:28 +0300
|
||||
|
||||
dangerzone (0.8.0) unstable; urgency=low
|
||||
|
||||
* Released Dangerzone 0.8.0
|
||||
|
||||
-- Freedom of the Press Foundation <info@freedom.press> Tue, 30 Oct 2024 01:56:28 +0300
|
||||
|
||||
dangerzone (0.7.1) unstable; urgency=low
|
||||
|
||||
* Released Dangerzone 0.7.1
|
||||
|
||||
-- Freedom of the Press Foundation <info@freedom.press> Tue, 1 Oct 2024 17:02:28 +0300
|
||||
|
||||
dangerzone (0.7.0) unstable; urgency=low
|
||||
|
||||
* Removed stdeb in favor of direct debian packaging tools
|
||||
|
||||
-- Freedom of the Press Foundation <info@freedom.press> Tue, 27 Aug 2024 14:39:28 +0200
|
||||
1
debian/compat
vendored
1
debian/compat
vendored
|
|
@ -1 +0,0 @@
|
|||
10
|
||||
15
debian/control
vendored
15
debian/control
vendored
|
|
@ -1,15 +0,0 @@
|
|||
Source: dangerzone
|
||||
Maintainer: Freedom of the Press Foundation <info@freedom.press>
|
||||
Section: python
|
||||
Priority: optional
|
||||
Build-Depends: dh-python, python3-setuptools, python3, dpkg-dev, debhelper (>= 9)
|
||||
Standards-Version: 4.5.1
|
||||
Homepage: https://github.com/freedomofpress/dangerzone
|
||||
Rules-Requires-Root: no
|
||||
|
||||
Package: dangerzone
|
||||
Architecture: any
|
||||
Depends: ${misc:Depends}, podman, python3, python3-pyside2.qtcore, python3-pyside2.qtgui, python3-pyside2.qtwidgets, python3-pyside2.qtsvg, python3-platformdirs | python3-appdirs, python3-click, python3-xdg, python3-colorama, python3-requests, python3-markdown, python3-packaging, tesseract-ocr-all
|
||||
Description: Take potentially dangerous PDFs, office documents, or images
|
||||
Dangerzone is an open source desktop application that takes potentially dangerous PDFs, office documents, or images and converts them to safe PDFs. It uses disposable VMs on Qubes OS, or container technology in other OSes, to convert the documents within a secure sandbox.
|
||||
.
|
||||
8
debian/copyright
vendored
8
debian/copyright
vendored
|
|
@ -1,8 +0,0 @@
|
|||
Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
|
||||
Upstream-Name: dangerzone
|
||||
Source: https://github.com/freedomofpress/dangerzone
|
||||
|
||||
Files: *
|
||||
Copyright: 2020-2021 First Look Media
|
||||
2022- Freedom of the Press Foundation, and Dangerzone contributors
|
||||
License: AGPL-3.0-or-later
|
||||
13
debian/rules
vendored
13
debian/rules
vendored
|
|
@ -1,13 +0,0 @@
|
|||
#!/usr/bin/make -f
|
||||
export PYBUILD_NAME=dangerzone
|
||||
export DEB_BUILD_OPTIONS=nocheck
|
||||
export PYBUILD_INSTALL_ARGS=--install-lib=/usr/lib/python3/dist-packages
|
||||
export PYTHONDONTWRITEBYTECODE=1
|
||||
export DH_VERBOSE=1
|
||||
|
||||
%:
|
||||
dh $@ --with python3 --buildsystem=pybuild
|
||||
|
||||
override_dh_builddeb:
|
||||
./install/linux/debian-vendor-pymupdf.py --dest debian/dangerzone/usr/lib/python3/dist-packages/dangerzone/vendor/
|
||||
dh_builddeb $@
|
||||
1
debian/source/format
vendored
1
debian/source/format
vendored
|
|
@ -1 +0,0 @@
|
|||
3.0 (native)
|
||||
7
debian/source/options
vendored
7
debian/source/options
vendored
|
|
@ -1,7 +0,0 @@
|
|||
compression = "gzip"
|
||||
tar-ignore = "dev_scripts"
|
||||
tar-ignore = ".*"
|
||||
tar-ignore = "__pycache__"
|
||||
# Ignore the 'share/tessdata' dir, since it slows down the process, and we
|
||||
# install Tesseract data via Debian packages anyway.
|
||||
tar-ignore = "share/tessdata"
|
||||
|
|
@ -1,7 +1,153 @@
|
|||
# Developer scripts
|
||||
|
||||
This directory holds some scripts that are helpful for developing on Dangerzone.
|
||||
Read the respective documentation for more details on some of the scripts.
|
||||
Read below for more details on these scripts.
|
||||
|
||||
* [`env.py`](../docs/developer/environments.md)
|
||||
* [`qa.py`](../docs/developer/qa.md)
|
||||
## Create Dangerzone environments (`env.py`)
|
||||
|
||||
This script creates environments where a user can run Dangerzone, allows the
|
||||
user to run arbitrary commands in these environments, as well as run Dangerzone
|
||||
(nested containerization).
|
||||
|
||||
It supports two types of environments:
|
||||
|
||||
1. Dev environment. This environment has developer tools, necessary for
|
||||
Dangerzone, baked in. Also, it mounts the Dangerzone source under
|
||||
`/home/user/dangerzone` in the container. The developer can then run
|
||||
Dangerzone from source, with `poetry run ./dev_scripts/dangerzone`.
|
||||
2. End-user environment. This environment has only Dangerzone installed in it,
|
||||
from the .deb/.rpm package that we have created. For convenience, it also has
|
||||
the Dangerzone source mounted under `/home/user/dangerzone`, but it lacks
|
||||
Poetry and other build tools. The developer can run Dangerzone there with
|
||||
`dangerzone`. This environment is the most vanilla Dangerzone environment,
|
||||
and should be closer to the end user's environment, than the development
|
||||
environment.
|
||||
|
||||
Each environment corresponds to a Dockerfile, which is generated on the fly. The
|
||||
developer can see this Dockerfile by passing `--show-dockerfile`.
|
||||
|
||||
For usage information, run `./dev_scripts/env.py --help`.
|
||||
|
||||
### Nested containerization
|
||||
|
||||
Since the Dangerzone environments are containers, this means that the Podman
|
||||
containers that Dangerzone creates have to be nested containers. This has some
|
||||
challenges that we will highlight below:
|
||||
|
||||
1. Containers typically only have a subset of syscalls allowed, and sometimes
|
||||
only for specific arguments. This happens with the use of
|
||||
[seccomp filters](https://docs.docker.com/engine/security/seccomp/). For
|
||||
instance, in Docker, the `clone` syscall is limited in containers and cannot
|
||||
create new namespaces
|
||||
(https://docs.docker.com/engine/security/seccomp/#significant-syscalls-blocked-by-the-default-profile). For testing/development purposes, we can get around this limitation
|
||||
by disabling the seccomp filters for the external container with
|
||||
`--security-opt seccomp=unconfined`. This has the same effect as developing
|
||||
Dangerzone locally, so it should probably be sufficient for now.
|
||||
|
||||
2. While Linux supports nested namespaces, we need extra handling for nested
|
||||
user namespaces. By default, the configuration for each user namespace (see
|
||||
[`man login.defs`](https://man7.org/linux/man-pages/man5/login.defs.5.html)
|
||||
is to reserve 65536 UIDs/GIDs, starting from UID/GID 100000. This works fine
|
||||
for the first container, but can't work for the nested container, since it
|
||||
doesn't have enough UIDs/GIDs to refer to UID 100000. Our solution to this is
|
||||
to restrict the number of UIDs/GIDs allowed in the nested container to 2000,
|
||||
which should be enough to run `podman` in it.
|
||||
|
||||
3. Containers also restrict the capabilities (see
|
||||
[`man capabilities`](https://man7.org/linux/man-pages/man7/capabilities.7.html))
|
||||
of the processes that run in them. By default, containers do not have mount
|
||||
capabilities, since it requires `CAP_SYS_ADMIN`, which effectively
|
||||
[makes the process root](https://lwn.net/Articles/486306/) in the specific
|
||||
user namespace. In our case, we have to give the Dangerzone environment this
|
||||
capability, since it will have to mount directories in Podman containers. For
|
||||
this reason, as well as some extra things we bumped into during development,
|
||||
we pass `--privileged` when creating the Dangerzone environment, which
|
||||
includes the `CAP_SYS_ADMIN` capability.
|
||||
|
||||
### GUI containerization
|
||||
|
||||
Running a GUI app in a container is a tricky subject for multi-platform apps. In
|
||||
our case, we deal specifically with Linux environments, so we can target just
|
||||
this platform.
|
||||
|
||||
To understand how a GUI app can draw in the user's screen from within a
|
||||
container, we must first understand how it does so outside the container. In
|
||||
Unix-like systems, GUI apps act like
|
||||
[clients to a display server](https://wayland.freedesktop.org/architecture.html).
|
||||
The most common display server implementation is X11, and the runner-up is
|
||||
Wayland. Both of these display servers share some common traits, mainly that
|
||||
they use Unix domain sockets as a way of letting clients communicate with them.
|
||||
|
||||
So, this gives us the answer on how one can run a containerized GUI app; they
|
||||
can simply mount the Unix Domain Socket in the container. In practice this is
|
||||
more nuanced, for two reasons:
|
||||
|
||||
1. Wayland support is not that mature on Linux, so we need to
|
||||
[set some extra environment variables](https://github.com/mviereck/x11docker/wiki/How-to-provide-Wayland-socket-to-docker-container). To simplify things, we will target
|
||||
X11 / XWayland hosts, which are the majority of the Linux OSes out there.
|
||||
2. Sharing the Unix Domain socket does not allow the client to talk to the
|
||||
display server, for security reasons. In order to allow the client, we need
|
||||
to mount a magic cookie stored in a file pointed at by the `$XAUTHORITY`
|
||||
envvar. Else, we can use `xhost`, which is considered slightly more dangerous
|
||||
for multi-user environments.
|
||||
|
||||
### Caching and Reproducibility
|
||||
|
||||
In order to build Dangerzone environments, the script uses the following inputs:
|
||||
|
||||
* Dev environment:
|
||||
- Distro name and version. Together, these comprise the base container image.
|
||||
- `poetry.lock` and `pyproject.toml`. Together, these comprise the build
|
||||
context.
|
||||
* End-user environment:
|
||||
- Distro name and version. Together, these comprise the base container image.
|
||||
- `.deb` / `.rpm` Dangerzone package, as found under `deb_dist/` or `dist/`
|
||||
respectively.
|
||||
|
||||
Any change in these inputs busts the cache for the corresponding image. In
|
||||
theory, this means that the Dangerzone environment for each commit can be built
|
||||
reproducibly. In practice, there are some issues that we haven't covered yet:
|
||||
|
||||
1. The output images are:
|
||||
* Dev: `dangerzone.rocks/build/{distro_name}:{distro_version}`
|
||||
* End-user: `dangerzone.rocks/{distro_name}:{distro_version}`
|
||||
|
||||
These images do not contain the commit/version of the Dangerzone source they
|
||||
got created from, so each one overrides the other.
|
||||
2. The end-user environment expects a `.deb.` / `.rpm` tagged with the version
|
||||
of Dangerzone, but it doesn't insist being built from the current Dangerzone
|
||||
commit. This means that stale packages may be installed in the end-user
|
||||
environment.
|
||||
3. The base images may be different in various environments, depending on when
|
||||
they where pulled.
|
||||
|
||||
### State
|
||||
|
||||
The main goal behind these Dangerzone environments is to make them immutable,
|
||||
so that they do not require to be stored somewhere, but can be recreated from
|
||||
their images. Any change to these environments should therefore be reflected to
|
||||
their Dockerfile.
|
||||
|
||||
To enforce immutability, we delete the containers every time we run a command or
|
||||
an interactive shell exits. This means that these environments are suitable only
|
||||
for running Dangerzone commands, and not doing actual development in them
|
||||
(install an editor, configure bash prompts, etc.)
|
||||
|
||||
The only point where we allow mutability is the directory where Podman stores
|
||||
the images and stopped containers, which may be useful for developers. If this
|
||||
proves to be an issue, we will reconsider.
|
||||
|
||||
## Run QA (`qa.py`)
|
||||
|
||||
This script runs the QA steps for a supported platform, in order to make sure
|
||||
that the dev does not skip something. These steps are taken from our [release
|
||||
instructions](../RELEASE.md#qa).
|
||||
|
||||
The idea behind this script is that it will present each step to the user and
|
||||
ask them to perform it manually and specify it passes, in order to continue to
|
||||
the next one. For specific steps, it allows the user to run them automatically.
|
||||
In steps that require a Dangerzone dev environment, this script uses the
|
||||
`env.py` script to create one.
|
||||
|
||||
Including all the supported platforms in this script is still a work in
|
||||
progress.
|
||||
|
|
|
|||
|
|
@ -1,3 +0,0 @@
|
|||
[engine]
|
||||
cgroup_manager="cgroupfs"
|
||||
events_logger="file"
|
||||
|
|
@ -1,33 +1,64 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import functools
|
||||
import os
|
||||
import pathlib
|
||||
import platform
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from datetime import date
|
||||
import urllib.request
|
||||
|
||||
DEFAULT_GUI = True
|
||||
DEFAULT_USER = "user"
|
||||
DEFAULT_DRY = False
|
||||
DEFAULT_DEV = False
|
||||
DEFAULT_SHOW_DOCKERFILE = False
|
||||
DEFAULT_DOWNLOAD_PYSIDE6 = False
|
||||
|
||||
PYSIDE6_VERSION = "6.6.3.1"
|
||||
PYSIDE6_RPM = "python3-pyside6-{pyside6_version}-1.fc{fedora_version}.x86_64.rpm"
|
||||
PYSIDE6_URL = (
|
||||
"https://packages.freedom.press/yum-tools-prod/dangerzone/f{fedora_version}/%s"
|
||||
% PYSIDE6_RPM
|
||||
)
|
||||
|
||||
PYSIDE6_DL_MESSAGE = """\
|
||||
Downloading PySide6 RPM from:
|
||||
|
||||
{pyside6_url}
|
||||
|
||||
into the following local path:
|
||||
|
||||
{pyside6_local_path}
|
||||
|
||||
The RPM is over 100 MB, so this operation may take a while...
|
||||
"""
|
||||
|
||||
PYSIDE6_NOT_FOUND_ERROR = """\
|
||||
The following package is not present in your system:
|
||||
|
||||
{pyside6_local_path}
|
||||
|
||||
You can build it locally and copy it in the expected path, following the instructions
|
||||
in:
|
||||
|
||||
https://github.com/freedomofpress/python3-pyside6-rpm
|
||||
|
||||
Alternatively, you can rerun the command adding the '--download-pyside6' flag, which
|
||||
will download it from:
|
||||
|
||||
{pyside6_url}
|
||||
"""
|
||||
|
||||
# The Linux distributions that we currently support.
|
||||
# FIXME: Add a version mapping to avoid mistakes.
|
||||
# FIXME: Maybe create an enum for these values.
|
||||
DISTROS = ["debian", "fedora", "ubuntu"]
|
||||
CONTAINER_RUNTIMES = ["podman", "docker"]
|
||||
IMAGES_REGISTRY = "ghcr.io/freedomofpress/"
|
||||
IMAGE_NAME_BUILD_DEV_FMT = (
|
||||
IMAGES_REGISTRY + "v2/dangerzone/build-dev/{distro}-{version}:{date}-{hash}"
|
||||
)
|
||||
IMAGE_NAME_BUILD_ENDUSER_FMT = (
|
||||
IMAGES_REGISTRY + "v2/dangerzone/end-user/{distro}-{version}:{date}-{hash}"
|
||||
)
|
||||
IMAGE_NAME_BUILD_DEV_FMT = "dangerzone.rocks/build/{distro}:{version}"
|
||||
IMAGE_NAME_BUILD_FMT = "dangerzone.rocks/{distro}:{version}"
|
||||
|
||||
EPILOG = """\
|
||||
Examples:
|
||||
|
|
@ -52,7 +83,7 @@ Run Dangerzone in the development environment:
|
|||
|
||||
env.py --distro ubuntu --version 22.04 run --dev bash
|
||||
user@dangerzone-dev:~$ cd dangerzone/
|
||||
user@dangerzone-dev:~$ poetry run ./dev_scripts/dangerzone
|
||||
user@dangerzone-dev:~$ poetry run ./dev/scripts/dangerzone
|
||||
|
||||
Run Dangerzone in the end-user environment:
|
||||
|
||||
|
|
@ -60,6 +91,24 @@ Run Dangerzone in the end-user environment:
|
|||
|
||||
"""
|
||||
|
||||
# NOTE: For Ubuntu 20.04 specifically, we need to install some extra deps, mainly for
|
||||
# Podman. This needs to take place both in our dev and end-user environment. See the
|
||||
# corresponding note in our Installation section:
|
||||
#
|
||||
# https://github.com/freedomofpress/dangerzone/blob/main/INSTALL.md#ubuntu-debian
|
||||
DOCKERFILE_UBUNTU_2004_DEPS = r"""
|
||||
ARG DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y python-all curl wget gnupg2 \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
RUN . /etc/os-release \
|
||||
&& sh -c "echo 'deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_$VERSION_ID/ /' \
|
||||
> /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list" \
|
||||
&& wget -nv https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable/xUbuntu_$VERSION_ID/Release.key -O- \
|
||||
| apt-key add -
|
||||
"""
|
||||
|
||||
# XXX: overcome the fact that ubuntu images (starting on 23.04) ship with the 'ubuntu'
|
||||
# user by default https://bugs.launchpad.net/cloud-images/+bug/2005129
|
||||
# Related issue https://github.com/freedomofpress/dangerzone/pull/461
|
||||
|
|
@ -90,31 +139,36 @@ ARG DEBIAN_FRONTEND=noninteractive
|
|||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends podman uidmap slirp4netns \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y passt || echo "Skipping installation of passt package" \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends dh-python make build-essential \
|
||||
git {qt_deps} pipx python3 python3-pip python3-venv dpkg-dev debhelper python3-setuptools \
|
||||
python3-dev \
|
||||
git fakeroot {qt_deps} pipx python3 python3-dev python3-venv python3-stdeb \
|
||||
python3-all \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
RUN pipx install poetry
|
||||
# NOTE: `pipx install poetry` fails on Ubuntu Focal, when installed through APT. By
|
||||
# installing the latest version, we sidestep this issue.
|
||||
RUN bash -c 'if [[ "$(pipx --version)" < "1" ]]; then \
|
||||
apt-get update \
|
||||
&& apt-get remove -y pipx \
|
||||
&& apt-get install -y --no-install-recommends python3-pip \
|
||||
&& pip install pipx \
|
||||
&& rm -rf /var/lib/apt/lists/*; \
|
||||
else true; fi'
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends mupdf thunar \
|
||||
&& apt-get install -y --no-install-recommends mupdf \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
"""
|
||||
|
||||
# FIXME: Install Poetry on Fedora via package manager.
|
||||
DOCKERFILE_BUILD_DEV_FEDORA_DEPS = r"""
|
||||
RUN dnf install -y git rpm-build podman python3 python3-devel python3-poetry-core \
|
||||
pipx make qt6-qtbase-gui gcc gcc-c++\
|
||||
pipx make qt6-qtbase-gui \
|
||||
&& dnf clean all
|
||||
|
||||
# FIXME: Drop this fix after it's resolved upstream.
|
||||
# See https://github.com/freedomofpress/dangerzone/issues/286#issuecomment-1347149783
|
||||
RUN rpm --restore shadow-utils
|
||||
|
||||
RUN dnf install -y mupdf thunar && dnf clean all
|
||||
RUN dnf install -y mupdf && dnf clean all
|
||||
"""
|
||||
|
||||
# The Dockerfile for building a development environment for Dangerzone. Parts of the
|
||||
|
|
@ -150,7 +204,6 @@ COPY storage.conf /home/user/.config/containers
|
|||
# FIXME: pipx install poetry does not work for Ubuntu Focal.
|
||||
ENV PATH="$PATH:/home/user/.local/bin"
|
||||
RUN pipx install poetry
|
||||
RUN pipx inject poetry poetry-plugin-export
|
||||
|
||||
COPY pyproject.toml poetry.lock /home/user/dangerzone/
|
||||
RUN cd /home/user/dangerzone && poetry --no-ansi install
|
||||
|
|
@ -159,12 +212,17 @@ RUN cd /home/user/dangerzone && poetry --no-ansi install
|
|||
DOCKERFILE_BUILD_DEBIAN_DEPS = r"""
|
||||
ARG DEBIAN_FRONTEND=noninteractive
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends mupdf thunar \
|
||||
&& apt-get install -y --no-install-recommends mupdf \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
"""
|
||||
|
||||
DOCKERFILE_BUILD_FEDORA_39_DEPS = r"""
|
||||
COPY {pyside6_rpm} /tmp/pyside6.rpm
|
||||
RUN dnf install -y /tmp/pyside6.rpm
|
||||
"""
|
||||
|
||||
DOCKERFILE_BUILD_FEDORA_DEPS = r"""
|
||||
RUN dnf install -y mupdf thunar && dnf clean all
|
||||
RUN dnf install -y mupdf && dnf clean all
|
||||
|
||||
# FIXME: Drop this fix after it's resolved upstream.
|
||||
# See https://github.com/freedomofpress/dangerzone/issues/286#issuecomment-1347149783
|
||||
|
|
@ -217,27 +275,9 @@ def git_root():
|
|||
return pathlib.Path(path)
|
||||
|
||||
|
||||
def user_data():
|
||||
"""Get the user data dir in (which differs on different OSes)"""
|
||||
home = pathlib.Path.home()
|
||||
system = platform.system()
|
||||
|
||||
if system == "Windows":
|
||||
return home / "AppData" / "Local"
|
||||
elif system == "Linux":
|
||||
return home / ".local" / "share"
|
||||
elif system == "Darwin":
|
||||
return home / "Library" / "Application Support"
|
||||
|
||||
|
||||
def dz_dev_root():
|
||||
"""Get the directory where we will store dangerzone-dev related files"""
|
||||
return user_data() / "dangerzone-dev"
|
||||
|
||||
|
||||
def distro_root(distro, version):
|
||||
"""Get the root directory for the specific Linux environment."""
|
||||
return dz_dev_root() / "envs" / distro / version
|
||||
return git_root() / f"dev_scripts/envs/{distro}/{version}"
|
||||
|
||||
|
||||
def distro_state(distro, version):
|
||||
|
|
@ -250,46 +290,14 @@ def distro_build(distro, version):
|
|||
return distro_root(distro, version) / "build"
|
||||
|
||||
|
||||
def get_current_date():
|
||||
return date.today().strftime("%Y-%m-%d")
|
||||
|
||||
|
||||
def get_build_dir_sources(distro, version):
|
||||
"""Return the files needed to build an image."""
|
||||
sources = [
|
||||
git_root() / "pyproject.toml",
|
||||
git_root() / "poetry.lock",
|
||||
git_root() / "dev_scripts" / "env.py",
|
||||
git_root() / "dev_scripts" / "storage.conf",
|
||||
git_root() / "dev_scripts" / "containers.conf",
|
||||
]
|
||||
|
||||
if distro == "ubuntu" and version in ("22.04", "jammy"):
|
||||
sources.extend(
|
||||
[
|
||||
git_root() / "dev_scripts" / "apt-tools-prod.pref",
|
||||
git_root() / "dev_scripts" / "apt-tools-prod.sources",
|
||||
]
|
||||
)
|
||||
return sources
|
||||
|
||||
|
||||
def image_name_build_dev(distro, version):
|
||||
def image_name_build(distro, version):
|
||||
"""Get the container image for the dev variant of a Dangerzone environment."""
|
||||
hash = hash_files(get_build_dir_sources(distro, version))
|
||||
|
||||
return IMAGE_NAME_BUILD_DEV_FMT.format(
|
||||
distro=distro, version=version, hash=hash, date=get_current_date()
|
||||
)
|
||||
return IMAGE_NAME_BUILD_DEV_FMT.format(distro=distro, version=version)
|
||||
|
||||
|
||||
def image_name_build_enduser(distro, version):
|
||||
"""Get the container image for the Dangerzone end-user environment."""
|
||||
|
||||
hash = hash_files(get_files_in("install/linux", "debian"))
|
||||
return IMAGE_NAME_BUILD_ENDUSER_FMT.format(
|
||||
distro=distro, version=version, hash=hash, date=get_current_date()
|
||||
)
|
||||
def image_name_install(distro, version):
|
||||
"""Get the container image for the Dangerzone environment."""
|
||||
return IMAGE_NAME_BUILD_FMT.format(distro=distro, version=version)
|
||||
|
||||
|
||||
def dz_version():
|
||||
|
|
@ -298,23 +306,71 @@ def dz_version():
|
|||
return f.read().strip()
|
||||
|
||||
|
||||
def hash_files(file_paths: list[pathlib.Path]) -> str:
|
||||
"""Returns the hash value of a list of files using the sha256 hashing algorithm."""
|
||||
hash_obj = hashlib.new("sha256")
|
||||
for path in file_paths:
|
||||
with open(path, "rb") as file:
|
||||
file_data = file.read()
|
||||
hash_obj.update(file_data)
|
||||
class PySide6Manager:
|
||||
"""Provision PySide6 RPMs in our Dangerzone environments.
|
||||
|
||||
return hash_obj.hexdigest()
|
||||
This class holds all the logic around checking and downloading PySide RPMs. It can
|
||||
check if the required RPM version is present under "/dist", and optionally download
|
||||
it.
|
||||
"""
|
||||
|
||||
def __init__(self, distro_name, distro_version):
|
||||
if distro_name != "fedora":
|
||||
raise RuntimeError("Managing PySide6 RPMs is available only in Fedora")
|
||||
self.distro_name = distro_name
|
||||
self.distro_version = distro_version
|
||||
|
||||
def get_files_in(*folders: list[str]) -> list[pathlib.Path]:
|
||||
"""Return the list of all files present in the given folders"""
|
||||
files = []
|
||||
for folder in folders:
|
||||
files.extend([p for p in (git_root() / folder).glob("**") if p.is_file()])
|
||||
return files
|
||||
@property
|
||||
def version(self):
|
||||
"""The version of the PySide6 RPM."""
|
||||
return PYSIDE6_VERSION
|
||||
|
||||
@property
|
||||
def rpm_name(self):
|
||||
"""The name of the PySide6 RPM."""
|
||||
return PYSIDE6_RPM.format(
|
||||
pyside6_version=self.version, fedora_version=self.distro_version
|
||||
)
|
||||
|
||||
@property
|
||||
def rpm_url(self):
|
||||
"""The URL of the PySide6 RPM, as hosted in FPF's RPM repo."""
|
||||
return PYSIDE6_URL.format(
|
||||
pyside6_version=self.version,
|
||||
fedora_version=self.distro_version,
|
||||
)
|
||||
|
||||
@property
|
||||
def rpm_local_path(self):
|
||||
"""The local path where this script will look for the PySide6 RPM."""
|
||||
return git_root() / "dist" / self.rpm_name
|
||||
|
||||
@property
|
||||
def is_rpm_present(self):
|
||||
"""Check if PySide6 RPM is present in the user's system."""
|
||||
return self.rpm_local_path.exists()
|
||||
|
||||
def download_rpm(self):
|
||||
"""Download PySide6 from FPF's RPM repo."""
|
||||
print(
|
||||
PYSIDE6_DL_MESSAGE.format(
|
||||
pyside6_url=self.rpm_url,
|
||||
pyside6_local_path=self.rpm_local_path,
|
||||
),
|
||||
file=sys.stderr,
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(self.rpm_url) as r, open(
|
||||
self.rpm_local_path, "wb"
|
||||
) as f:
|
||||
shutil.copyfileobj(r, f)
|
||||
except:
|
||||
# NOTE: We purposefully catch all exceptions, since we want to catch Ctrl-C
|
||||
# as well.
|
||||
print("Download interrupted, removing file", file=sys.stderr)
|
||||
self.rpm_local_path.unlink()
|
||||
raise
|
||||
print("PySide6 was downloaded successfully", file=sys.stderr)
|
||||
|
||||
|
||||
class Env:
|
||||
|
|
@ -357,28 +413,6 @@ class Env:
|
|||
"""Create an Env class from CLI arguments"""
|
||||
return cls(distro=args.distro, version=args.version, runtime=args.runtime)
|
||||
|
||||
def find_dz_package(self, path, pattern):
|
||||
"""Get the full path of the Dangerzone package in the specified dir.
|
||||
|
||||
There are times where we don't know the exact name of the Dangerzone package
|
||||
that we've built, e.g., because its patch level may have changed.
|
||||
|
||||
Auto-detect the Dangerzone package based on a pattern that a user has provided,
|
||||
and fail if there are none, or multiple matches. If there's a single match, then
|
||||
return the full path for the package.
|
||||
"""
|
||||
matches = list(path.glob(pattern))
|
||||
if len(matches) == 0:
|
||||
raise RuntimeError(
|
||||
f"Could not find Dangerzone package '{pattern}' in '{path}'"
|
||||
)
|
||||
elif len(matches) > 1:
|
||||
raise RuntimeError(
|
||||
f"Found more than one matches for Dangerzone package '{pattern}' in"
|
||||
f" '{path}'"
|
||||
)
|
||||
return matches[0]
|
||||
|
||||
def runtime_run(self, *args):
|
||||
"""Run a command for a specific container runtime.
|
||||
|
||||
|
|
@ -480,13 +514,13 @@ class Env:
|
|||
run_cmd += [
|
||||
"--hostname",
|
||||
"dangerzone-dev",
|
||||
image_name_build_dev(self.distro, self.version),
|
||||
image_name_build(self.distro, self.version),
|
||||
]
|
||||
else:
|
||||
run_cmd += [
|
||||
"--hostname",
|
||||
"dangerzone",
|
||||
image_name_build_enduser(self.distro, self.version),
|
||||
image_name_install(self.distro, self.version),
|
||||
]
|
||||
|
||||
run_cmd += cmd
|
||||
|
|
@ -502,33 +536,8 @@ class Env:
|
|||
(dist_state / ".bash_history").touch(exist_ok=True)
|
||||
self.runtime_run(*run_cmd)
|
||||
|
||||
def pull_image_from_registry(self, image):
|
||||
try:
|
||||
subprocess.run(self.runtime_cmd + ["pull", image], check=True)
|
||||
return True
|
||||
except subprocess.CalledProcessError:
|
||||
# Do not log an error here, we are just checking if the image exists
|
||||
# on the registry.
|
||||
return False
|
||||
|
||||
def push_image_to_registry(self, image):
|
||||
try:
|
||||
subprocess.run(self.runtime_cmd + ["push", image], check=True)
|
||||
return True
|
||||
except subprocess.CalledProcessError as e:
|
||||
print("An error occured when pulling the image: ", e)
|
||||
return False
|
||||
|
||||
def build_dev(self, show_dockerfile=DEFAULT_SHOW_DOCKERFILE, sync=False):
|
||||
def build_dev(self, show_dockerfile=DEFAULT_SHOW_DOCKERFILE):
|
||||
"""Build a Linux environment and install tools for Dangerzone development."""
|
||||
image = image_name_build_dev(self.distro, self.version)
|
||||
|
||||
if sync and self.pull_image_from_registry(image):
|
||||
print("Image has been pulled from the registry, no need to build it.")
|
||||
return
|
||||
elif sync:
|
||||
print("Image label not in registry, building it")
|
||||
|
||||
if self.distro == "fedora":
|
||||
install_deps = DOCKERFILE_BUILD_DEV_FEDORA_DEPS
|
||||
else:
|
||||
|
|
@ -538,7 +547,12 @@ class Env:
|
|||
# See https://github.com/freedomofpress/dangerzone/issues/482
|
||||
qt_deps = "libqt6gui6 libxcb-cursor0"
|
||||
install_deps = DOCKERFILE_BUILD_DEV_DEBIAN_DEPS
|
||||
if self.distro == "ubuntu" and self.version in ("22.04", "jammy"):
|
||||
if self.distro == "ubuntu" and self.version in ("20.04", "focal"):
|
||||
qt_deps = "libqt5gui5 libxcb-cursor0" # Ubuntu Focal has only Qt5.
|
||||
install_deps = (
|
||||
DOCKERFILE_UBUNTU_2004_DEPS + DOCKERFILE_BUILD_DEV_DEBIAN_DEPS
|
||||
)
|
||||
elif self.distro == "ubuntu" and self.version in ("22.04", "jammy"):
|
||||
# Ubuntu Jammy misses a dependency to `libxkbcommon-x11-0`, which we can
|
||||
# install indirectly via `qt6-qpa-plugins`.
|
||||
qt_deps += " qt6-qpa-plugins"
|
||||
|
|
@ -548,12 +562,10 @@ class Env:
|
|||
DOCKERFILE_CONMON_UPDATE + DOCKERFILE_BUILD_DEV_DEBIAN_DEPS
|
||||
)
|
||||
elif self.distro == "ubuntu" and self.version in (
|
||||
"23.10",
|
||||
"mantic",
|
||||
"24.04",
|
||||
"noble",
|
||||
"24.10",
|
||||
"ocular",
|
||||
"25.04",
|
||||
"plucky",
|
||||
):
|
||||
install_deps = (
|
||||
DOCKERFILE_UBUNTU_REM_USER + DOCKERFILE_BUILD_DEV_DEBIAN_DEPS
|
||||
|
|
@ -575,21 +587,24 @@ class Env:
|
|||
os.makedirs(build_dir, exist_ok=True)
|
||||
|
||||
# Populate the build context.
|
||||
for source in get_build_dir_sources(self.distro, self.version):
|
||||
shutil.copy(source, build_dir)
|
||||
|
||||
shutil.copy(git_root() / "pyproject.toml", build_dir)
|
||||
shutil.copy(git_root() / "poetry.lock", build_dir)
|
||||
shutil.copy(git_root() / "dev_scripts" / "storage.conf", build_dir)
|
||||
if self.distro == "ubuntu" and self.version in ("22.04", "jammy"):
|
||||
shutil.copy(git_root() / "dev_scripts" / "apt-tools-prod.pref", build_dir)
|
||||
shutil.copy(
|
||||
git_root() / "dev_scripts" / "apt-tools-prod.sources", build_dir
|
||||
)
|
||||
with open(build_dir / "Dockerfile", mode="w") as f:
|
||||
f.write(dockerfile)
|
||||
|
||||
image = image_name_build(self.distro, self.version)
|
||||
self.runtime_run("build", "-t", image, build_dir)
|
||||
|
||||
if sync:
|
||||
if not self.push_image_to_registry(image):
|
||||
print("An error occured while trying to push to the container registry")
|
||||
|
||||
def build(
|
||||
self,
|
||||
show_dockerfile=DEFAULT_SHOW_DOCKERFILE,
|
||||
download_pyside6=DEFAULT_DOWNLOAD_PYSIDE6,
|
||||
):
|
||||
"""Build a Linux environment and install Dangerzone in it."""
|
||||
build_dir = distro_build(self.distro, self.version)
|
||||
|
|
@ -597,29 +612,53 @@ class Env:
|
|||
version = dz_version()
|
||||
if self.distro == "fedora":
|
||||
install_deps = DOCKERFILE_BUILD_FEDORA_DEPS
|
||||
package_pattern = f"dangerzone-{version}-*.fc{self.version}.x86_64.rpm"
|
||||
package_src = self.find_dz_package(git_root() / "dist", package_pattern)
|
||||
package = package_src.name
|
||||
package = f"dangerzone-{version}-1.fc{self.version}.x86_64.rpm"
|
||||
package_src = git_root() / "dist" / package
|
||||
package_dst = build_dir / package
|
||||
install_cmd = "dnf install -y"
|
||||
|
||||
# NOTE: For Fedora 39+ onward, we check if a PySide6 RPM package exists in
|
||||
# the user's system. If not, we either throw an error or download it from
|
||||
# FPF's repo, according to the user's choice.
|
||||
# FIXME: Unconditionally check for PySide6, once Fedora 38 is no longer
|
||||
# supported.
|
||||
if self.version != "38":
|
||||
pyside6 = PySide6Manager(self.distro, self.version)
|
||||
if not pyside6.is_rpm_present:
|
||||
if download_pyside6:
|
||||
pyside6.download_rpm()
|
||||
else:
|
||||
print(
|
||||
PYSIDE6_NOT_FOUND_ERROR.format(
|
||||
pyside6_local_path=pyside6.rpm_local_path,
|
||||
pyside6_url=pyside6.rpm_url,
|
||||
),
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
shutil.copy(pyside6.rpm_local_path, build_dir / pyside6.rpm_name)
|
||||
install_deps = (
|
||||
DOCKERFILE_BUILD_FEDORA_DEPS + DOCKERFILE_BUILD_FEDORA_39_DEPS
|
||||
).format(pyside6_rpm=pyside6.rpm_name)
|
||||
else:
|
||||
install_deps = DOCKERFILE_BUILD_DEBIAN_DEPS
|
||||
if self.distro == "ubuntu" and self.version in ("22.04", "jammy"):
|
||||
if self.distro == "ubuntu" and self.version in ("20.04", "focal"):
|
||||
install_deps = (
|
||||
DOCKERFILE_UBUNTU_2004_DEPS + DOCKERFILE_BUILD_DEBIAN_DEPS
|
||||
)
|
||||
elif self.distro == "ubuntu" and self.version in ("22.04", "jammy"):
|
||||
# Ubuntu Jammy requires a more up-to-date conmon
|
||||
# package (see https://github.com/freedomofpress/dangerzone/issues/685)
|
||||
install_deps = DOCKERFILE_CONMON_UPDATE + DOCKERFILE_BUILD_DEBIAN_DEPS
|
||||
elif self.distro == "ubuntu" and self.version in (
|
||||
"23.10",
|
||||
"mantic",
|
||||
"24.04",
|
||||
"noble",
|
||||
"24.10",
|
||||
"ocular",
|
||||
"25.04",
|
||||
"plucky",
|
||||
):
|
||||
install_deps = DOCKERFILE_UBUNTU_REM_USER + DOCKERFILE_BUILD_DEBIAN_DEPS
|
||||
package_pattern = f"dangerzone_{version}-*_*.deb"
|
||||
package_src = self.find_dz_package(git_root() / "deb_dist", package_pattern)
|
||||
package = package_src.name
|
||||
package = f"dangerzone_{version}-1_all.deb"
|
||||
package_src = git_root() / "deb_dist" / package
|
||||
package_dst = build_dir / package
|
||||
install_cmd = "apt-get update && apt-get install -y"
|
||||
|
||||
|
|
@ -637,7 +676,6 @@ class Env:
|
|||
# Populate the build context.
|
||||
shutil.copy(package_src, package_dst)
|
||||
shutil.copy(git_root() / "dev_scripts" / "storage.conf", build_dir)
|
||||
shutil.copy(git_root() / "dev_scripts" / "containers.conf", build_dir)
|
||||
if self.distro == "ubuntu" and self.version in ("22.04", "jammy"):
|
||||
shutil.copy(git_root() / "dev_scripts" / "apt-tools-prod.pref", build_dir)
|
||||
shutil.copy(
|
||||
|
|
@ -646,7 +684,7 @@ class Env:
|
|||
with open(build_dir / "Dockerfile", mode="w") as f:
|
||||
f.write(dockerfile)
|
||||
|
||||
image = image_name_build_enduser(self.distro, self.version)
|
||||
image = image_name_install(self.distro, self.version)
|
||||
self.runtime_run("build", "-t", image, build_dir)
|
||||
|
||||
|
||||
|
|
@ -665,7 +703,7 @@ def env_run(args):
|
|||
def env_build_dev(args):
|
||||
"""Invoke the 'build-dev' command based on the CLI args."""
|
||||
env = Env.from_args(args)
|
||||
return env.build_dev(show_dockerfile=args.show_dockerfile, sync=args.sync)
|
||||
return env.build_dev(show_dockerfile=args.show_dockerfile)
|
||||
|
||||
|
||||
def env_build(args):
|
||||
|
|
@ -673,6 +711,7 @@ def env_build(args):
|
|||
env = Env.from_args(args)
|
||||
return env.build(
|
||||
show_dockerfile=args.show_dockerfile,
|
||||
download_pyside6=args.download_pyside6,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -750,12 +789,6 @@ def parse_args():
|
|||
action="store_true",
|
||||
help="Do not build, only show the Dockerfile",
|
||||
)
|
||||
parser_build_dev.add_argument(
|
||||
"--sync",
|
||||
default=False,
|
||||
action="store_true",
|
||||
help="Attempt to pull the image, build it if not found and push it to the container registry",
|
||||
)
|
||||
|
||||
# Build a development variant of a Dangerzone environment.
|
||||
parser_build = subparsers.add_parser(
|
||||
|
|
@ -769,6 +802,12 @@ def parse_args():
|
|||
action="store_true",
|
||||
help="Do not build, only show the Dockerfile",
|
||||
)
|
||||
parser_build.add_argument(
|
||||
"--download-pyside6",
|
||||
default=DEFAULT_DOWNLOAD_PYSIDE6,
|
||||
action="store_true",
|
||||
help="Download PySide6 from FPF's RPM repo",
|
||||
)
|
||||
|
||||
return parser.parse_args()
|
||||
|
||||
|
|
|
|||
2
dev_scripts/envs/.gitignore
vendored
Normal file
2
dev_scripts/envs/.gitignore
vendored
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
*
|
||||
!.gitignore
|
||||
|
|
@ -1,254 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import re
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
|
||||
import httpx
|
||||
|
||||
REPOSITORY = "https://github.com/freedomofpress/dangerzone/"
|
||||
TEMPLATE = "- {title} ([#{number}]({url}))"
|
||||
|
||||
|
||||
def parse_version(version: str) -> Tuple[int, int]:
|
||||
"""Extract major.minor from version string, ignoring patch"""
|
||||
match = re.match(r"v?(\d+)\.(\d+)", version)
|
||||
if not match:
|
||||
raise ValueError(f"Invalid version format: {version}")
|
||||
return (int(match.group(1)), int(match.group(2)))
|
||||
|
||||
|
||||
async def get_last_minor_release(
|
||||
client: httpx.AsyncClient, owner: str, repo: str
|
||||
) -> Optional[str]:
|
||||
"""Get the latest minor release date (ignoring patches)"""
|
||||
response = await client.get(f"https://api.github.com/repos/{owner}/{repo}/releases")
|
||||
response.raise_for_status()
|
||||
releases = response.json()
|
||||
|
||||
if not releases:
|
||||
return None
|
||||
|
||||
# Get the latest minor version by comparing major.minor numbers
|
||||
current_version = parse_version(releases[0]["tag_name"])
|
||||
latest_date = None
|
||||
|
||||
for release in releases:
|
||||
try:
|
||||
version = parse_version(release["tag_name"])
|
||||
if version < current_version:
|
||||
latest_date = release["published_at"]
|
||||
break
|
||||
except ValueError:
|
||||
continue
|
||||
|
||||
return latest_date
|
||||
|
||||
|
||||
async def get_issue_details(
|
||||
client: httpx.AsyncClient, owner: str, repo: str, issue_number: int
|
||||
) -> Optional[dict]:
|
||||
"""Get issue title and number if it exists"""
|
||||
response = await client.get(
|
||||
f"https://api.github.com/repos/{owner}/{repo}/issues/{issue_number}"
|
||||
)
|
||||
if response.is_success:
|
||||
data = response.json()
|
||||
return {
|
||||
"title": data["title"],
|
||||
"number": data["number"],
|
||||
"url": data["html_url"],
|
||||
}
|
||||
return None
|
||||
|
||||
|
||||
def extract_issue_number(pr_body: Optional[str]) -> Optional[int]:
|
||||
"""Extract issue number from PR body looking for common formats like 'Fixes #123' or 'Closes #123'"""
|
||||
if not pr_body:
|
||||
return None
|
||||
|
||||
patterns = [
|
||||
r"(?:closes|fixes|resolves)\s*#(\d+)",
|
||||
r"(?:close|fix|resolve)\s*#(\d+)",
|
||||
]
|
||||
|
||||
for pattern in patterns:
|
||||
match = re.search(pattern, pr_body.lower())
|
||||
if match:
|
||||
return int(match.group(1))
|
||||
|
||||
return None
|
||||
|
||||
|
||||
async def verify_commit_in_master(
|
||||
client: httpx.AsyncClient, owner: str, repo: str, commit_id: str
|
||||
) -> bool:
|
||||
"""Verify if a commit exists in master"""
|
||||
response = await client.get(
|
||||
f"https://api.github.com/repos/{owner}/{repo}/commits/{commit_id}"
|
||||
)
|
||||
return response.is_success and response.json().get("commit") is not None
|
||||
|
||||
|
||||
async def process_issue_events(
|
||||
client: httpx.AsyncClient, owner: str, repo: str, issue: Dict
|
||||
) -> Optional[Dict]:
|
||||
"""Process events for a single issue"""
|
||||
events_response = await client.get(f"{issue['url']}/events")
|
||||
if not events_response.is_success:
|
||||
return None
|
||||
|
||||
for event in events_response.json():
|
||||
if event["event"] == "closed" and event.get("commit_id"):
|
||||
if await verify_commit_in_master(client, owner, repo, event["commit_id"]):
|
||||
return {
|
||||
"title": issue["title"],
|
||||
"number": issue["number"],
|
||||
"url": issue["html_url"],
|
||||
}
|
||||
return None
|
||||
|
||||
|
||||
async def get_closed_issues(
|
||||
client: httpx.AsyncClient, owner: str, repo: str, since: str
|
||||
) -> List[Dict]:
|
||||
"""Get issues closed by commits to master since the given date"""
|
||||
response = await client.get(
|
||||
f"https://api.github.com/repos/{owner}/{repo}/issues",
|
||||
params={
|
||||
"state": "closed",
|
||||
"sort": "updated",
|
||||
"direction": "desc",
|
||||
"since": since,
|
||||
"per_page": 100,
|
||||
},
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
tasks = []
|
||||
since_date = datetime.strptime(since, "%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
for issue in response.json():
|
||||
if "pull_request" in issue:
|
||||
continue
|
||||
|
||||
closed_at = datetime.strptime(issue["closed_at"], "%Y-%m-%dT%H:%M:%SZ")
|
||||
if closed_at <= since_date:
|
||||
continue
|
||||
|
||||
tasks.append(process_issue_events(client, owner, repo, issue))
|
||||
|
||||
results = await asyncio.gather(*tasks)
|
||||
return [r for r in results if r is not None]
|
||||
|
||||
|
||||
async def process_pull_request(
|
||||
client: httpx.AsyncClient,
|
||||
owner: str,
|
||||
repo: str,
|
||||
pr: Dict,
|
||||
closed_issues: List[Dict],
|
||||
) -> Optional[str]:
|
||||
"""Process a single pull request"""
|
||||
issue_number = extract_issue_number(pr.get("body"))
|
||||
if issue_number:
|
||||
issue = await get_issue_details(client, owner, repo, issue_number)
|
||||
if issue:
|
||||
if not any(i["number"] == issue["number"] for i in closed_issues):
|
||||
return TEMPLATE.format(**issue)
|
||||
return None
|
||||
|
||||
return TEMPLATE.format(title=pr["title"], number=pr["number"], url=pr["html_url"])
|
||||
|
||||
|
||||
async def get_changes_since_last_release(
|
||||
owner: str, repo: str, token: Optional[str] = None
|
||||
) -> List[str]:
|
||||
headers = {
|
||||
"Accept": "application/vnd.github.v3+json",
|
||||
}
|
||||
if token:
|
||||
headers["Authorization"] = f"token {token}"
|
||||
else:
|
||||
print(
|
||||
"Warning: No token provided. API rate limiting may occur.", file=sys.stderr
|
||||
)
|
||||
|
||||
async with httpx.AsyncClient(headers=headers, timeout=30.0) as client:
|
||||
# Get the date of last minor release
|
||||
since = await get_last_minor_release(client, owner, repo)
|
||||
if not since:
|
||||
return []
|
||||
|
||||
changes = []
|
||||
|
||||
# Get issues closed by commits to master
|
||||
closed_issues = await get_closed_issues(client, owner, repo, since)
|
||||
changes.extend([TEMPLATE.format(**issue) for issue in closed_issues])
|
||||
|
||||
# Get merged PRs
|
||||
response = await client.get(
|
||||
f"https://api.github.com/repos/{owner}/{repo}/pulls",
|
||||
params={
|
||||
"state": "closed",
|
||||
"sort": "updated",
|
||||
"direction": "desc",
|
||||
"per_page": 100,
|
||||
},
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
# Process PRs in parallel
|
||||
pr_tasks = []
|
||||
for pr in response.json():
|
||||
if not pr["merged_at"]:
|
||||
continue
|
||||
if since and pr["merged_at"] <= since:
|
||||
break
|
||||
|
||||
pr_tasks.append(
|
||||
process_pull_request(client, owner, repo, pr, closed_issues)
|
||||
)
|
||||
|
||||
pr_results = await asyncio.gather(*pr_tasks)
|
||||
changes.extend([r for r in pr_results if r is not None])
|
||||
|
||||
return changes
|
||||
|
||||
|
||||
async def main_async():
|
||||
parser = argparse.ArgumentParser(description="Generate release notes from GitHub")
|
||||
parser.add_argument("--token", "-t", help="the file path to the GitHub API token")
|
||||
args = parser.parse_args()
|
||||
|
||||
token = None
|
||||
if args.token:
|
||||
with open(args.token) as f:
|
||||
token = f.read().strip()
|
||||
try:
|
||||
url_path = REPOSITORY.rstrip("/").split("github.com/")[1]
|
||||
owner, repo = url_path.split("/")[-2:]
|
||||
except (ValueError, IndexError):
|
||||
print("Error: Invalid GitHub URL", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
notes = await get_changes_since_last_release(owner, repo, token)
|
||||
print("\n".join(notes))
|
||||
except httpx.HTTPError as e:
|
||||
print(f"Error: {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
except Exception as e:
|
||||
print(f"Error: {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def main():
|
||||
asyncio.run(main_async())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -1,67 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
import pathlib
|
||||
import subprocess
|
||||
|
||||
RELEASE_FILE = "RELEASE.md"
|
||||
QA_FILE = "QA.md"
|
||||
|
||||
|
||||
def git_root():
|
||||
"""Get the root directory of the Git repo."""
|
||||
# FIXME: Use a Git Python binding for this.
|
||||
# FIXME: Make this work if called outside the repo.
|
||||
path = (
|
||||
subprocess.run(
|
||||
["git", "rev-parse", "--show-toplevel"],
|
||||
check=True,
|
||||
stdout=subprocess.PIPE,
|
||||
)
|
||||
.stdout.decode()
|
||||
.strip("\n")
|
||||
)
|
||||
return pathlib.Path(path)
|
||||
|
||||
|
||||
def extract_checkboxes(filename):
|
||||
headers = []
|
||||
result = []
|
||||
|
||||
with open(filename, "r") as f:
|
||||
lines = f.readlines()
|
||||
|
||||
current_level = 0
|
||||
for line in lines:
|
||||
line = line.rstrip()
|
||||
|
||||
# If it's a header, store it
|
||||
if line.startswith("#"):
|
||||
# Count number of # to determine header level
|
||||
level = len(line) - len(line.lstrip("#"))
|
||||
if level < current_level or not current_level:
|
||||
headers.extend(["", line, ""])
|
||||
current_level = level
|
||||
elif level > current_level:
|
||||
continue
|
||||
else:
|
||||
headers = ["", line, ""]
|
||||
|
||||
# If it's a checkbox
|
||||
elif "- [ ]" in line or "- [x]" in line or "- [X]" in line:
|
||||
# Print the last header if we haven't already
|
||||
if headers:
|
||||
result.extend(headers)
|
||||
headers = []
|
||||
current_level = 0
|
||||
|
||||
# If this is the "Do the QA tasks" line, recursively get QA tasks
|
||||
if "Do the QA tasks" in line:
|
||||
result.append(line)
|
||||
qa_tasks = extract_checkboxes(git_root() / QA_FILE)
|
||||
result.append(qa_tasks)
|
||||
else:
|
||||
result.append(line)
|
||||
return "\n".join(result)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print(extract_checkboxes(git_root() / RELEASE_FILE))
|
||||
|
|
@ -3,49 +3,28 @@
|
|||
import abc
|
||||
import argparse
|
||||
import difflib
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import selectors
|
||||
import subprocess
|
||||
import sys
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
PYTHON_VERSION = "3.12"
|
||||
EOL_PYTHON_URL = "https://endoflife.date/api/python.json"
|
||||
|
||||
CONTENT_QA = r"""## QA
|
||||
|
||||
To ensure that new releases do not introduce regressions, and support existing
|
||||
and newer platforms, we have to test that the produced packages work as expected.
|
||||
|
||||
Check the following:
|
||||
and newer platforms, we have to do the following:
|
||||
|
||||
- [ ] Make sure that the tip of the `main` branch passes the CI tests.
|
||||
- [ ] Make sure that the Apple account has a valid application password and has
|
||||
agreed to the latest Apple terms (see [macOS release](#macos-release)
|
||||
section).
|
||||
|
||||
Because it is repetitive, we wrote a script to help with the QA.
|
||||
It can run the tasks for you, pausing when it needs manual intervention.
|
||||
|
||||
You can run it with a command like:
|
||||
|
||||
```bash
|
||||
poetry run ./dev_scripts/qa.py {distro}-{version}
|
||||
```
|
||||
|
||||
### The checklist
|
||||
|
||||
- [ ] Create a test build in Windows and make sure it works:
|
||||
- [ ] Check if the suggested Python version is still supported.
|
||||
- [ ] Create a new development environment with Poetry.
|
||||
- [ ] Build the container image and ensure the development environment uses
|
||||
the new image.
|
||||
- [ ] Download the OCR language data using `./install/common/download-tessdata.py`
|
||||
- [ ] Run the Dangerzone tests.
|
||||
- [ ] Build and run the Dangerzone .exe
|
||||
- [ ] Test some QA scenarios (see [Scenarios](#Scenarios) below).
|
||||
|
|
@ -54,7 +33,6 @@ poetry run ./dev_scripts/qa.py {distro}-{version}
|
|||
- [ ] Create a new development environment with Poetry.
|
||||
- [ ] Build the container image and ensure the development environment uses
|
||||
the new image.
|
||||
- [ ] Download the OCR language data using `./install/common/download-tessdata.py`
|
||||
- [ ] Run the Dangerzone tests.
|
||||
- [ ] Create and run an app bundle.
|
||||
- [ ] Test some QA scenarios (see [Scenarios](#Scenarios) below).
|
||||
|
|
@ -63,7 +41,6 @@ poetry run ./dev_scripts/qa.py {distro}-{version}
|
|||
- [ ] Create a new development environment with Poetry.
|
||||
- [ ] Build the container image and ensure the development environment uses
|
||||
the new image.
|
||||
- [ ] Download the OCR language data using `./install/common/download-tessdata.py`
|
||||
- [ ] Run the Dangerzone tests.
|
||||
- [ ] Create and run an app bundle.
|
||||
- [ ] Test some QA scenarios (see [Scenarios](#Scenarios) below).
|
||||
|
|
@ -72,20 +49,18 @@ poetry run ./dev_scripts/qa.py {distro}-{version}
|
|||
- [ ] Create a new development environment with Poetry.
|
||||
- [ ] Build the container image and ensure the development environment uses
|
||||
the new image.
|
||||
- [ ] Download the OCR language data using `./install/common/download-tessdata.py`
|
||||
- [ ] Run the Dangerzone tests.
|
||||
- [ ] Create a .deb package and install it system-wide.
|
||||
- [ ] Test some QA scenarios (see [Scenarios](#Scenarios) below).
|
||||
- [ ] Create a test build in the most recent Fedora platform (Fedora 41 as of
|
||||
- [ ] Create a test build in the most recent Fedora platform (Fedora 40 as of
|
||||
writing this) and make sure it works:
|
||||
- [ ] Create a new development environment with Poetry.
|
||||
- [ ] Build the container image and ensure the development environment uses
|
||||
the new image.
|
||||
- [ ] Download the OCR language data using `./install/common/download-tessdata.py`
|
||||
- [ ] Run the Dangerzone tests.
|
||||
- [ ] Create an .rpm package and install it system-wide.
|
||||
- [ ] Test some QA scenarios (see [Scenarios](#Scenarios) below).
|
||||
- [ ] Create a test build in the most recent Qubes Fedora template (Fedora 40 as
|
||||
- [ ] Create a test build in the most recent Qubes Fedora template (Fedora 39 as
|
||||
of writing this) and make sure it works:
|
||||
- [ ] Create a new development environment with Poetry.
|
||||
- [ ] Run the Dangerzone tests.
|
||||
|
|
@ -127,9 +102,9 @@ Close the Dangerzone application and get the container image for that
|
|||
version. For example:
|
||||
|
||||
```
|
||||
$ docker images dangerzone.rocks/dangerzone
|
||||
$ docker images dangerzone.rocks/dangerzone:latest
|
||||
REPOSITORY TAG IMAGE ID CREATED SIZE
|
||||
dangerzone.rocks/dangerzone <tag> <image ID> <date> <size>
|
||||
dangerzone.rocks/dangerzone latest <image ID> <date> <size>
|
||||
```
|
||||
|
||||
Then run the version under QA and ensure that the settings remain changed.
|
||||
|
|
@ -138,9 +113,9 @@ Afterwards check that new docker image was installed by running the same command
|
|||
and seeing the following differences:
|
||||
|
||||
```
|
||||
$ docker images dangerzone.rocks/dangerzone
|
||||
$ docker images dangerzone.rocks/dangerzone:latest
|
||||
REPOSITORY TAG IMAGE ID CREATED SIZE
|
||||
dangerzone.rocks/dangerzone <other tag> <different ID> <newer date> <different size>
|
||||
dangerzone.rocks/dangerzone latest <different ID> <newer date> <different size>
|
||||
```
|
||||
|
||||
#### 4. Dangerzone successfully installs the container image
|
||||
|
|
@ -178,30 +153,21 @@ Run Dangerzone against a list of documents, and tick all options. Ensure that:
|
|||
location.
|
||||
* The original files have been saved in the `unsafe/` directory.
|
||||
|
||||
#### 8. Dangerzone is able to handle drag-n-drop
|
||||
|
||||
Run Dangerzone against a set of documents that you drag-n-drop. Files should be
|
||||
added and conversion should run without issue.
|
||||
|
||||
> [!TIP]
|
||||
> On our end-user container environments for Linux, we can start a file manager
|
||||
> with `thunar &`.
|
||||
|
||||
#### 9. Dangerzone CLI succeeds in converting multiple documents
|
||||
#### 8. Dangerzone CLI succeeds in converting multiple documents
|
||||
|
||||
_(Only for Windows and Linux)_
|
||||
|
||||
Run Dangerzone CLI against a list of documents. Ensure that conversions happen
|
||||
sequentially, are completed successfully, and we see their progress.
|
||||
|
||||
#### 10. Dangerzone can open a document for conversion via right-click -> "Open With"
|
||||
#### 9. Dangerzone can open a document for conversion via right-click -> "Open With"
|
||||
|
||||
_(Only for Windows, MacOS and Qubes)_
|
||||
|
||||
Go to a directory with office documents, right-click on one, and click on "Open
|
||||
With". We should be able to open the file with Dangerzone, and then convert it.
|
||||
|
||||
#### 11. Dangerzone shows helpful errors for setup issues on Qubes
|
||||
#### 10. Dangerzone shows helpful errors for setup issues on Qubes
|
||||
|
||||
_(Only for Qubes)_
|
||||
|
||||
|
|
@ -250,10 +216,9 @@ Install dependencies:
|
|||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
```sh
|
||||
sudo apt install -y podman dh-python build-essential make libqt6gui6 \
|
||||
pipx python3 python3-dev
|
||||
sudo apt install -y podman dh-python build-essential fakeroot make libqt6gui6 \
|
||||
pipx python3 python3-dev python3-stdeb python3-all
|
||||
```
|
||||
|
||||
Install Poetry using `pipx` (recommended) and add it to your `$PATH`:
|
||||
|
|
@ -264,7 +229,6 @@ methods](https://python-poetry.org/docs/#installation))_
|
|||
```sh
|
||||
pipx ensurepath
|
||||
pipx install poetry
|
||||
pipx inject poetry poetry-plugin-export
|
||||
```
|
||||
|
||||
After this, restart the terminal window, for the `poetry` command to be in your
|
||||
|
|
@ -292,12 +256,6 @@ Build the latest container:
|
|||
python3 ./install/common/build-image.py
|
||||
```
|
||||
|
||||
Download the OCR language data:
|
||||
|
||||
```sh
|
||||
python3 ./install/common/download-tessdata.py
|
||||
```
|
||||
|
||||
Run from source tree:
|
||||
|
||||
```sh
|
||||
|
|
@ -331,7 +289,6 @@ Install Poetry using `pipx`:
|
|||
|
||||
```sh
|
||||
pipx install poetry
|
||||
pipx inject poetry
|
||||
```
|
||||
|
||||
Clone this repository:
|
||||
|
|
@ -355,12 +312,6 @@ Build the latest container:
|
|||
python3 ./install/common/build-image.py
|
||||
```
|
||||
|
||||
Download the OCR language data:
|
||||
|
||||
```sh
|
||||
python3 ./install/common/download-tessdata.py
|
||||
```
|
||||
|
||||
Run from source tree:
|
||||
|
||||
```sh
|
||||
|
|
@ -389,7 +340,7 @@ CONTENT_BUILD_WINDOWS = r"""## Windows
|
|||
|
||||
Install [Docker Desktop](https://www.docker.com/products/docker-desktop).
|
||||
|
||||
Install the latest version of Python 3.12 (64-bit) [from python.org](https://www.python.org/downloads/windows/). Make sure to check the "Add Python 3.12 to PATH" checkbox on the first page of the installer.
|
||||
Install the latest version of Python 3.11 (64-bit) [from python.org](https://www.python.org/downloads/windows/). Make sure to check the "Add Python 3.11 to PATH" checkbox on the first page of the installer.
|
||||
|
||||
|
||||
Install Microsoft Visual C++ 14.0 or greater. Get it with ["Microsoft C++ Build Tools"](https://visualstudio.microsoft.com/visual-cpp-build-tools/) and make sure to select "Desktop development with C++" when installing.
|
||||
|
|
@ -419,12 +370,6 @@ Build the dangerzone container image:
|
|||
python3 .\install\common\build-image.py
|
||||
```
|
||||
|
||||
Download the OCR language data:
|
||||
|
||||
```sh
|
||||
python3 .\install\common\download-tessdata.py
|
||||
```
|
||||
|
||||
After that you can launch dangerzone during development with:
|
||||
|
||||
```
|
||||
|
|
@ -529,7 +474,7 @@ class Reference:
|
|||
# Convert spaces to dashes
|
||||
anchor = anchor.replace(" ", "-")
|
||||
# Remove non-alphanumeric (except dash and underscore)
|
||||
anchor = re.sub("[^a-zA-Z-_]", "", anchor)
|
||||
anchor = re.sub("[^a-zA-Z\-_]", "", anchor)
|
||||
|
||||
return anchor
|
||||
|
||||
|
|
@ -548,8 +493,8 @@ class QABase(abc.ABC):
|
|||
|
||||
platforms = {}
|
||||
|
||||
REF_QA = Reference("QA.md", content=CONTENT_QA)
|
||||
REF_QA_SCENARIOS = Reference("QA.md", content=CONTENT_QA_SCENARIOS)
|
||||
REF_QA = Reference("RELEASE.md", content=CONTENT_QA)
|
||||
REF_QA_SCENARIOS = Reference("RELEASE.md", content=CONTENT_QA_SCENARIOS)
|
||||
|
||||
# The following class method is available since Python 3.6. For more details, see:
|
||||
# https://docs.python.org/3.6/whatsnew/3.6.html#pep-487-simpler-customization-of-class-creation
|
||||
|
|
@ -758,10 +703,6 @@ class QABase(abc.ABC):
|
|||
self.prompt("Does it pass?", choices=["y", "n"])
|
||||
logger.info("Successfully completed QA scenarios")
|
||||
|
||||
@task("Download Tesseract data", auto=True)
|
||||
def download_tessdata(self):
|
||||
self.run("python", str(Path("install", "common", "download-tessdata.py")))
|
||||
|
||||
@classmethod
|
||||
@abc.abstractmethod
|
||||
def get_id(cls):
|
||||
|
|
@ -788,40 +729,6 @@ class QAWindows(QABase):
|
|||
while msvcrt.kbhit():
|
||||
msvcrt.getch()
|
||||
|
||||
def get_latest_python_release(self):
|
||||
with urllib.request.urlopen(EOL_PYTHON_URL) as f:
|
||||
resp = f.read()
|
||||
releases = json.loads(resp)
|
||||
for release in releases:
|
||||
if release["cycle"] == PYTHON_VERSION:
|
||||
# Transform the Python version string (e.g., "3.12.7") into a list
|
||||
# (e.g., [3, 12, 7]), and return it
|
||||
return [int(num) for num in release["latest"].split(".")]
|
||||
|
||||
raise RuntimeError(
|
||||
f"Could not find a Python release for version {PYTHON_VERSION}"
|
||||
)
|
||||
|
||||
@QABase.task(
|
||||
f"Install the latest version of Python {PYTHON_VERSION}", ref=REF_BUILD
|
||||
)
|
||||
def install_python(self):
|
||||
logger.info("Getting latest Python release")
|
||||
try:
|
||||
latest_version = self.get_latest_python_release()
|
||||
except Exception:
|
||||
logger.error("Could not verify that the latest Python version is installed")
|
||||
|
||||
cur_version = list(sys.version_info[:3])
|
||||
if latest_version > cur_version:
|
||||
self.prompt(
|
||||
f"You need to install the latest Python version ({latest_version})"
|
||||
)
|
||||
elif latest_version == cur_version:
|
||||
logger.info(
|
||||
f"Verified that the latest Python version ({latest_version}) is installed"
|
||||
)
|
||||
|
||||
@QABase.task("Install and Run Docker Desktop", ref=REF_BUILD)
|
||||
def install_docker(self):
|
||||
logger.info("Checking if Docker Desktop is installed and running")
|
||||
|
|
@ -836,7 +743,7 @@ class QAWindows(QABase):
|
|||
)
|
||||
def install_poetry(self):
|
||||
self.run("python", "-m", "pip", "install", "poetry")
|
||||
self.run("poetry", "sync")
|
||||
self.run("poetry", "install")
|
||||
|
||||
@QABase.task("Build Dangerzone container image", ref=REF_BUILD, auto=True)
|
||||
def build_image(self):
|
||||
|
|
@ -858,11 +765,9 @@ class QAWindows(QABase):
|
|||
return "windows"
|
||||
|
||||
def start(self):
|
||||
self.install_python()
|
||||
self.install_docker()
|
||||
self.install_poetry()
|
||||
self.build_image()
|
||||
self.download_tessdata()
|
||||
self.run_tests()
|
||||
self.build_dangerzone_exe()
|
||||
|
||||
|
|
@ -937,6 +842,7 @@ class QALinux(QABase):
|
|||
"--version",
|
||||
self.VERSION,
|
||||
"build",
|
||||
"--download-pyside6",
|
||||
)
|
||||
|
||||
@classmethod
|
||||
|
|
@ -954,11 +860,10 @@ class QALinux(QABase):
|
|||
def start(self):
|
||||
self.build_dev_image()
|
||||
self.build_container_image()
|
||||
self.download_tessdata()
|
||||
self.run_tests()
|
||||
self.build_package()
|
||||
self.build_qa_image()
|
||||
self.qa_scenarios(skip=[1, 2, 3, 10, 11])
|
||||
self.qa_scenarios(skip=[1, 2, 8, 9])
|
||||
|
||||
|
||||
class QADebianBased(QALinux):
|
||||
|
|
@ -990,26 +895,26 @@ class QADebianTrixie(QADebianBased):
|
|||
VERSION = "trixie"
|
||||
|
||||
|
||||
class QAUbuntu2004(QADebianBased):
|
||||
DISTRO = "ubuntu"
|
||||
VERSION = "20.04"
|
||||
|
||||
|
||||
class QAUbuntu2204(QADebianBased):
|
||||
DISTRO = "ubuntu"
|
||||
VERSION = "22.04"
|
||||
|
||||
|
||||
class QAUbuntu2310(QADebianBased):
|
||||
DISTRO = "ubuntu"
|
||||
VERSION = "23.10"
|
||||
|
||||
|
||||
class QAUbuntu2404(QADebianBased):
|
||||
DISTRO = "ubuntu"
|
||||
VERSION = "24.04"
|
||||
|
||||
|
||||
class QAUbuntu2410(QADebianBased):
|
||||
DISTRO = "ubuntu"
|
||||
VERSION = "24.10"
|
||||
|
||||
|
||||
class QAUbuntu2504(QADebianBased):
|
||||
DISTRO = "ubuntu"
|
||||
VERSION = "25.04"
|
||||
|
||||
|
||||
class QAFedora(QALinux):
|
||||
"""Base class for Fedora distros.
|
||||
|
||||
|
|
@ -1027,18 +932,18 @@ class QAFedora(QALinux):
|
|||
)
|
||||
|
||||
|
||||
class QAFedora42(QAFedora):
|
||||
VERSION = "42"
|
||||
|
||||
|
||||
class QAFedora41(QAFedora):
|
||||
VERSION = "41"
|
||||
|
||||
|
||||
class QAFedora40(QAFedora):
|
||||
VERSION = "40"
|
||||
|
||||
|
||||
class QAFedora39(QAFedora):
|
||||
VERSION = "39"
|
||||
|
||||
|
||||
class QAFedora38(QAFedora):
|
||||
VERSION = "38"
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser(
|
||||
prog=sys.argv[0],
|
||||
|
|
|
|||
|
|
@ -1,680 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
import argparse
|
||||
import datetime
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import pprint
|
||||
import shlex
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tarfile
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
MEDIA_TYPE_INDEX_V1_JSON = "application/vnd.oci.image.index.v1+json"
|
||||
MEDIA_TYPE_MANIFEST_V1_JSON = "application/vnd.oci.image.manifest.v1+json"
|
||||
|
||||
ENV_RUNTIME = "REPRO_RUNTIME"
|
||||
ENV_DATETIME = "REPRO_DATETIME"
|
||||
ENV_SDE = "REPRO_SOURCE_DATE_EPOCH"
|
||||
ENV_CACHE = "REPRO_CACHE"
|
||||
ENV_BUILDKIT = "REPRO_BUILDKIT_IMAGE"
|
||||
ENV_ROOTLESS = "REPRO_ROOTLESS"
|
||||
|
||||
DEFAULT_BUILDKIT_IMAGE = "moby/buildkit:v0.19.0@sha256:14aa1b4dd92ea0a4cd03a54d0c6079046ea98cd0c0ae6176bdd7036ba370cbbe"
|
||||
DEFAULT_BUILDKIT_IMAGE_ROOTLESS = "moby/buildkit:v0.19.0-rootless@sha256:e901cffdad753892a7c3afb8b9972549fca02c73888cf340c91ed801fdd96d71"
|
||||
|
||||
MSG_BUILD_CTX = """Build environment:
|
||||
- Container runtime: {runtime}
|
||||
- BuildKit image: {buildkit_image}
|
||||
- Rootless support: {rootless}
|
||||
- Caching enabled: {use_cache}
|
||||
- Build context: {context}
|
||||
- Dockerfile: {dockerfile}
|
||||
- Output: {output}
|
||||
|
||||
Build parameters:
|
||||
- SOURCE_DATE_EPOCH: {sde}
|
||||
- Build args: {build_args}
|
||||
- Tag: {tag}
|
||||
- Platform: {platform}
|
||||
|
||||
Podman-only arguments:
|
||||
- BuildKit arguments: {buildkit_args}
|
||||
|
||||
Docker-only arguments:
|
||||
- Docker Buildx arguments: {buildx_args}
|
||||
"""
|
||||
|
||||
|
||||
def pretty_error(obj: dict, msg: str):
|
||||
raise Exception(f"{msg}\n{pprint.pprint(obj)}")
|
||||
|
||||
|
||||
def get_key(obj: dict, key: str) -> object:
|
||||
if key not in obj:
|
||||
pretty_error(f"Could not find key '{key}' in the dictionary:", obj)
|
||||
return obj[key]
|
||||
|
||||
|
||||
def run(cmd, dry=False, check=True):
|
||||
action = "Would have run" if dry else "Running"
|
||||
logger.debug(f"{action}: {shlex.join(cmd)}")
|
||||
if not dry:
|
||||
subprocess.run(cmd, check=check)
|
||||
|
||||
|
||||
def snip_contents(contents: str, num: int) -> str:
|
||||
contents = contents.replace("\n", "")
|
||||
if len(contents) > num:
|
||||
return (
|
||||
contents[:num]
|
||||
+ f" [... {len(contents) - num} characters omitted."
|
||||
+ " Pass --show-contents to print them in their entirety]"
|
||||
)
|
||||
return contents
|
||||
|
||||
|
||||
def detect_container_runtime() -> str:
|
||||
"""Auto-detect the installed container runtime in the system."""
|
||||
if shutil.which("docker"):
|
||||
return "docker"
|
||||
elif shutil.which("podman"):
|
||||
return "podman"
|
||||
else:
|
||||
return None
|
||||
|
||||
|
||||
def parse_runtime(args) -> str:
|
||||
if args.runtime is not None:
|
||||
return args.runtime
|
||||
|
||||
runtime = os.environ.get(ENV_RUNTIME)
|
||||
if runtime is None:
|
||||
raise RuntimeError("No container runtime detected in your system")
|
||||
if runtime not in ("docker", "podman"):
|
||||
raise RuntimeError(
|
||||
"Only 'docker' or 'podman' container runtimes"
|
||||
" are currently supported by this script"
|
||||
)
|
||||
|
||||
|
||||
def parse_use_cache(args) -> bool:
|
||||
if args.no_cache:
|
||||
return False
|
||||
return bool(int(os.environ.get(ENV_CACHE, "1")))
|
||||
|
||||
|
||||
def parse_rootless(args, runtime: str) -> bool:
|
||||
rootless = args.rootless or bool(int(os.environ.get(ENV_ROOTLESS, "0")))
|
||||
if runtime != "podman" and rootless:
|
||||
raise RuntimeError("Rootless mode is only supported with Podman runtime")
|
||||
return rootless
|
||||
|
||||
|
||||
def parse_sde(args) -> str:
|
||||
sde = os.environ.get(ENV_SDE, args.source_date_epoch)
|
||||
dt = os.environ.get(ENV_DATETIME, args.datetime)
|
||||
|
||||
if (sde is not None and dt is not None) or (sde is None and dt is None):
|
||||
raise RuntimeError("You need to pass either a source date epoch or a datetime")
|
||||
|
||||
if sde is not None:
|
||||
return str(sde)
|
||||
|
||||
if dt is not None:
|
||||
d = datetime.datetime.fromisoformat(dt)
|
||||
# If the datetime is naive, assume its timezone is UTC. The check is
|
||||
# taken from:
|
||||
# https://docs.python.org/3/library/datetime.html#determining-if-an-object-is-aware-or-naive
|
||||
if d.tzinfo is None or d.tzinfo.utcoffset(d) is None:
|
||||
d = d.replace(tzinfo=datetime.timezone.utc)
|
||||
return int(d.timestamp())
|
||||
|
||||
|
||||
def parse_buildkit_image(args, rootless: bool, runtime: str) -> str:
|
||||
default = DEFAULT_BUILDKIT_IMAGE_ROOTLESS if rootless else DEFAULT_BUILDKIT_IMAGE
|
||||
img = args.buildkit_image or os.environ.get(ENV_BUILDKIT, default)
|
||||
|
||||
if runtime == "podman" and not img.startswith("docker.io/"):
|
||||
img = "docker.io/" + img
|
||||
|
||||
return img
|
||||
|
||||
|
||||
def parse_build_args(args) -> str:
|
||||
return args.build_arg or []
|
||||
|
||||
|
||||
def parse_buildkit_args(args, runtime: str) -> str:
|
||||
if not args.buildkit_args:
|
||||
return []
|
||||
|
||||
if runtime != "podman":
|
||||
raise RuntimeError("Cannot specify BuildKit arguments using the Podman runtime")
|
||||
|
||||
return shlex.split(args.buildkit_args)
|
||||
|
||||
|
||||
def parse_buildx_args(args, runtime: str) -> str:
|
||||
if not args.buildx_args:
|
||||
return []
|
||||
|
||||
if runtime != "docker":
|
||||
raise RuntimeError(
|
||||
"Cannot specify Docker Buildx arguments using the Podman runtime"
|
||||
)
|
||||
|
||||
return shlex.split(args.buildx_args)
|
||||
|
||||
|
||||
def parse_image_digest(args) -> str | None:
|
||||
if not args.expected_image_digest:
|
||||
return None
|
||||
parsed = args.expected_image_digest.split(":", 1)
|
||||
if len(parsed) == 1:
|
||||
return parsed[0]
|
||||
else:
|
||||
return parsed[1]
|
||||
|
||||
|
||||
def parse_path(path: str | None) -> str | None:
|
||||
return path and str(Path(path).absolute())
|
||||
|
||||
|
||||
##########################
|
||||
# OCI parsing logic
|
||||
#
|
||||
# Compatible with:
|
||||
# * https://github.com/opencontainers/image-spec/blob/main/image-layout.md
|
||||
|
||||
|
||||
def oci_print_info(parsed: dict, full: bool) -> None:
|
||||
print(f"The OCI tarball contains an index and {len(parsed) - 1} manifest(s):")
|
||||
print()
|
||||
print(f"Image digest: {parsed[1]['digest']}")
|
||||
for i, info in enumerate(parsed):
|
||||
print()
|
||||
if i == 0:
|
||||
print(f"Index ({info['path']}):")
|
||||
else:
|
||||
print(f"Manifest {i} ({info['path']}):")
|
||||
print(f" Digest: {info['digest']}")
|
||||
print(f" Media type: {info['media_type']}")
|
||||
print(f" Platform: {info['platform'] or '-'}")
|
||||
contents = info["contents"] if full else snip_contents(info["contents"], 600)
|
||||
print(f" Contents: {contents}")
|
||||
print()
|
||||
|
||||
|
||||
def oci_normalize_path(path):
|
||||
if path.startswith("sha256:"):
|
||||
hash_algo, checksum = path.split(":")
|
||||
path = f"blobs/{hash_algo}/{checksum}"
|
||||
return path
|
||||
|
||||
|
||||
def oci_get_file_from_tarball(tar: tarfile.TarFile, path: str) -> dict:
|
||||
"""Get file from an OCI tarball.
|
||||
|
||||
If the filename cannot be found, search again by prefixing it with "./", since we
|
||||
have encountered path names in OCI tarballs prefixed with "./".
|
||||
"""
|
||||
try:
|
||||
return tar.extractfile(path).read().decode()
|
||||
except KeyError:
|
||||
if not path.startswith("./") and not path.startswith("/"):
|
||||
path = "./" + path
|
||||
try:
|
||||
return tar.extractfile(path).read().decode()
|
||||
except KeyError:
|
||||
# Do not raise here, so that we can raise the original exception below.
|
||||
pass
|
||||
raise
|
||||
|
||||
|
||||
def oci_parse_manifest(tar: tarfile.TarFile, path: str, platform: dict | None) -> dict:
|
||||
"""Parse manifest information in JSON format.
|
||||
|
||||
Interestingly, the platform info for a manifest is not included in the
|
||||
manifest itself, but in the descriptor that points to it. So, we have to
|
||||
carry it from the previous manifest and include in the info here.
|
||||
"""
|
||||
path = oci_normalize_path(path)
|
||||
contents = oci_get_file_from_tarball(tar, path)
|
||||
digest = "sha256:" + hashlib.sha256(contents.encode()).hexdigest()
|
||||
contents_dict = json.loads(contents)
|
||||
media_type = get_key(contents_dict, "mediaType")
|
||||
manifests = contents_dict.get("manifests", [])
|
||||
|
||||
if platform:
|
||||
os = get_key(platform, "os")
|
||||
arch = get_key(platform, "architecture")
|
||||
platform = f"{os}/{arch}"
|
||||
|
||||
return {
|
||||
"path": path,
|
||||
"contents": contents,
|
||||
"digest": digest,
|
||||
"media_type": media_type,
|
||||
"platform": platform,
|
||||
"manifests": manifests,
|
||||
}
|
||||
|
||||
|
||||
def oci_parse_manifests_dfs(
|
||||
tar: tarfile.TarFile, path: str, parsed: list, platform: dict | None = None
|
||||
) -> None:
|
||||
info = oci_parse_manifest(tar, path, platform)
|
||||
parsed.append(info)
|
||||
for m in info["manifests"]:
|
||||
oci_parse_manifests_dfs(tar, m["digest"], parsed, m.get("platform"))
|
||||
|
||||
|
||||
def oci_parse_tarball(path: Path) -> dict:
|
||||
parsed = []
|
||||
with tarfile.TarFile.open(path) as tar:
|
||||
oci_parse_manifests_dfs(tar, "index.json", parsed)
|
||||
return parsed
|
||||
|
||||
|
||||
##########################
|
||||
# Image building logic
|
||||
|
||||
|
||||
def podman_build(
|
||||
context: str,
|
||||
dockerfile: str | None,
|
||||
tag: str | None,
|
||||
buildkit_image: str,
|
||||
sde: int,
|
||||
rootless: bool,
|
||||
use_cache: bool,
|
||||
output: Path,
|
||||
build_args: list,
|
||||
platform: str,
|
||||
buildkit_args: list,
|
||||
dry: bool,
|
||||
):
|
||||
rootless_args = []
|
||||
rootful_args = []
|
||||
if rootless:
|
||||
rootless_args = [
|
||||
"--userns",
|
||||
"keep-id:uid=1000,gid=1000",
|
||||
"--security-opt",
|
||||
"seccomp=unconfined",
|
||||
"--security-opt",
|
||||
"apparmor=unconfined",
|
||||
"-e",
|
||||
"BUILDKITD_FLAGS=--oci-worker-no-process-sandbox",
|
||||
]
|
||||
else:
|
||||
rootful_args = ["--privileged"]
|
||||
|
||||
dockerfile_args_podman = []
|
||||
dockerfile_args_buildkit = []
|
||||
if dockerfile:
|
||||
dockerfile_args_podman = ["-v", f"{dockerfile}:/tmp/Dockerfile"]
|
||||
dockerfile_args_buildkit = ["--local", "dockerfile=/tmp"]
|
||||
else:
|
||||
dockerfile_args_buildkit = ["--local", "dockerfile=/tmp/work"]
|
||||
|
||||
tag_args = f",name={tag}" if tag else ""
|
||||
|
||||
cache_args = []
|
||||
if use_cache:
|
||||
cache_args = [
|
||||
"--export-cache",
|
||||
"type=local,mode=max,dest=/tmp/cache",
|
||||
"--import-cache",
|
||||
"type=local,src=/tmp/cache",
|
||||
]
|
||||
|
||||
_build_args = []
|
||||
for arg in build_args:
|
||||
_build_args.append("--opt")
|
||||
_build_args.append(f"build-arg:{arg}")
|
||||
platform_args = ["--opt", f"platform={platform}"] if platform else []
|
||||
|
||||
cmd = [
|
||||
"podman",
|
||||
"run",
|
||||
"-it",
|
||||
"--rm",
|
||||
"-v",
|
||||
"buildkit_cache:/tmp/cache",
|
||||
"-v",
|
||||
f"{output.parent}:/tmp/image",
|
||||
"-v",
|
||||
f"{context}:/tmp/work",
|
||||
"--entrypoint",
|
||||
"buildctl-daemonless.sh",
|
||||
*rootless_args,
|
||||
*rootful_args,
|
||||
*dockerfile_args_podman,
|
||||
buildkit_image,
|
||||
"build",
|
||||
"--frontend",
|
||||
"dockerfile.v0",
|
||||
"--local",
|
||||
"context=/tmp/work",
|
||||
"--opt",
|
||||
f"build-arg:SOURCE_DATE_EPOCH={sde}",
|
||||
*_build_args,
|
||||
"--output",
|
||||
f"type=docker,dest=/tmp/image/{output.name},rewrite-timestamp=true{tag_args}",
|
||||
*cache_args,
|
||||
*dockerfile_args_buildkit,
|
||||
*platform_args,
|
||||
*buildkit_args,
|
||||
]
|
||||
|
||||
run(cmd, dry)
|
||||
|
||||
|
||||
def docker_build(
|
||||
context: str,
|
||||
dockerfile: str | None,
|
||||
tag: str | None,
|
||||
buildkit_image: str,
|
||||
sde: int,
|
||||
use_cache: bool,
|
||||
output: Path,
|
||||
build_args: list,
|
||||
platform: str,
|
||||
buildx_args: list,
|
||||
dry: bool,
|
||||
):
|
||||
builder_id = hashlib.sha256(buildkit_image.encode()).hexdigest()
|
||||
builder_name = f"repro-build-{builder_id}"
|
||||
tag_args = ["-t", tag] if tag else []
|
||||
cache_args = [] if use_cache else ["--no-cache", "--pull"]
|
||||
|
||||
cmd = [
|
||||
"docker",
|
||||
"buildx",
|
||||
"create",
|
||||
"--name",
|
||||
builder_name,
|
||||
"--driver-opt",
|
||||
f"image={buildkit_image}",
|
||||
]
|
||||
run(cmd, dry, check=False)
|
||||
|
||||
dockerfile_args = ["-f", dockerfile] if dockerfile else []
|
||||
_build_args = []
|
||||
for arg in build_args:
|
||||
_build_args.append("--build-arg")
|
||||
_build_args.append(arg)
|
||||
platform_args = ["--platform", platform] if platform else []
|
||||
|
||||
cmd = [
|
||||
"docker",
|
||||
"buildx",
|
||||
"--builder",
|
||||
builder_name,
|
||||
"build",
|
||||
"--build-arg",
|
||||
f"SOURCE_DATE_EPOCH={sde}",
|
||||
*_build_args,
|
||||
"--provenance",
|
||||
"false",
|
||||
"--output",
|
||||
f"type=docker,dest={output},rewrite-timestamp=true",
|
||||
*cache_args,
|
||||
*tag_args,
|
||||
*dockerfile_args,
|
||||
*platform_args,
|
||||
*buildx_args,
|
||||
context,
|
||||
]
|
||||
run(cmd, dry)
|
||||
|
||||
|
||||
##########################
|
||||
# Command logic
|
||||
|
||||
|
||||
def build(args):
|
||||
runtime = parse_runtime(args)
|
||||
use_cache = parse_use_cache(args)
|
||||
sde = parse_sde(args)
|
||||
rootless = parse_rootless(args, runtime)
|
||||
buildkit_image = parse_buildkit_image(args, rootless, runtime)
|
||||
build_args = parse_build_args(args)
|
||||
platform = args.platform
|
||||
buildkit_args = parse_buildkit_args(args, runtime)
|
||||
buildx_args = parse_buildx_args(args, runtime)
|
||||
tag = args.tag
|
||||
dockerfile = parse_path(args.file)
|
||||
output = Path(parse_path(args.output))
|
||||
dry = args.dry
|
||||
context = parse_path(args.context)
|
||||
|
||||
logger.info(
|
||||
MSG_BUILD_CTX.format(
|
||||
runtime=runtime,
|
||||
buildkit_image=buildkit_image,
|
||||
sde=sde,
|
||||
rootless=rootless,
|
||||
use_cache=use_cache,
|
||||
context=context,
|
||||
dockerfile=dockerfile or "(not provided)",
|
||||
tag=tag or "(not provided)",
|
||||
output=output,
|
||||
build_args=",".join(build_args) or "(not provided)",
|
||||
platform=platform or "(default)",
|
||||
buildkit_args=" ".join(buildkit_args) or "(not provided)",
|
||||
buildx_args=" ".join(buildx_args) or "(not provided)",
|
||||
)
|
||||
)
|
||||
|
||||
try:
|
||||
if runtime == "docker":
|
||||
docker_build(
|
||||
context,
|
||||
dockerfile,
|
||||
tag,
|
||||
buildkit_image,
|
||||
sde,
|
||||
use_cache,
|
||||
output,
|
||||
build_args,
|
||||
platform,
|
||||
buildx_args,
|
||||
dry,
|
||||
)
|
||||
else:
|
||||
podman_build(
|
||||
context,
|
||||
dockerfile,
|
||||
tag,
|
||||
buildkit_image,
|
||||
sde,
|
||||
rootless,
|
||||
use_cache,
|
||||
output,
|
||||
build_args,
|
||||
platform,
|
||||
buildkit_args,
|
||||
dry,
|
||||
)
|
||||
except subprocess.CalledProcessError as e:
|
||||
logger.error(f"Failed with {e.returncode}")
|
||||
sys.exit(e.returncode)
|
||||
|
||||
|
||||
def analyze(args) -> None:
|
||||
expected_image_digest = parse_image_digest(args)
|
||||
tarball_path = Path(args.tarball)
|
||||
|
||||
parsed = oci_parse_tarball(tarball_path)
|
||||
oci_print_info(parsed, args.show_contents)
|
||||
|
||||
if expected_image_digest:
|
||||
cur_digest = parsed[1]["digest"].split(":")[1]
|
||||
if cur_digest != expected_image_digest:
|
||||
raise Exception(
|
||||
f"The image does not have the expected digest: {cur_digest} != {expected_image_digest}"
|
||||
)
|
||||
print(f"✅ Image digest matches {expected_image_digest}")
|
||||
|
||||
|
||||
def define_build_cmd_args(parser: argparse.ArgumentParser) -> None:
|
||||
parser.add_argument(
|
||||
"--runtime",
|
||||
choices=["docker", "podman"],
|
||||
default=detect_container_runtime(),
|
||||
help="The container runtime for building the image (default: %(default)s)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--datetime",
|
||||
metavar="YYYY-MM-DD",
|
||||
default=None,
|
||||
help=(
|
||||
"Provide a date and (optionally) a time in ISO format, which will"
|
||||
" be used as the timestamp of the image layers"
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--buildkit-image",
|
||||
metavar="NAME:TAG@DIGEST",
|
||||
default=None,
|
||||
help=(
|
||||
"The BuildKit container image which will be used for building the"
|
||||
" reproducible container image. Make sure to pass the '-rootless'"
|
||||
" variant if you are using rootless Podman"
|
||||
" (default: docker.io/moby/buildkit:v0.19.0)"
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--source-date-epoch",
|
||||
"--sde",
|
||||
metavar="SECONDS",
|
||||
type=int,
|
||||
default=None,
|
||||
help="Provide a Unix timestamp for the image layers",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--no-cache",
|
||||
default=False,
|
||||
action="store_true",
|
||||
help="Do not use existing cached images for the container build. Build from the start with a new set of cached layers.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--rootless",
|
||||
default=False,
|
||||
action="store_true",
|
||||
help="Run BuildKit in rootless mode (Podman only)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-f",
|
||||
"--file",
|
||||
metavar="FILE",
|
||||
default=None,
|
||||
help="Pathname of a Dockerfile",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-o",
|
||||
"--output",
|
||||
metavar="FILE",
|
||||
default=Path.cwd() / "image.tar",
|
||||
help="Path to save OCI tarball (default: %(default)s)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-t",
|
||||
"--tag",
|
||||
metavar="TAG",
|
||||
default=None,
|
||||
help="Tag the built image with the name %(metavar)s",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--build-arg",
|
||||
metavar="ARG=VALUE",
|
||||
action="append",
|
||||
default=None,
|
||||
help="Set build-time variables",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--platform",
|
||||
metavar="PLAT1,PLAT2",
|
||||
default=None,
|
||||
help="Set platform for the image",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--buildkit-args",
|
||||
metavar="'ARG1 ARG2'",
|
||||
default=None,
|
||||
help="Extra arguments for BuildKit (Podman only)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--buildx-args",
|
||||
metavar="'ARG1 ARG2'",
|
||||
default=None,
|
||||
help="Extra arguments for Docker Buildx (Docker only)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dry",
|
||||
default=False,
|
||||
action="store_true",
|
||||
help="Do not run any commands, just print what would happen",
|
||||
)
|
||||
parser.add_argument(
|
||||
"context",
|
||||
metavar="CONTEXT",
|
||||
help="Path to the build context",
|
||||
)
|
||||
|
||||
|
||||
def parse_args() -> dict:
|
||||
parser = argparse.ArgumentParser()
|
||||
subparsers = parser.add_subparsers(dest="command", help="Available commands")
|
||||
|
||||
build_parser = subparsers.add_parser("build", help="Perform a build operation")
|
||||
build_parser.set_defaults(func=build)
|
||||
define_build_cmd_args(build_parser)
|
||||
|
||||
analyze_parser = subparsers.add_parser("analyze", help="Analyze an OCI tarball")
|
||||
analyze_parser.set_defaults(func=analyze)
|
||||
analyze_parser.add_argument(
|
||||
"tarball",
|
||||
metavar="FILE",
|
||||
help="Path to OCI image in .tar format",
|
||||
)
|
||||
analyze_parser.add_argument(
|
||||
"--expected-image-digest",
|
||||
metavar="DIGEST",
|
||||
default=None,
|
||||
help="The expected digest for the provided image",
|
||||
)
|
||||
analyze_parser.add_argument(
|
||||
"--show-contents",
|
||||
default=False,
|
||||
action="store_true",
|
||||
help="Show full file contents",
|
||||
)
|
||||
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
logging.basicConfig(
|
||||
level=logging.DEBUG,
|
||||
format="%(asctime)s - %(levelname)s - %(message)s",
|
||||
datefmt="%Y-%m-%d %H:%M:%S",
|
||||
)
|
||||
args = parse_args()
|
||||
|
||||
if not hasattr(args, "func"):
|
||||
args.func = build
|
||||
args.func(args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
|
|
@ -1,115 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import logging
|
||||
import pathlib
|
||||
import platform
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
import urllib.request
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
if platform.system() in ["Darwin", "Windows"]:
|
||||
CONTAINER_RUNTIME = "docker"
|
||||
elif platform.system() == "Linux":
|
||||
CONTAINER_RUNTIME = "podman"
|
||||
|
||||
|
||||
def run(*args):
|
||||
"""Simple function that runs a command and checks the result."""
|
||||
logger.debug(f"Running command: {' '.join(args)}")
|
||||
return subprocess.run(args, check=True)
|
||||
|
||||
|
||||
def build_image(
|
||||
platform=None,
|
||||
runtime=None,
|
||||
cache=True,
|
||||
date=None,
|
||||
):
|
||||
"""Build the Dangerzone container image with a special tag."""
|
||||
platform_args = [] if not platform else ["--platform", platform]
|
||||
runtime_args = [] if not runtime else ["--runtime", runtime]
|
||||
cache_args = [] if cache else ["--use-cache", "no"]
|
||||
date_args = [] if not date else ["--debian-archive-date", date]
|
||||
run(
|
||||
"python3",
|
||||
"./install/common/build-image.py",
|
||||
*platform_args,
|
||||
*runtime_args,
|
||||
*cache_args,
|
||||
*date_args,
|
||||
)
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser(
|
||||
prog=sys.argv[0],
|
||||
description="Dev script for verifying container image reproducibility",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--platform",
|
||||
default=None,
|
||||
help=f"The platform for building the image (default: current platform)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--runtime",
|
||||
choices=["docker", "podman"],
|
||||
default=CONTAINER_RUNTIME,
|
||||
help=f"The container runtime for building the image (default: {CONTAINER_RUNTIME})",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--no-cache",
|
||||
default=False,
|
||||
action="store_true",
|
||||
help=(
|
||||
"Do not use existing cached images for the container build."
|
||||
" Build from the start with a new set of cached layers."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--debian-archive-date",
|
||||
default=None,
|
||||
help="Use a specific Debian snapshot archive, by its date",
|
||||
)
|
||||
parser.add_argument(
|
||||
"digest",
|
||||
help="The digest of the image that you want to reproduce",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main():
|
||||
logging.basicConfig(
|
||||
level=logging.DEBUG,
|
||||
format="%(asctime)s - %(levelname)s - %(message)s",
|
||||
datefmt="%Y-%m-%d %H:%M:%S",
|
||||
)
|
||||
args = parse_args()
|
||||
|
||||
logger.info(f"Building container image")
|
||||
build_image(
|
||||
args.platform,
|
||||
args.runtime,
|
||||
not args.no_cache,
|
||||
args.debian_archive_date,
|
||||
)
|
||||
|
||||
logger.info(
|
||||
f"Check that the reproduced image has the expected digest: {args.digest}"
|
||||
)
|
||||
run(
|
||||
"./dev_scripts/repro-build.py",
|
||||
"analyze",
|
||||
"--show-contents",
|
||||
"share/container.tar",
|
||||
"--expected-image-digest",
|
||||
args.digest,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
|
|
@ -11,12 +11,10 @@ log = logging.getLogger(__name__)
|
|||
|
||||
|
||||
DZ_ASSETS = [
|
||||
"container-{version}-i686.tar",
|
||||
"container-{version}-arm64.tar",
|
||||
"container.tar.gz",
|
||||
"Dangerzone-{version}.msi",
|
||||
"Dangerzone-{version}-arm64.dmg",
|
||||
"Dangerzone-{version}-i686.dmg",
|
||||
"dangerzone-{version}.tar.gz",
|
||||
]
|
||||
DZ_SIGNING_PUBKEY = "DE28AB241FA48260FAC9B8BAA7C9B38522604281"
|
||||
|
||||
|
|
@ -95,11 +93,11 @@ def main():
|
|||
parser.add_argument(
|
||||
"--version",
|
||||
required=True,
|
||||
help="look for assets with this Dangerzone version",
|
||||
help=f"look for assets with this Dangerzone version",
|
||||
)
|
||||
parser.add_argument(
|
||||
"dir",
|
||||
help="look for assets in this directory",
|
||||
help=f"look for assets in this directory",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
setup_logging()
|
||||
|
|
|
|||
|
|
@ -2,9 +2,12 @@
|
|||
|
||||
import argparse
|
||||
import getpass
|
||||
import inspect
|
||||
import logging
|
||||
import os
|
||||
import pathlib
|
||||
import sys
|
||||
import urllib
|
||||
|
||||
import requests
|
||||
|
||||
|
|
@ -83,16 +86,16 @@ def main():
|
|||
)
|
||||
parser.add_argument(
|
||||
"--tag",
|
||||
help="use the release with this tag",
|
||||
help=f"use the release with this tag",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--release-id",
|
||||
help="use the release with this ID",
|
||||
help=f"use the release with this ID",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--draft",
|
||||
action="store_true",
|
||||
help="use the latest draft release",
|
||||
help=f"use the latest draft release",
|
||||
)
|
||||
parser.add_argument(
|
||||
"file",
|
||||
|
|
@ -104,8 +107,6 @@ def main():
|
|||
|
||||
if args.token:
|
||||
log.debug(f"Reading token from {args.token}")
|
||||
# Ensure we are not uploading the token as an asset
|
||||
assert args.file != args.token
|
||||
with open(args.token) as f:
|
||||
token = f.read().strip()
|
||||
else:
|
||||
|
|
@ -118,7 +119,7 @@ def main():
|
|||
elif args.release_id:
|
||||
release_id = args.release_id
|
||||
else:
|
||||
log.debug("Getting the ID of the latest draft release")
|
||||
log.debug(f"Getting the ID of the latest draft release")
|
||||
release_id = get_latest_draft_release(token)
|
||||
log.debug(f"The latest draft release has ID '{release_id}'")
|
||||
|
||||
|
|
|
|||
|
|
@ -1,33 +0,0 @@
|
|||
Security Advisory 2024-12-24
|
||||
|
||||
In Dangerzone, a security vulnerability was detected in the quarantined
|
||||
environment where documents are opened. Vulnerabilities like this are expected
|
||||
and do not compromise the security of Dangerzone. However, in combination with
|
||||
another more serious vulnerability (also called container escape), a malicious
|
||||
document may be able to breach the security of Dangerzone. We are not aware of
|
||||
any container escapes that affect Dangerzone. **To reduce that risk, you are
|
||||
strongly advised to update Dangerzone to the latest version**.
|
||||
|
||||
# Summary
|
||||
|
||||
A series of vulnerabilities in gst-plugins-base (CVE-2024-47538, CVE-2024-47607
|
||||
and CVE-2024-47615) affects the **contained** environment where the document
|
||||
rendering takes place.
|
||||
|
||||
If one attempts to convert a malicious file with an embedded Vorbis or Opus
|
||||
media elements, arbitrary code may run within that environment. Such files
|
||||
look like regular Office documents, which means that you cannot avoid a specific
|
||||
extension. Other programs that open Office documents, such as LibreOffice, are
|
||||
also affected, unless the system has been upgraded in the meantime.
|
||||
|
||||
# How does this impact me?
|
||||
|
||||
The expectation is that malicious code will run in a container without Internet
|
||||
access, meaning that it won't be able to infect the rest of the system.
|
||||
|
||||
If you are running Dangerzone via the Qubes OS, you are not impacted.
|
||||
|
||||
# What do I need to do?
|
||||
|
||||
You are **strongly** advised to update your Dangerzone installation to 0.8.1 as
|
||||
soon as possible.
|
||||
|
|
@ -1,54 +0,0 @@
|
|||
# Using the Doit Automation Tool
|
||||
|
||||
Developers can use the [Doit](https://pydoit.org/) automation tool to create
|
||||
release artifacts. The purpose of the tool is to automate the manual release
|
||||
instructions in `RELEASE.md` file. Not everything is automated yet, since we're
|
||||
still experimenting with this tool. You can find our task definitions in this
|
||||
repo's `dodo.py` file.
|
||||
|
||||
## Why Doit?
|
||||
|
||||
We picked Doit out of the various tools out there for the following reasons:
|
||||
|
||||
* **Pythonic:** The configuration file and tasks can be written in Python. Where
|
||||
applicable, it's easy to issue shell commands as well.
|
||||
* **File targets:** Doit borrows the file target concept from Makefiles. Tasks
|
||||
can have file dependencies, and targets they build. This makes it easy to
|
||||
define a dependency graph for tasks.
|
||||
* **Hash-based caching:** Unlike Makefiles, doit does not look at the
|
||||
modification timestamp of source/target files, to figure out if it needs to
|
||||
run them. Instead, it hashes those files, and will run a task only if the
|
||||
hash of a file dependency has changed.
|
||||
* **Parallelization:** Tasks can be run in parallel with the `-n` argument,
|
||||
which is similar to `make`'s `-j` argument.
|
||||
|
||||
## How to Doit?
|
||||
|
||||
First, enter your Poetry shell. Then, make sure that your environment is clean,
|
||||
and you have ample disk space. You can run:
|
||||
|
||||
```bash
|
||||
doit clean --dry-run # if you want to see what would happen
|
||||
doit clean # you'll be asked to cofirm that you want to clean everything
|
||||
```
|
||||
|
||||
Finally, you can build all the release artifacts with `doit`, or a specific task
|
||||
with:
|
||||
|
||||
```
|
||||
doit <task>
|
||||
```
|
||||
|
||||
## Tips and tricks
|
||||
|
||||
* You can run `doit list --all -s` to see the full list of tasks, their
|
||||
dependencies, and whether they are up to date (U) or will run (R). Note that
|
||||
certain small tasks are always configured to run.
|
||||
* You can run `doit info <task>` to see which dependencies are missing.
|
||||
* You can pass the following environment variables to the script, in order to
|
||||
affect some global parameters:
|
||||
- `CONTAINER_RUNTIME`: The container runtime to use. Either `podman` (default)
|
||||
or `docker`.
|
||||
- `RELEASE_DIR`: Where to store the release artifacts. Default path is
|
||||
`~/release-assets/<version>`
|
||||
- `APPLE_ID`: The Apple ID to use when signing/notarizing the macOS DMG.
|
||||
|
|
@ -1,133 +0,0 @@
|
|||
# Create Dangerzone environments
|
||||
|
||||
The `dev_scripts/env.py` script creates environments where a user can run
|
||||
Dangerzone, allows the user to run arbitrary commands in these environments, as
|
||||
well as run Dangerzone (nested containerization).
|
||||
|
||||
It supports two types of environments:
|
||||
|
||||
1. Dev environment. This environment has developer tools, necessary for
|
||||
Dangerzone, baked in. Also, it mounts the Dangerzone source under
|
||||
`/home/user/dangerzone` in the container. The developer can then run
|
||||
Dangerzone from source, with `poetry run ./dev_scripts/dangerzone`.
|
||||
2. End-user environment. This environment has only Dangerzone installed in it,
|
||||
from the .deb/.rpm package that we have created. For convenience, it also has
|
||||
the Dangerzone source mounted under `/home/user/dangerzone`, but it lacks
|
||||
Poetry and other build tools. The developer can run Dangerzone there with
|
||||
`dangerzone`. This environment is the most vanilla Dangerzone environment,
|
||||
and should be closer to the end user's environment, than the development
|
||||
environment.
|
||||
|
||||
Each environment corresponds to a Dockerfile, which is generated on the fly. The
|
||||
developer can see this Dockerfile by passing `--show-dockerfile`.
|
||||
|
||||
For usage information, run `./dev_scripts/env.py --help`.
|
||||
|
||||
## Nested containerization
|
||||
|
||||
Since the Dangerzone environments are containers, this means that the Podman
|
||||
containers that Dangerzone creates have to be nested containers. This has some
|
||||
challenges that we will highlight below:
|
||||
|
||||
1. Containers typically only have a subset of syscalls allowed, and sometimes
|
||||
only for specific arguments. This happens with the use of
|
||||
[seccomp filters](https://docs.docker.com/engine/security/seccomp/). For
|
||||
instance, in Docker, the `clone` syscall is limited in containers and cannot
|
||||
create new namespaces
|
||||
(https://docs.docker.com/engine/security/seccomp/#significant-syscalls-blocked-by-the-default-profile). For testing/development purposes, we can get around this limitation
|
||||
by disabling the seccomp filters for the external container with
|
||||
`--security-opt seccomp=unconfined`. This has the same effect as developing
|
||||
Dangerzone locally, so it should probably be sufficient for now.
|
||||
|
||||
2. While Linux supports nested namespaces, we need extra handling for nested
|
||||
user namespaces. By default, the configuration for each user namespace (see
|
||||
[`man login.defs`](https://man7.org/linux/man-pages/man5/login.defs.5.html)
|
||||
is to reserve 65536 UIDs/GIDs, starting from UID/GID 100000. This works fine
|
||||
for the first container, but can't work for the nested container, since it
|
||||
doesn't have enough UIDs/GIDs to refer to UID 100000. Our solution to this is
|
||||
to restrict the number of UIDs/GIDs allowed in the nested container to 2000,
|
||||
which should be enough to run `podman` in it.
|
||||
|
||||
3. Containers also restrict the capabilities (see
|
||||
[`man capabilities`](https://man7.org/linux/man-pages/man7/capabilities.7.html))
|
||||
of the processes that run in them. By default, containers do not have mount
|
||||
capabilities, since it requires `CAP_SYS_ADMIN`, which effectively
|
||||
[makes the process root](https://lwn.net/Articles/486306/) in the specific
|
||||
user namespace. In our case, we have to give the Dangerzone environment this
|
||||
capability, since it will have to mount directories in Podman containers. For
|
||||
this reason, as well as some extra things we bumped into during development,
|
||||
we pass `--privileged` when creating the Dangerzone environment, which
|
||||
includes the `CAP_SYS_ADMIN` capability.
|
||||
|
||||
## GUI containerization
|
||||
|
||||
Running a GUI app in a container is a tricky subject for multi-platform apps. In
|
||||
our case, we deal specifically with Linux environments, so we can target just
|
||||
this platform.
|
||||
|
||||
To understand how a GUI app can draw in the user's screen from within a
|
||||
container, we must first understand how it does so outside the container. In
|
||||
Unix-like systems, GUI apps act like
|
||||
[clients to a display server](https://wayland.freedesktop.org/architecture.html).
|
||||
The most common display server implementation is X11, and the runner-up is
|
||||
Wayland. Both of these display servers share some common traits, mainly that
|
||||
they use Unix domain sockets as a way of letting clients communicate with them.
|
||||
|
||||
So, this gives us the answer on how one can run a containerized GUI app; they
|
||||
can simply mount the Unix Domain Socket in the container. In practice this is
|
||||
more nuanced, for two reasons:
|
||||
|
||||
1. Wayland support is not that mature on Linux, so we need to
|
||||
[set some extra environment variables](https://github.com/mviereck/x11docker/wiki/How-to-provide-Wayland-socket-to-docker-container). To simplify things, we will target
|
||||
X11 / XWayland hosts, which are the majority of the Linux OSes out there.
|
||||
2. Sharing the Unix Domain socket does not allow the client to talk to the
|
||||
display server, for security reasons. In order to allow the client, we need
|
||||
to mount a magic cookie stored in a file pointed at by the `$XAUTHORITY`
|
||||
envvar. Else, we can use `xhost`, which is considered slightly more dangerous
|
||||
for multi-user environments.
|
||||
|
||||
## Caching and Reproducibility
|
||||
|
||||
In order to build Dangerzone environments, the script uses the following inputs:
|
||||
|
||||
* Dev environment:
|
||||
- Distro name and version. Together, these comprise the base container image.
|
||||
- `poetry.lock` and `pyproject.toml`. Together, these comprise the build
|
||||
context.
|
||||
* End-user environment:
|
||||
- Distro name and version. Together, these comprise the base container image.
|
||||
- `.deb` / `.rpm` Dangerzone package, as found under `deb_dist/` or `dist/`
|
||||
respectively.
|
||||
|
||||
Any change in these inputs busts the cache for the corresponding image. In
|
||||
theory, this means that the Dangerzone environment for each commit can be built
|
||||
reproducibly. In practice, there are some issues that we haven't covered yet:
|
||||
|
||||
1. The output images are:
|
||||
* Dev: `dangerzone.rocks/build/{distro_name}:{distro_version}`
|
||||
* End-user: `dangerzone.rocks/{distro_name}:{distro_version}`
|
||||
|
||||
These images do not contain the commit/version of the Dangerzone source they
|
||||
got created from, so each one overrides the other.
|
||||
2. The end-user environment expects a `.deb.` / `.rpm` tagged with the version
|
||||
of Dangerzone, but it doesn't insist being built from the current Dangerzone
|
||||
commit. This means that stale packages may be installed in the end-user
|
||||
environment.
|
||||
3. The base images may be different in various environments, depending on when
|
||||
they where pulled.
|
||||
|
||||
## State
|
||||
|
||||
The main goal behind these Dangerzone environments is to make them immutable,
|
||||
so that they do not require to be stored somewhere, but can be recreated from
|
||||
their images. Any change to these environments should therefore be reflected to
|
||||
their Dockerfile.
|
||||
|
||||
To enforce immutability, we delete the containers every time we run a command or
|
||||
an interactive shell exits. This means that these environments are suitable only
|
||||
for running Dangerzone commands, and not doing actual development in them
|
||||
(install an editor, configure bash prompts, etc.)
|
||||
|
||||
The only point where we allow mutability is the directory where Podman stores
|
||||
the images and stopped containers, which may be useful for developers. If this
|
||||
proves to be an issue, we will reconsider.
|
||||
|
|
@ -1,295 +0,0 @@
|
|||
# gVisor integration
|
||||
|
||||
> [!NOTE]
|
||||
> **Update on 2025-01-13:** There is no longer a copied container image under
|
||||
> `/home/dangerzone/dangerzone-image/rootfs`. We now reuse the same container
|
||||
> image both for the inner and outer container. See
|
||||
> [#1048](https://github.com/freedomofpress/dangerzone/issues/1048).
|
||||
|
||||
Dangerzone has relied on the container runtime available in each supported
|
||||
operating system (Docker Desktop on Windows / macOS, Podman on Linux) to isolate
|
||||
the host from the sanitization process. The problem with this type of isolation
|
||||
is that it exposes a rather large attack surface; the Linux kernel.
|
||||
|
||||
[gVisor](https://gvisor.dev/) is an application kernel, that emulates a
|
||||
substantial portion of the Linux Kernel API in Go. What's more interesting to
|
||||
Dangerzone is that it also offers an OCI runtime (`runsc`) that enables
|
||||
containers to transparently run this application kernel.
|
||||
|
||||
As of writing this, Dangerzone uses two containers to sanitize a document:
|
||||
* The first container reads a document from stdin, converts each page to pixels,
|
||||
and writes them to stdout.
|
||||
* The second container reads the pixels from a mounted volume (the host has
|
||||
taken care of this), and saves the final PDF to another mounted volume.
|
||||
|
||||
Our threat model considers the computation and output of the first container
|
||||
as **untrusted**, and the computation and output of the second container as
|
||||
trusted. For this reason, and because we are about to remove the need for the
|
||||
second container, our integration plan will focus on the first container.
|
||||
|
||||
## Design overview
|
||||
|
||||
Our integration goals are to:
|
||||
* Make gVisor available to all of our supported platforms.
|
||||
* Do not ask from users to run any commands on their system to do so.
|
||||
|
||||
Because gVisor does not support Windows and macOS systems out of the box,
|
||||
Dangerzone will be responsible for "shipping" gVisor to those users. It will do
|
||||
so using nested containers:
|
||||
* The **outer** container is the Docker/Podman container that Dangerzone uses
|
||||
already. This container acts as our **portability** layer. It's main purpose
|
||||
is to bundle all the necessary configuration files and program to run gVisor
|
||||
in all of our platforms.
|
||||
* The **inner** container is the gVisor container, created with `runsc`. This
|
||||
container acts as our **isolation layer**. It is responsible for running the
|
||||
Python code that rasterizes a document, in a way that will be fully isolated
|
||||
from the host.
|
||||
|
||||
### Building the container image
|
||||
|
||||
This nested container approach directly affects the container image as well,
|
||||
which will also have two layers:
|
||||
* The **outer** container image will contain just Python3 and `runsc`, the
|
||||
latter downloaded from the official gVisor website. It will also contain an
|
||||
entrypoint that will launch `runsc`. Finally, it will contain the **inner**
|
||||
container image (see below) as filesystem clone under
|
||||
`/dangerzone-image/rootfs`.
|
||||
* The **inner** container image is practically the original Dangerzone image, as
|
||||
we've always built it, which contains the necessary tooling to rasterize a
|
||||
document.
|
||||
|
||||
### Spawning the container
|
||||
|
||||
Spawning the container now becomes a multi-stage process:
|
||||
|
||||
The `Container` isolation provider spawns the container as before, with the
|
||||
following changes:
|
||||
|
||||
* It adds the `SYS_CHROOT` Linux capability, which was previously dropped, to
|
||||
the **outer** container. This capability is necessary to run `runsc`
|
||||
rootless, and is not inherited by the **inner** container.
|
||||
* It removes the `--userns keep-id` argument, which mapped the user outside the
|
||||
container to the same UID (normally `1000`) within the container. This was
|
||||
originally required when we were mounting host directories within the
|
||||
container, but this no longer applies to the gVisor integration. By removing
|
||||
this flag, the host user maps to the root user within the container (UID `0`).
|
||||
- In distributions that offer Podman version 4 or greater, we use the
|
||||
`--userns nomap` flag. This flag greatly minimizes the attack surface,
|
||||
since the host user is not mapped within the container at all.
|
||||
* We use our custom seccomp policy across container engines, since some do not
|
||||
allow the `ptrace` syscall (see
|
||||
[#846](https://github.com/freedomofpress/dangerzone/issues/846)).
|
||||
* It labels the **outer** container with the `container_engine_t` SELinux label.
|
||||
This label is reserved for running a container engine within a container, and
|
||||
is necessary in environments where SELinux is enabled in enforcing mode (see
|
||||
[#880](https://github.com/freedomofpress/dangerzone/issues/880)).
|
||||
|
||||
Then, the following happens when Podman/Docker spawns the container:
|
||||
|
||||
1. _(outer container)_ The entrypoint code finds from `sys.argv` the command
|
||||
that Dangerzone passed to the `docker run` / `podman run` invocation.
|
||||
Typically, this command is:
|
||||
|
||||
```
|
||||
/usr/bin/python3 -m dangerzone.conversion.doc_to_pixels
|
||||
```
|
||||
|
||||
2. _(outer container)_ The entrypoint code then creates an OCI config for
|
||||
`runsc` with the following properties:
|
||||
* Use UID/GID 1000 in the **inner** container image.
|
||||
* Run the command we detected on step 1.
|
||||
* Drop all Linux capabilities.
|
||||
* Limit the number of open files to 4096.
|
||||
* Use the `/dangerzone-image/rootfs` directory as the root path for the
|
||||
**inner** container.
|
||||
* Mount a gVisor view of the `procfs` hierarchy under `/proc` , and then
|
||||
mount `tmpfs` in the `/dev`, `/sys` and `/tmp` mount points. This way, no
|
||||
host-specific info may leak to the **inner** container.
|
||||
- Mount `tmpfs` on some more mountpoints where we want write access.
|
||||
3. _(outer container)_ If `RUNSC_DEBUG` has been specified, add some debug
|
||||
arguments to `runsc` (applies to development environments only).
|
||||
4. _(outer container)_ If `RUNSC_FLAGS` has been specified, pass some
|
||||
user-specified flags to `runsc` (applies to development environments only).
|
||||
5. _(outer container)_ Spawn `runsc` as a Python subprocess, and wait for it to
|
||||
complete.
|
||||
6. _(inner container)_ Read the document from stdin and write pixels to stdout.
|
||||
- In practice, nothing changes here, as far as the document conversion is
|
||||
concerned. The Python process transparently uses the emulated Linux Kernel
|
||||
API that gVisor provides.
|
||||
7. _(outer container)_ Exit the container with the same exit code as the inner
|
||||
container.
|
||||
|
||||
## Implementation details
|
||||
|
||||
### Creating the outer container image
|
||||
|
||||
In order to achieve the above, we add one more build stage in our Dockerfile
|
||||
(see [multi-stage builds](https://docs.docker.com/build/building/multi-stage/))
|
||||
that copies the result of the previous stages under `/dangerzone-image/rootfs`.
|
||||
Also, we install `runsc` and Python, and copy our entrypoint to that layer.
|
||||
|
||||
Here's how it looks like:
|
||||
|
||||
```dockerfile
|
||||
# NOTE: The following lines are appended to the end of our original Dockerfile.
|
||||
|
||||
# Install some commands required by the entrypoint.
|
||||
FROM alpine:latest
|
||||
RUN apk --no-cache -U upgrade && \
|
||||
apk --no-cache add \
|
||||
python3 \
|
||||
su-exec
|
||||
|
||||
# Add the previous build stage (`dangerzone-image`) as a filesystem clone under
|
||||
# the /dangerzone-image/rootfs directory.
|
||||
RUN mkdir --mode=0755 -p /dangerzone-image/rootfs
|
||||
COPY --from=dangerzone-image / /dangerzone-image/rootfs
|
||||
|
||||
# Download and install gVisor, based on the official instructions.
|
||||
RUN GVISOR_URL="https://storage.googleapis.com/gvisor/releases/release/latest/$(uname -m)"; \
|
||||
wget "${GVISOR_URL}/runsc" "${GVISOR_URL}/runsc.sha512" && \
|
||||
sha512sum -c runsc.sha512 && \
|
||||
rm -f runsc.sha512 && \
|
||||
chmod 555 runsc /entrypoint.py && \
|
||||
mv runsc /usr/bin/
|
||||
|
||||
COPY gvisor_wrapper/entrypoint.py /
|
||||
ENTRYPOINT ["/entrypoint.py"]
|
||||
```
|
||||
|
||||
### OCI config
|
||||
|
||||
The OCI config that gets produced is similar to this:
|
||||
|
||||
```json
|
||||
{
|
||||
"ociVersion": "1.0.0",
|
||||
"process": {
|
||||
"user": {
|
||||
"uid": 1000,
|
||||
"gid": 1000
|
||||
},
|
||||
"args": [
|
||||
"/usr/bin/python3",
|
||||
"-m",
|
||||
"dangerzone.conversion.doc_to_pixels"
|
||||
],
|
||||
"env": [
|
||||
"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
|
||||
"PYTHONPATH=/opt/dangerzone",
|
||||
"TERM=xterm"
|
||||
],
|
||||
"cwd": "/",
|
||||
"capabilities": {
|
||||
"bounding": [],
|
||||
"effective": [],
|
||||
"inheritable": [],
|
||||
"permitted": [],
|
||||
},
|
||||
"rlimits": [
|
||||
{
|
||||
"type": "RLIMIT_NOFILE",
|
||||
"hard": 4096,
|
||||
"soft": 4096
|
||||
}
|
||||
]
|
||||
},
|
||||
"root": {
|
||||
"path": "rootfs",
|
||||
"readonly": true
|
||||
},
|
||||
"hostname": "dangerzone",
|
||||
"mounts": [
|
||||
{
|
||||
"destination": "/proc",
|
||||
"type": "proc",
|
||||
"source": "proc"
|
||||
},
|
||||
{
|
||||
"destination": "/dev",
|
||||
"type": "tmpfs",
|
||||
"source": "tmpfs",
|
||||
"options": [
|
||||
"nosuid",
|
||||
"noexec",
|
||||
"nodev"
|
||||
]
|
||||
},
|
||||
{
|
||||
"destination": "/sys",
|
||||
"type": "tmpfs",
|
||||
"source": "tmpfs",
|
||||
"options": [
|
||||
"nosuid",
|
||||
"noexec",
|
||||
"nodev",
|
||||
"ro"
|
||||
]
|
||||
},
|
||||
{
|
||||
"destination": "/tmp",
|
||||
"type": "tmpfs",
|
||||
"source": "tmpfs",
|
||||
"options": [
|
||||
"nosuid",
|
||||
"noexec",
|
||||
"nodev"
|
||||
]
|
||||
},
|
||||
{
|
||||
"destination": "/home/dangerzone",
|
||||
"type": "tmpfs",
|
||||
"source": "tmpfs",
|
||||
"options": [
|
||||
"nosuid",
|
||||
"noexec",
|
||||
"nodev"
|
||||
]
|
||||
},
|
||||
{
|
||||
"destination": "/usr/lib/libreoffice/share/extensions/",
|
||||
"type": "tmpfs",
|
||||
"source": "tmpfs",
|
||||
"options": [
|
||||
"nosuid",
|
||||
"noexec",
|
||||
"nodev"
|
||||
]
|
||||
}
|
||||
],
|
||||
"linux": {
|
||||
"namespaces": [
|
||||
{
|
||||
"type": "pid"
|
||||
},
|
||||
{
|
||||
"type": "network"
|
||||
},
|
||||
{
|
||||
"type": "ipc"
|
||||
},
|
||||
{
|
||||
"type": "uts"
|
||||
},
|
||||
{
|
||||
"type": "mount"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
```
|
||||
|
||||
## Security considerations
|
||||
|
||||
* gVisor does not have an official release on Alpine Linux. The developers
|
||||
provide gVisor binaries from a GCS bucket. In order to verify the integrity of
|
||||
these binaries, they also provide a SHA-512 hash of the files.
|
||||
- If we choose to pin the hash, then we essentially pin gVisor, and we may
|
||||
lose security updates.
|
||||
|
||||
## Alternatives
|
||||
|
||||
gVisor can be integrated with Podman/Docker, but this is the case only on Linux.
|
||||
Because we want gVisor on Windows and macOS as well, we decided to not move
|
||||
forward with this approach.
|
||||
|
|
@ -1,14 +0,0 @@
|
|||
# Scripted QA
|
||||
|
||||
The `dev_scripts/qa.py` script runs the QA steps for a supported platform, in
|
||||
order to make sure that the dev does not skip something. These steps are taken
|
||||
from our [release instructions](../../RELEASE.md#qa).
|
||||
|
||||
The idea behind this script is that it will present each step to the user and
|
||||
ask them to perform it manually and specify it passes, in order to continue to
|
||||
the next one. For specific steps, it allows the user to run them automatically.
|
||||
In steps that require a Dangerzone dev environment, this script uses the
|
||||
`env.py` script to create one.
|
||||
|
||||
Including all the supported platforms in this script is still a work in
|
||||
progress.
|
||||
|
|
@ -1,67 +0,0 @@
|
|||
# Reproducible builds
|
||||
|
||||
We want to improve the transparency and auditability of our build artifacts, and
|
||||
a way to achieve this is via reproducible builds. For a broader understanding of
|
||||
what reproducible builds entail, check out https://reproducible-builds.org/.
|
||||
|
||||
Our build artifacts consist of:
|
||||
* Container images (`amd64` and `arm64` architectures)
|
||||
* macOS installers (for Intel and Apple Silicon CPUs)
|
||||
* Windows installer
|
||||
* Fedora packages (for regular Fedora distros and Qubes)
|
||||
* Debian packages (for Debian and Ubuntu)
|
||||
|
||||
As of writing this, only the following artifacts are reproducible:
|
||||
* Container images (see [#1047](https://github.com/freedomofpress/dangerzone/issues/1047))
|
||||
|
||||
In the following sections, we'll mention some specifics about enforcing
|
||||
reproducibility for each artifact type.
|
||||
|
||||
## Container image
|
||||
|
||||
### Updating the image
|
||||
|
||||
The fact that our image is reproducible also means that it's frozen in time.
|
||||
This means that rebuilding the image without updating our Dockerfile will
|
||||
**not** receive security updates.
|
||||
|
||||
Here are the necessary variables that make up our image in the `Dockerfile.env`
|
||||
file:
|
||||
* `DEBIAN_IMAGE_DIGEST`: The index digest for the Debian container image
|
||||
* `DEBIAN_ARCHIVE_DATE`: The Debian snapshot repo that we want to use
|
||||
* `GVISOR_ARCHIVE_DATE`: The gVisor APT repo that we want to use
|
||||
* `H2ORESTART_CHECKSUM`: The SHA-256 checksum of the H2ORestart plugin
|
||||
* `H2ORESTART_VERSION`: The version of the H2ORestart plugin
|
||||
|
||||
If you update these values in `Dockerfile.env`, you must also create a new
|
||||
Dockerfile with:
|
||||
|
||||
```
|
||||
make Dockerfile
|
||||
```
|
||||
|
||||
Updating `Dockerfile` without bumping `Dockerfile.in` is detected and should
|
||||
trigger a CI error.
|
||||
|
||||
### Reproducing the image
|
||||
|
||||
For a simple way to reproduce a Dangerzone container image, you can checkout the
|
||||
commit this image was built from (you can find it from the image tag in its
|
||||
`g<commit>` portion), retrieve the date it was built (also included in the image
|
||||
tag), and run the following command in any environment:
|
||||
|
||||
```
|
||||
./dev_scripts/reproduce-image.py \
|
||||
--debian-archive-date <date> \
|
||||
<digest>
|
||||
```
|
||||
|
||||
where:
|
||||
* `<date>` should be given in YYYYMMDD format, e.g, 20250226
|
||||
* `<digest>` is the SHA-256 hash of the image for the **current platform**, with
|
||||
or without the `sha256:` prefix.
|
||||
|
||||
This command will build a container image from the current Git commit and the
|
||||
provided date for the Debian archives. Then, it will compare the digest of the
|
||||
manifest against the provided one. This is a simple way to ensure that the
|
||||
created image is bit-for-bit reproducible.
|
||||
|
|
@ -1,222 +0,0 @@
|
|||
# Update notifications
|
||||
|
||||
This design document explains how the notification mechanism for Dangerzone
|
||||
updates works, what are its benefits and limitations, and what other
|
||||
alternatives we have considered. It has been adapted by discussions on GitHub
|
||||
issue [#189](https://github.com/freedomofpress/dangerzone/issues/189), and has
|
||||
been updated to reflect the current design.
|
||||
|
||||
A user-facing document on how update notifications work can be found in
|
||||
https://github.com/freedomofpress/dangerzone/wiki/Updates
|
||||
|
||||
## Design overview
|
||||
|
||||
This feature introduces a hamburger icon that will be visible across almost all
|
||||
of the Dangerzone windows. This will be used to notify the users about updates.
|
||||
|
||||
### First run
|
||||
|
||||
_We detect it's the first time Dangerzone runs because the
|
||||
`settings["updater_last_check"] is None`._
|
||||
|
||||
Add the following keys in our `settings.json` file.
|
||||
|
||||
* `"updater_check": None`: Whether to check for updates or not. `None` means
|
||||
that the user has not decided yet, and is the default.
|
||||
* `"updater_last_check": None`: The last time we checked for updates (in seconds
|
||||
from Unix epoch). None means that we haven't checked yet.
|
||||
* `"updater_latest_version": "0.4.2"`: The latest version that the Dangerzone
|
||||
updater has detected. By default it's the current version.
|
||||
* `"updater_latest_changelog": ""`: The latest changelog that the Dangerzone
|
||||
updater has detected. By default it's empty.
|
||||
* `"updater_errors: 0`: The number of update check errors that we have
|
||||
encountered in a row.
|
||||
|
||||
Note:
|
||||
|
||||
* If on Linux, make `"updater_check": False`, since we normally have
|
||||
other update channels for these platforms.
|
||||
|
||||
### Second run
|
||||
|
||||
_We detect it's the second time Dangerzone runs because
|
||||
`settings["updater_check"] is not None and settings["updater_last_check"] is
|
||||
None`._
|
||||
|
||||
Before starting up the main window, show this window:
|
||||
|
||||
* Title: Dangerzone Updater
|
||||
* Body:
|
||||
|
||||
> Do you want Dangerzone to automatically check for updates?
|
||||
>
|
||||
> If you accept, Dangerzone will check the latest releases page in github.com
|
||||
> on startup. Otherwise it will make no network requests and won't inform you
|
||||
> about new releases.
|
||||
>
|
||||
> If you prefer another way of getting notified about new releases, we suggest adding
|
||||
> to your RSS reader our [Mastodon feed](https://fosstodon.org/@dangerzone.rss). For more information
|
||||
> about updates, check [this webpage](https://github.com/freedomofpress/dangerzone/wiki/Updates).
|
||||
|
||||
* Buttons:
|
||||
- Check Automaticaly: Store `settings["updater_check"] = True`
|
||||
- Don't Check: Store `settings["updater_check"] = False`
|
||||
|
||||
Note:
|
||||
* Users will be able to change their choice from the hamburger menu, which will
|
||||
contain an entry called "Check for updates", that users can check and uncheck.
|
||||
|
||||
### Subsequent runs
|
||||
|
||||
_We perform the following only if `settings["updater_check"] == True`._
|
||||
|
||||
1. Spawn a new thread so that we don't block the main window.
|
||||
2. Check if we have cached information about a release (version and changelog).
|
||||
If yes, return those immediately.
|
||||
3. Check if the last time we checked for new releases was less than 12 hours
|
||||
ago. In that case, skip this update check so that we don't leak telemetry
|
||||
stats to GitHub.
|
||||
4. Hit the GitHub releases API and get the [latest release](https://api.github.com/repos/freedomofpress/dangerzone/releases/latest).
|
||||
Store the current time as the last check time, even if the call fails.
|
||||
5. Check if the latest release matches `settings["updater_latest_version"]`. If
|
||||
yes, return an empty update report.
|
||||
6. If a new update has been detected, return the version number and the
|
||||
changelog.
|
||||
7. Add a green bubble in the notification icon, and a menu entry called "New
|
||||
version available".
|
||||
8. Users who click on this entry will see a dialog with more info:
|
||||
|
||||
* Title: "Dangerzone v0.5.0 has been released"
|
||||
* Body:
|
||||
|
||||
> A new Dangerzone version been released. Please visit our [downloads page](https://dangerzone.rocks#downloads) to install this update.
|
||||
>
|
||||
> (Show changelog rendered from Markdown in a collapsible text box)
|
||||
* Buttons:
|
||||
- OK: Return
|
||||
|
||||
Notes:
|
||||
* Any successful attempt to fetch info from GitHub will result in clearing the
|
||||
`settings["updater_errors"]` key.
|
||||
|
||||
### Error handling
|
||||
|
||||
_We trigger error handling when the updater thread encounters an error (either
|
||||
due to an HTTPS failure or a Python exception) and does not complete
|
||||
successfully._
|
||||
|
||||
1. Bump the number of errors we've encountered in a row
|
||||
(`settings["updater_errors"] += 1`)
|
||||
2. Return an update report with the error we've encountered.
|
||||
3. Update the hamburger menu with a red notification bubble, and add a menu
|
||||
entry called "Update error".
|
||||
4. If a user clicks on this menu entry, show a dialog window:
|
||||
* Title: "Update check error"
|
||||
* Body:
|
||||
|
||||
> Something went wrong while checking for Dangerzone updates:
|
||||
>
|
||||
> You are strongly advised to visit our [downloads page](https://dangerzone.rocks#downloads) and check for new updates manually, or consult [this page](https://github.com/freedomofpress/dangerzone/wiki/Updates) for common causes of errors . Alternatively, you can uncheck "Check for updates", if you are in an air-gapped environment and have another way of learning about updates.
|
||||
>
|
||||
> (Show the latest error message in a scrollable, copyable text box)
|
||||
|
||||
* Buttons:
|
||||
- Close: Return
|
||||
|
||||
## Key Benefits
|
||||
|
||||
1. The above approach future-proofs Dangerzone against API changes or bugs in
|
||||
the update check process, by asking users to manually visit
|
||||
https://dangerzone.rocks.
|
||||
2. If we want to draw the attention of users to immediately install a release,
|
||||
we can do so in the release body, which we will show in a pop-up window.
|
||||
3. If we are aware of issues that prevent updates, we can add them in the wiki
|
||||
page that we show in the error popup. Wiki pages are not versioned, so we can
|
||||
add useful info even after a release.
|
||||
|
||||
## Security Considerations
|
||||
|
||||
Because this approach does not download binaries / auto-updates, it **does not
|
||||
add any more security issues** than the existing, manual way of installing
|
||||
updates. These issues have to do with a compromised/malicous GitHub service, and
|
||||
are the following:
|
||||
|
||||
1. GitHub pages can alter the contents of our main site
|
||||
(https://dangerzone.rocks)
|
||||
2. GitHub releases can serve an older, vulnerable version of Dangerzone, instead
|
||||
of a new update.
|
||||
3. GitHub releases can serve a malicious binary (requires a joint operation from
|
||||
a malicious CA as well, for extra legitimacy).
|
||||
4. GitHub releases can silently drop updates.
|
||||
5. GitHub releases can know which users download Dangerzone updates.
|
||||
6. Network attackers can know that a user has Dangerzone installed (because we ask the user to visit https://dangerzone.rocks)
|
||||
|
||||
A good update framework would probably defend against 1,2,3. This is not to say
|
||||
that our users are currently unprotected, since 1-4 can be detected by the
|
||||
general public and the developers (unless GitHub specifically targets an
|
||||
individual, but that's another story).
|
||||
|
||||
## Usability Considerations
|
||||
|
||||
1. We do not have an update story for users that only use the Dangerzone CLI. A
|
||||
good assumption is that they are on Linux, so they auto-update.
|
||||
|
||||
## Alternatives
|
||||
|
||||
We researched a bit on this subject and found out that there are update
|
||||
frameworks that do this job for us. While working on this issue, we decided that
|
||||
integrating with one framework will certainly take a bit of work, especially
|
||||
given that we target both Windows and MacOS systems. In the meantime though, we
|
||||
didn't want to have releases out without including at least a notification
|
||||
channel, since staying behind on updates has a huge negative impact on the
|
||||
users' safety.
|
||||
|
||||
The update frameworks that we learned about are:
|
||||
|
||||
## Sparkle Project
|
||||
|
||||
[Sparkle project](https://sparkle-project.org) seems to be the de-facto update
|
||||
framework in MacOS. Integrators in practice need to care about two things:
|
||||
creating a proper `Appcast.xml` file on the server-side, and calling the Sparkle
|
||||
code from the client-side. These are covered in the project's
|
||||
[documentation](https://sparkle-project.org/documentation/).
|
||||
|
||||
The client-side part is not very straight-forward, since Sparkle is written in
|
||||
Objective-C. Thankfully, there are others who have ventured into this before:
|
||||
https://fman.io/blog/codesigning-and-automatic-updates-for-pyqt-apps/
|
||||
|
||||
The server-side part is also not very straight-forward. For integrators that use
|
||||
GitHub releases (like us), this issue may be of help:
|
||||
https://github.com/sparkle-project/Sparkle/issues/648
|
||||
|
||||
The Windows platform is not covered by Sparkle itself, but there are other
|
||||
projects, such as [WinSparkle](https://winsparkle.org/), that follow a similar
|
||||
approach. I see that there's a [Python library (`pywinsparkle`)](https://pypi.org/project/pywinsparkle/)
|
||||
for interacting with WinSparkle, so this may alleviate some pains.
|
||||
|
||||
Note that the Sparkle project is not a silver bullet. Development missteps can
|
||||
happen, and users can be left without updates. Here's an [example issue](https://github.com/sparkle-project/Sparkle/issues/345) that showcases this.
|
||||
|
||||
## The Update Framework
|
||||
|
||||
[The Update Framework](https://theupdateframework.io/) is a graduated CNCF
|
||||
project hosted by Linux Foundation. It's based on the
|
||||
[Thandy](https://chromium.googlesource.com/chromium/src.git/+/master/docs/updater/protocol_3_1.md)
|
||||
updater for Tor. It's [not widely adopted](https://github.com/sparkle-project/Sparkle/issues/345), but some of its
|
||||
adopters are high-profile, and it has passed security audits.
|
||||
|
||||
It's more of a [specification](https://github.com/sparkle-project/Sparkle/issues/345)
|
||||
and less of a software project, although a well-maintained
|
||||
[reference implementation](https://github.com/sparkle-project/Sparkle/issues/345)
|
||||
in Python exists. Also, a [Python project (`tufup`)](https://doc.qt.io/qtinstallerframework/ifw-updates.html)
|
||||
that builds upon this implementation makes it even easier to generate the
|
||||
required keys and files.
|
||||
|
||||
Regardless of whether we use it, knowing about the [threat vectors](https://theupdateframework.io/security/) that it's protecting against is very important.
|
||||
|
||||
## Other Projects
|
||||
|
||||
* Qt has some updater framework as well: https://doc.qt.io/qtinstallerframework/ifw-updates.html
|
||||
* Google Chrome has it's own updater framework: https://chromium.googlesource.com/chromium/src.git/+/master/docs/updater/protocol_3_1.md
|
||||
* Keepass rolls out its own way to update: https://github.com/keepassxreboot/keepassxc/blob/develop/src/updatecheck/UpdateChecker.cpp
|
||||
* [PyUpdater](https://github.com/Digital-Sapphire/PyUpdater) was another popular updater project for Python, but is now archived.
|
||||
|
|
@ -1,53 +0,0 @@
|
|||
# Podman Desktop support
|
||||
|
||||
Starting with Dangerzone 0.9.0, it is possible to use Podman Desktop on
|
||||
Windows and macOS. The support for this container runtime is currently only
|
||||
experimental. If you try it out and encounter issues, please reach to us, we'll
|
||||
be glad to help.
|
||||
|
||||
With [Podman Desktop](https://podman-desktop.io/) installed on your machine,
|
||||
here are the required steps to change the dangerzone container runtime.
|
||||
|
||||
You will be required to open a terminal and follow these steps:
|
||||
|
||||
## On macOS
|
||||
|
||||
You will need to configure podman to access the shared Dangerzone resources:
|
||||
|
||||
```bash
|
||||
podman machine stop
|
||||
podman machine rm
|
||||
cat > ~/.config/containers/containers.conf <<EOF
|
||||
[machine]
|
||||
volumes = ["/Users:/Users", "/private:/private", "/var/folders:/var/folders", "/Applications/Dangerzone.app:/Applications/Dangerzone.app"]
|
||||
EOF
|
||||
podman machine init
|
||||
podman machine set --rootful=false
|
||||
podman machine start
|
||||
```
|
||||
Then, set the container runtime to podman using this command:
|
||||
|
||||
```bash
|
||||
/Applications/Dangerzone.app/Contents/MacOS/dangerzone-cli --set-container-runtime podman
|
||||
```
|
||||
|
||||
In order to get back to the default behaviour (Docker Desktop on macOS), pass
|
||||
the `default` value instead:
|
||||
|
||||
```bash
|
||||
/Applications/Dangerzone.app/Contents/MacOS/dangerzone-cli --set-container-runtime default
|
||||
```
|
||||
|
||||
## On Windows
|
||||
|
||||
To set the container runtime to podman, use this command:
|
||||
|
||||
```bash
|
||||
'C:\Program Files\Dangerzone\dangerzone-cli.exe' --set-container-runtime podman
|
||||
```
|
||||
|
||||
To revert back to the default behavior, pass the `default` value:
|
||||
|
||||
```bash
|
||||
'C:\Program Files\Dangerzone\dangerzone-cli.exe' --set-container-runtime podman
|
||||
```
|
||||
379
dodo.py
379
dodo.py
|
|
@ -1,379 +0,0 @@
|
|||
import json
|
||||
import os
|
||||
import platform
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
from doit.action import CmdAction
|
||||
|
||||
ARCH = "arm64" if platform.machine() == "arm64" else "i686"
|
||||
VERSION = open("share/version.txt").read().strip()
|
||||
FEDORA_VERSIONS = ["40", "41", "42"]
|
||||
|
||||
### Global parameters
|
||||
|
||||
CONTAINER_RUNTIME = os.environ.get("CONTAINER_RUNTIME", "podman")
|
||||
DEFAULT_RELEASE_DIR = Path.home() / "release-assets" / VERSION
|
||||
RELEASE_DIR = Path(os.environ.get("RELEASE_DIR", DEFAULT_RELEASE_DIR))
|
||||
APPLE_ID = os.environ.get("APPLE_ID", None)
|
||||
|
||||
### Task Parameters
|
||||
|
||||
PARAM_APPLE_ID = {
|
||||
"name": "apple_id",
|
||||
"long": "apple-id",
|
||||
"default": APPLE_ID,
|
||||
"help": "The Apple developer ID that will be used to sign the .dmg",
|
||||
}
|
||||
|
||||
### File dependencies
|
||||
#
|
||||
# Define all the file dependencies for our tasks in a single place, since some file
|
||||
# dependencies are shared between tasks.
|
||||
|
||||
|
||||
def list_files(path, recursive=False):
|
||||
"""List files in a directory, and optionally traverse into subdirectories."""
|
||||
glob_fn = Path(path).rglob if recursive else Path(path).glob
|
||||
return [f for f in glob_fn("*") if f.is_file() and not f.suffix == ".pyc"]
|
||||
|
||||
|
||||
def list_language_data():
|
||||
"""List the expected language data that Dangerzone downloads and stores locally."""
|
||||
tessdata_dir = Path("share") / "tessdata"
|
||||
langs = json.loads(open(tessdata_dir.parent / "ocr-languages.json").read()).values()
|
||||
targets = [tessdata_dir / f"{lang}.traineddata" for lang in langs]
|
||||
return targets
|
||||
|
||||
|
||||
TESSDATA_DEPS = ["install/common/download-tessdata.py", "share/ocr-languages.json"]
|
||||
TESSDATA_TARGETS = list_language_data()
|
||||
|
||||
IMAGE_DEPS = [
|
||||
"Dockerfile",
|
||||
*list_files("dangerzone/conversion"),
|
||||
*list_files("dangerzone/container_helpers"),
|
||||
"install/common/build-image.py",
|
||||
]
|
||||
IMAGE_TARGETS = ["share/container.tar", "share/image-id.txt"]
|
||||
|
||||
SOURCE_DEPS = [
|
||||
*list_files("assets"),
|
||||
*list_files("share"),
|
||||
*list_files("dangerzone", recursive=True),
|
||||
]
|
||||
|
||||
PYTHON_DEPS = ["poetry.lock", "pyproject.toml"]
|
||||
|
||||
DMG_DEPS = [
|
||||
*list_files("install/macos"),
|
||||
*TESSDATA_TARGETS,
|
||||
*IMAGE_TARGETS,
|
||||
*PYTHON_DEPS,
|
||||
*SOURCE_DEPS,
|
||||
]
|
||||
|
||||
LINUX_DEPS = [
|
||||
*list_files("install/linux"),
|
||||
*IMAGE_TARGETS,
|
||||
*PYTHON_DEPS,
|
||||
*SOURCE_DEPS,
|
||||
]
|
||||
|
||||
DEB_DEPS = [*LINUX_DEPS, *list_files("debian")]
|
||||
RPM_DEPS = [*LINUX_DEPS, *list_files("qubes")]
|
||||
|
||||
|
||||
def copy_dir(src, dst):
|
||||
"""Copy a directory to a destination dir, and overwrite it if it exists."""
|
||||
shutil.rmtree(dst, ignore_errors=True)
|
||||
shutil.copytree(src, dst)
|
||||
|
||||
|
||||
def create_release_dir():
|
||||
RELEASE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
(RELEASE_DIR / "tmp").mkdir(exist_ok=True)
|
||||
|
||||
|
||||
def build_linux_pkg(distro, version, cwd, qubes=False):
|
||||
"""Generic command for building a .deb/.rpm in a Dangerzone dev environment."""
|
||||
pkg = "rpm" if distro == "fedora" else "deb"
|
||||
cmd = [
|
||||
"python3",
|
||||
"./dev_scripts/env.py",
|
||||
"--distro",
|
||||
distro,
|
||||
"--version",
|
||||
version,
|
||||
"run",
|
||||
"--no-gui",
|
||||
"--dev",
|
||||
f"./dangerzone/install/linux/build-{pkg}.py",
|
||||
]
|
||||
if qubes:
|
||||
cmd += ["--qubes"]
|
||||
return CmdAction(" ".join(cmd), cwd=cwd)
|
||||
|
||||
|
||||
def build_deb(cwd):
|
||||
"""Build a .deb package on Debian Bookworm."""
|
||||
return build_linux_pkg(distro="debian", version="bookworm", cwd=cwd)
|
||||
|
||||
|
||||
def build_rpm(version, cwd, qubes=False):
|
||||
"""Build an .rpm package on the requested Fedora distro."""
|
||||
return build_linux_pkg(distro="fedora", version=version, cwd=cwd, qubes=qubes)
|
||||
|
||||
|
||||
### Tasks
|
||||
|
||||
|
||||
def task_clean_container_runtime():
|
||||
"""Clean the storage space of the container runtime."""
|
||||
return {
|
||||
"actions": None,
|
||||
"clean": [
|
||||
[CONTAINER_RUNTIME, "system", "prune", "-a", "-f"],
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def task_check_container_runtime():
|
||||
"""Test that the container runtime is ready."""
|
||||
return {
|
||||
"actions": [
|
||||
["which", CONTAINER_RUNTIME],
|
||||
[CONTAINER_RUNTIME, "ps"],
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def task_macos_check_cert():
|
||||
"""Test that the Apple developer certificate can be used."""
|
||||
return {
|
||||
"actions": [
|
||||
"xcrun notarytool history --apple-id %(apple_id)s --keychain-profile dz-notarytool-release-key"
|
||||
],
|
||||
"params": [PARAM_APPLE_ID],
|
||||
}
|
||||
|
||||
|
||||
def task_macos_check_system():
|
||||
"""Run macOS specific system checks, as well as the generic ones."""
|
||||
return {
|
||||
"actions": None,
|
||||
"task_dep": ["check_container_runtime", "macos_check_cert"],
|
||||
}
|
||||
|
||||
|
||||
def task_init_release_dir():
|
||||
"""Create a directory for release artifacts."""
|
||||
return {
|
||||
"actions": [create_release_dir],
|
||||
"clean": [f"rm -rf {RELEASE_DIR}"],
|
||||
}
|
||||
|
||||
|
||||
def task_download_tessdata():
|
||||
"""Download the Tesseract data using ./install/common/download-tessdata.py"""
|
||||
return {
|
||||
"actions": ["python install/common/download-tessdata.py"],
|
||||
"file_dep": TESSDATA_DEPS,
|
||||
"targets": TESSDATA_TARGETS,
|
||||
"clean": True,
|
||||
}
|
||||
|
||||
|
||||
def task_build_image():
|
||||
"""Build the container image using ./install/common/build-image.py"""
|
||||
img_src = "share/container.tar"
|
||||
img_dst = RELEASE_DIR / f"container-{VERSION}-{ARCH}.tar" # FIXME: Add arch
|
||||
img_id_src = "share/image-id.txt"
|
||||
img_id_dst = RELEASE_DIR / "image-id.txt" # FIXME: Add arch
|
||||
|
||||
return {
|
||||
"actions": [
|
||||
f"python install/common/build-image.py --runtime={CONTAINER_RUNTIME}",
|
||||
["cp", img_src, img_dst],
|
||||
["cp", img_id_src, img_id_dst],
|
||||
],
|
||||
"file_dep": IMAGE_DEPS,
|
||||
"targets": [img_src, img_dst, img_id_src, img_id_dst],
|
||||
"task_dep": ["init_release_dir", "check_container_runtime"],
|
||||
"clean": True,
|
||||
}
|
||||
|
||||
|
||||
def task_poetry_install():
|
||||
"""Setup the Poetry environment"""
|
||||
return {"actions": ["poetry sync"], "clean": ["poetry env remove --all"]}
|
||||
|
||||
|
||||
def task_macos_build_dmg():
|
||||
"""Build the macOS .dmg file for Dangerzone."""
|
||||
dz_dir = RELEASE_DIR / "tmp" / "macos"
|
||||
dmg_src = dz_dir / "dist" / "Dangerzone.dmg"
|
||||
dmg_dst = RELEASE_DIR / f"Dangerzone-{VERSION}-{ARCH}.dmg" # FIXME: Add -arch
|
||||
|
||||
return {
|
||||
"actions": [
|
||||
(copy_dir, [".", dz_dir]),
|
||||
f"cd {dz_dir} && poetry run install/macos/build-app.py --with-codesign",
|
||||
(
|
||||
"xcrun notarytool submit --wait --apple-id %(apple_id)s"
|
||||
f" --keychain-profile dz-notarytool-release-key {dmg_src}"
|
||||
),
|
||||
f"xcrun stapler staple {dmg_src}",
|
||||
["cp", dmg_src, dmg_dst],
|
||||
["rm", "-rf", dz_dir],
|
||||
],
|
||||
"params": [PARAM_APPLE_ID],
|
||||
"file_dep": DMG_DEPS,
|
||||
"task_dep": [
|
||||
"macos_check_system",
|
||||
"init_release_dir",
|
||||
"poetry_install",
|
||||
"download_tessdata",
|
||||
],
|
||||
"targets": [dmg_src, dmg_dst],
|
||||
"clean": True,
|
||||
}
|
||||
|
||||
|
||||
def task_debian_env():
|
||||
"""Build a Debian Bookworm dev environment."""
|
||||
return {
|
||||
"actions": [
|
||||
[
|
||||
"python3",
|
||||
"./dev_scripts/env.py",
|
||||
"--distro",
|
||||
"debian",
|
||||
"--version",
|
||||
"bookworm",
|
||||
"build-dev",
|
||||
]
|
||||
],
|
||||
"task_dep": ["check_container_runtime"],
|
||||
}
|
||||
|
||||
|
||||
def task_debian_deb():
|
||||
"""Build a Debian package for Debian Bookworm."""
|
||||
dz_dir = RELEASE_DIR / "tmp" / "debian"
|
||||
deb_name = f"dangerzone_{VERSION}-1_amd64.deb"
|
||||
deb_src = dz_dir / "deb_dist" / deb_name
|
||||
deb_dst = RELEASE_DIR / deb_name
|
||||
|
||||
return {
|
||||
"actions": [
|
||||
(copy_dir, [".", dz_dir]),
|
||||
build_deb(cwd=dz_dir),
|
||||
["cp", deb_src, deb_dst],
|
||||
["rm", "-rf", dz_dir],
|
||||
],
|
||||
"file_dep": DEB_DEPS,
|
||||
"task_dep": ["init_release_dir", "debian_env"],
|
||||
"targets": [deb_dst],
|
||||
"clean": True,
|
||||
}
|
||||
|
||||
|
||||
def task_fedora_env():
|
||||
"""Build Fedora dev environments."""
|
||||
for version in FEDORA_VERSIONS:
|
||||
yield {
|
||||
"name": version,
|
||||
"doc": f"Build Fedora {version} dev environments",
|
||||
"actions": [
|
||||
[
|
||||
"python3",
|
||||
"./dev_scripts/env.py",
|
||||
"--distro",
|
||||
"fedora",
|
||||
"--version",
|
||||
version,
|
||||
"build-dev",
|
||||
],
|
||||
],
|
||||
"task_dep": ["check_container_runtime"],
|
||||
}
|
||||
|
||||
|
||||
def task_fedora_rpm():
|
||||
"""Build Fedora packages for every supported version."""
|
||||
for version in FEDORA_VERSIONS:
|
||||
for qubes in (True, False):
|
||||
qubes_ident = "-qubes" if qubes else ""
|
||||
qubes_desc = " for Qubes" if qubes else ""
|
||||
dz_dir = RELEASE_DIR / "tmp" / f"f{version}{qubes_ident}"
|
||||
rpm_names = [
|
||||
f"dangerzone{qubes_ident}-{VERSION}-1.fc{version}.x86_64.rpm",
|
||||
f"dangerzone{qubes_ident}-{VERSION}-1.fc{version}.src.rpm",
|
||||
]
|
||||
rpm_src = [dz_dir / "dist" / rpm_name for rpm_name in rpm_names]
|
||||
rpm_dst = [RELEASE_DIR / rpm_name for rpm_name in rpm_names]
|
||||
|
||||
yield {
|
||||
"name": version + qubes_ident,
|
||||
"doc": f"Build a Fedora {version} package{qubes_desc}",
|
||||
"actions": [
|
||||
(copy_dir, [".", dz_dir]),
|
||||
build_rpm(version, cwd=dz_dir, qubes=qubes),
|
||||
["cp", *rpm_src, RELEASE_DIR],
|
||||
["rm", "-rf", dz_dir],
|
||||
],
|
||||
"file_dep": RPM_DEPS,
|
||||
"task_dep": ["init_release_dir", f"fedora_env:{version}"],
|
||||
"targets": rpm_dst,
|
||||
"clean": True,
|
||||
}
|
||||
|
||||
|
||||
def task_git_archive():
|
||||
"""Build a Git archive of the repo."""
|
||||
target = f"{RELEASE_DIR}/dangerzone-{VERSION}.tar.gz"
|
||||
return {
|
||||
"actions": [
|
||||
f"git archive --format=tar.gz -o {target} --prefix=dangerzone/ v{VERSION}"
|
||||
],
|
||||
"targets": [target],
|
||||
"task_dep": ["init_release_dir"],
|
||||
}
|
||||
|
||||
|
||||
#######################################################################################
|
||||
#
|
||||
# END OF TASKS
|
||||
#
|
||||
# The following task should be the LAST one in the dodo file, so that it runs first when
|
||||
# running `do clean`.
|
||||
|
||||
|
||||
def clean_prompt():
|
||||
ans = input(
|
||||
f"""
|
||||
You have not specified a target to clean.
|
||||
This means that doit will clean the following targets:
|
||||
|
||||
* ALL the containers, images, and build cache in {CONTAINER_RUNTIME.capitalize()}
|
||||
* ALL the built targets and directories
|
||||
|
||||
For a full list of the targets that doit will clean, run: doit clean --dry-run
|
||||
|
||||
Are you sure you want to clean everything (y/N): \
|
||||
"""
|
||||
)
|
||||
if ans.lower() in ["yes", "y"]:
|
||||
return
|
||||
else:
|
||||
print("Exiting...")
|
||||
exit(1)
|
||||
|
||||
|
||||
def task_clean_prompt():
|
||||
"""Make sure that the user really wants to run the clean tasks."""
|
||||
return {
|
||||
"actions": None,
|
||||
"clean": [clean_prompt],
|
||||
}
|
||||
|
|
@ -1,62 +1,20 @@
|
|||
import argparse
|
||||
import gzip
|
||||
import os
|
||||
import platform
|
||||
import secrets
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
BUILD_CONTEXT = "dangerzone"
|
||||
IMAGE_NAME = "dangerzone.rocks/dangerzone"
|
||||
BUILD_CONTEXT = "dangerzone/"
|
||||
TAG = "dangerzone.rocks/dangerzone:latest"
|
||||
REQUIREMENTS_TXT = "container-pip-requirements.txt"
|
||||
if platform.system() in ["Darwin", "Windows"]:
|
||||
CONTAINER_RUNTIME = "docker"
|
||||
elif platform.system() == "Linux":
|
||||
CONTAINER_RUNTIME = "podman"
|
||||
|
||||
|
||||
def str2bool(v):
|
||||
if isinstance(v, bool):
|
||||
return v
|
||||
if v.lower() in ("yes", "true", "t", "y", "1"):
|
||||
return True
|
||||
elif v.lower() in ("no", "false", "f", "n", "0"):
|
||||
return False
|
||||
else:
|
||||
raise argparse.ArgumentTypeError("Boolean value expected.")
|
||||
|
||||
|
||||
def determine_git_tag():
|
||||
# Designate a unique tag for this image, depending on the Git commit it was created
|
||||
# from:
|
||||
# 1. If created from a Git tag (e.g., 0.8.0), the image tag will be `0.8.0`.
|
||||
# 2. If created from a commit, it will be something like `0.8.0-31-g6bdaa7a`.
|
||||
# 3. If the contents of the Git repo are dirty, we will append a unique identifier
|
||||
# for this run, something like `0.8.0-31-g6bdaa7a-fdcb` or `0.8.0-fdcb`.
|
||||
dirty_ident = secrets.token_hex(2)
|
||||
return (
|
||||
subprocess.check_output(
|
||||
[
|
||||
"git",
|
||||
"describe",
|
||||
"--long",
|
||||
"--first-parent",
|
||||
f"--dirty=-{dirty_ident}",
|
||||
],
|
||||
)
|
||||
.decode()
|
||||
.strip()[1:] # remove the "v" prefix of the tag.
|
||||
)
|
||||
|
||||
|
||||
def determine_debian_archive_date():
|
||||
"""Get the date of the Debian archive from Dockerfile.env."""
|
||||
for env in Path("Dockerfile.env").read_text().split("\n"):
|
||||
if env.startswith("DEBIAN_ARCHIVE_DATE"):
|
||||
return env.split("=")[1]
|
||||
raise Exception(
|
||||
"Could not find 'DEBIAN_ARCHIVE_DATE' build argument in Dockerfile.env"
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
|
|
@ -66,85 +24,108 @@ def main():
|
|||
help=f"The container runtime for building the image (default: {CONTAINER_RUNTIME})",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--platform",
|
||||
default=None,
|
||||
help=f"The platform for building the image (default: current platform)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
"-o",
|
||||
default=str(Path("share") / "container.tar"),
|
||||
help="Path to store the container image",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--use-cache",
|
||||
type=str2bool,
|
||||
nargs="?",
|
||||
default=True,
|
||||
const=True,
|
||||
help="Use the builder's cache to speed up the builds",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--tag",
|
||||
default=None,
|
||||
help="Provide a custom tag for the image (for development only)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--debian-archive-date",
|
||||
"-d",
|
||||
default=determine_debian_archive_date(),
|
||||
help="Use a specific Debian snapshot archive, by its date (default %(default)s)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dry",
|
||||
default=False,
|
||||
"--no-save",
|
||||
action="store_true",
|
||||
help="Do not run any commands, just print what would happen",
|
||||
help="Do not save the container image as a tarball in share/container.tar.gz",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--compress-level",
|
||||
type=int,
|
||||
choices=range(0, 10),
|
||||
default=9,
|
||||
help="The Gzip compression level, from 0 (lowest) to 9 (highest, default)",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
tag = args.tag or f"{args.debian_archive_date}-{determine_git_tag()}"
|
||||
image_name_tagged = f"{IMAGE_NAME}:{tag}"
|
||||
print("Exporting container pip dependencies")
|
||||
with ContainerPipDependencies():
|
||||
print("Pulling base image")
|
||||
subprocess.run(
|
||||
[
|
||||
args.runtime,
|
||||
"pull",
|
||||
"alpine:latest",
|
||||
],
|
||||
check=True,
|
||||
)
|
||||
|
||||
print(f"Will tag the container image as '{image_name_tagged}'")
|
||||
image_id_path = Path("share") / "image-id.txt"
|
||||
if not args.dry:
|
||||
with open(image_id_path, "w") as f:
|
||||
f.write(tag)
|
||||
print("Building container image")
|
||||
subprocess.run(
|
||||
[
|
||||
args.runtime,
|
||||
"build",
|
||||
BUILD_CONTEXT,
|
||||
"--build-arg",
|
||||
f"REQUIREMENTS_TXT={REQUIREMENTS_TXT}",
|
||||
"-f",
|
||||
"Dockerfile",
|
||||
"--tag",
|
||||
TAG,
|
||||
],
|
||||
check=True,
|
||||
)
|
||||
|
||||
# Build the container image, and tag it with the calculated tag
|
||||
print("Building container image")
|
||||
cache_args = [] if args.use_cache else ["--no-cache"]
|
||||
platform_args = [] if not args.platform else ["--platform", args.platform]
|
||||
rootless_args = [] if args.runtime == "docker" else ["--rootless"]
|
||||
rootless_args = []
|
||||
dry_args = [] if not args.dry else ["--dry"]
|
||||
if not args.no_save:
|
||||
print("Saving container image")
|
||||
cmd = subprocess.Popen(
|
||||
[
|
||||
CONTAINER_RUNTIME,
|
||||
"save",
|
||||
TAG,
|
||||
],
|
||||
stdout=subprocess.PIPE,
|
||||
)
|
||||
|
||||
subprocess.run(
|
||||
print("Compressing container image")
|
||||
chunk_size = 4 << 20
|
||||
with gzip.open(
|
||||
"share/container.tar.gz",
|
||||
"wb",
|
||||
compresslevel=args.compress_level,
|
||||
) as gzip_f:
|
||||
while True:
|
||||
chunk = cmd.stdout.read(chunk_size)
|
||||
if len(chunk) > 0:
|
||||
gzip_f.write(chunk)
|
||||
else:
|
||||
break
|
||||
cmd.wait(5)
|
||||
|
||||
print("Looking up the image id")
|
||||
image_id = subprocess.check_output(
|
||||
[
|
||||
sys.executable,
|
||||
str(Path("dev_scripts") / "repro-build.py"),
|
||||
"build",
|
||||
"--runtime",
|
||||
args.runtime,
|
||||
"--build-arg",
|
||||
f"DEBIAN_ARCHIVE_DATE={args.debian_archive_date}",
|
||||
"--datetime",
|
||||
args.debian_archive_date,
|
||||
*dry_args,
|
||||
*cache_args,
|
||||
*platform_args,
|
||||
*rootless_args,
|
||||
"--tag",
|
||||
image_name_tagged,
|
||||
"--output",
|
||||
args.output,
|
||||
"-f",
|
||||
"Dockerfile",
|
||||
BUILD_CONTEXT,
|
||||
"image",
|
||||
"list",
|
||||
"--format",
|
||||
"{{.ID}}",
|
||||
TAG,
|
||||
],
|
||||
check=True,
|
||||
text=True,
|
||||
)
|
||||
with open("share/image-id.txt", "w") as f:
|
||||
f.write(image_id)
|
||||
|
||||
|
||||
class ContainerPipDependencies:
|
||||
"""Generates PIP dependencies within container"""
|
||||
|
||||
def __enter__(self):
|
||||
try:
|
||||
container_requirements_txt = subprocess.check_output(
|
||||
["poetry", "export", "--only", "container"], universal_newlines=True
|
||||
)
|
||||
except subprocess.CalledProcessError as e:
|
||||
print("FAILURE", e.returncode, e.output)
|
||||
print(f"REQUIREMENTS: {container_requirements_txt}")
|
||||
# XXX Export container dependencies and exclude pymupdfb since it is not needed in container
|
||||
req_txt_pymupdfb_stripped = container_requirements_txt.split("pymupdfb")[0]
|
||||
with open(Path(BUILD_CONTEXT) / REQUIREMENTS_TXT, "w") as f:
|
||||
f.write(req_txt_pymupdfb_stripped)
|
||||
|
||||
def __exit__(self, exc_type, exc_value, exc_tb):
|
||||
print("Leaving the context...")
|
||||
os.remove(Path(BUILD_CONTEXT) / REQUIREMENTS_TXT)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
|
|||
|
|
@ -1,94 +0,0 @@
|
|||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import logging
|
||||
import pathlib
|
||||
import subprocess
|
||||
import sys
|
||||
import tarfile
|
||||
import urllib.request
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
TESSDATA_RELEASES_URL = (
|
||||
"https://api.github.com/repos/tesseract-ocr/tessdata_fast/releases/latest"
|
||||
)
|
||||
TESSDATA_ARCHIVE_URL = "https://github.com/tesseract-ocr/tessdata_fast/archive/{tessdata_version}/tessdata_fast-{tessdata_version}.tar.gz"
|
||||
TESSDATA_CHECKSUM = "d0e3bb6f3b4e75748680524a1d116f2bfb145618f8ceed55b279d15098a530f9"
|
||||
|
||||
|
||||
def git_root():
|
||||
"""Get the root directory of the Git repo."""
|
||||
# FIXME: Use a Git Python binding for this.
|
||||
# FIXME: Make this work if called outside the repo.
|
||||
cmd = ["git", "rev-parse", "--show-toplevel"]
|
||||
path = (
|
||||
subprocess.run(cmd, check=True, stdout=subprocess.PIPE)
|
||||
.stdout.decode()
|
||||
.strip("\n")
|
||||
)
|
||||
return pathlib.Path(path)
|
||||
|
||||
|
||||
def main():
|
||||
logging.basicConfig(
|
||||
level=logging.DEBUG,
|
||||
format="%(asctime)s - %(levelname)s - %(message)s",
|
||||
datefmt="%Y-%m-%d %H:%M:%S",
|
||||
)
|
||||
|
||||
share_dir = git_root() / "share"
|
||||
tessdata_dir = share_dir / "tessdata"
|
||||
|
||||
# Get the list of OCR languages that Dangerzone supports.
|
||||
with open(share_dir / "ocr-languages.json") as f:
|
||||
langs_short = sorted(json.loads(f.read()).values())
|
||||
|
||||
# Check if these languages have already been downloaded.
|
||||
if tessdata_dir.exists():
|
||||
expected_files = {f"{lang}.traineddata" for lang in langs_short}
|
||||
files = {f.name for f in tessdata_dir.iterdir()}
|
||||
if files == expected_files:
|
||||
logger.info("Skipping tessdata download, language data already exists")
|
||||
return
|
||||
elif not files:
|
||||
logger.info("Tesseract dir is empty, proceeding to download language data")
|
||||
else:
|
||||
logger.info(f"Found {tessdata_dir} but contents do not match")
|
||||
return 1
|
||||
|
||||
# Get latest release of Tesseract data.
|
||||
logger.info("Getting latest tessdata release")
|
||||
with urllib.request.urlopen(TESSDATA_RELEASES_URL) as f:
|
||||
resp = f.read()
|
||||
releases = json.loads(resp)
|
||||
tag = releases["tag_name"]
|
||||
|
||||
# Get latest release of Tesseract data.
|
||||
logger.info(f"Downloading tessdata release {tag}")
|
||||
archive_url = TESSDATA_ARCHIVE_URL.format(tessdata_version=tag)
|
||||
with urllib.request.urlopen(archive_url) as f:
|
||||
archive = f.read()
|
||||
digest = hashlib.sha256(archive).hexdigest()
|
||||
if digest != TESSDATA_CHECKSUM:
|
||||
raise RuntimeError(f"Checksum mismatch {digest} != {TESSDATA_CHECKSUM}")
|
||||
|
||||
# Extract the languages models from the tessdata archive.
|
||||
logger.info(f"Extracting tessdata archive into {tessdata_dir}")
|
||||
with tarfile.open(fileobj=io.BytesIO(archive)) as t:
|
||||
for lang in langs_short:
|
||||
member = f"tessdata_fast-{tag}/{lang}.traineddata"
|
||||
logger.info(f"Extracting {member}")
|
||||
# NOTE: We want `filter="data"` because it ignores ownership info, as
|
||||
# recorded in the tarfile. This filter will become the default in Python
|
||||
# 3.14. See:
|
||||
#
|
||||
# https://docs.python.org/3/library/tarfile.html#tarfile-extraction-filter
|
||||
t.extract(member=member, path=share_dir, filter="data")
|
||||
|
||||
tessdata_dl_dir = share_dir / f"tessdata_fast-{tag}"
|
||||
tessdata_dl_dir.rename(tessdata_dir)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
|
|
@ -2,17 +2,19 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
import argparse
|
||||
import inspect
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# .absolute() is needed for python<=3.8, for which
|
||||
# __file__ returns an absolute path.
|
||||
root = Path(__file__).parent.parent.parent.absolute()
|
||||
root = os.path.dirname(
|
||||
os.path.dirname(
|
||||
os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
|
||||
)
|
||||
)
|
||||
|
||||
with open(root / "share" / "version.txt") as f:
|
||||
with open(os.path.join(root, "share", "version.txt")) as f:
|
||||
version = f.read().strip()
|
||||
|
||||
|
||||
|
|
@ -37,8 +39,8 @@ def main():
|
|||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
dist_path = root / "dist"
|
||||
deb_dist_path = root / "deb_dist"
|
||||
dist_path = os.path.join(root, "dist")
|
||||
deb_dist_path = os.path.join(root, "deb_dist")
|
||||
|
||||
print("* Deleting old dist and deb_dist")
|
||||
if os.path.exists(dist_path):
|
||||
|
|
@ -47,27 +49,31 @@ def main():
|
|||
shutil.rmtree(deb_dist_path)
|
||||
|
||||
print("* Building DEB package")
|
||||
# NOTE: This command first builds the Debian source package, and then creates the
|
||||
# final DEB package. We could simply call `bdist_deb`, which performs `sdist_dsc`
|
||||
# implicitly, but we wouldn't be able to pass the Debian version argument. Because
|
||||
# we do this in a single invocation though, there's no performance cost.
|
||||
if args.distro is None:
|
||||
deb_ver_args = ()
|
||||
deb_ver = "1"
|
||||
else:
|
||||
deb_ver_args = ("--debian-version", args.distro)
|
||||
deb_ver = args.distro
|
||||
|
||||
run(
|
||||
[
|
||||
"dpkg-buildpackage",
|
||||
"python3",
|
||||
"setup.py",
|
||||
"--command-packages=stdeb.command",
|
||||
"sdist_dsc",
|
||||
*deb_ver_args,
|
||||
"bdist_deb",
|
||||
]
|
||||
)
|
||||
|
||||
os.makedirs(deb_dist_path, exist_ok=True)
|
||||
print("")
|
||||
print("* To install run:")
|
||||
|
||||
# dpkg-buildpackage produces a .deb file in the parent folder
|
||||
# that needs to be copied to the `deb_dist` folder manually
|
||||
src = root.parent / f"dangerzone_{version}_amd64.deb"
|
||||
destination = root / "deb_dist" / f"dangerzone_{version}-{deb_ver}_amd64.deb"
|
||||
shutil.move(src, destination)
|
||||
print(f"sudo dpkg -i {destination}")
|
||||
print(f"sudo dpkg -i deb_dist/dangerzone_{version}-{deb_ver}_all.deb")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
|
|||
|
|
@ -1,12 +1,14 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
import argparse
|
||||
import inspect
|
||||
import os
|
||||
import pathlib
|
||||
import shutil
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
|
||||
root = Path(__file__).parent.parent.parent
|
||||
root = pathlib.Path(__file__).parent.parent.parent
|
||||
|
||||
with open(os.path.join(root, "share", "version.txt")) as f:
|
||||
version = f.read().strip()
|
||||
|
|
@ -21,27 +23,26 @@ def remove_contents(d):
|
|||
shutil.rmtree(p)
|
||||
|
||||
|
||||
def build(build_dir, qubes=False):
|
||||
def build(qubes=False):
|
||||
"""Build an RPM package in a temporary directory.
|
||||
|
||||
The build process is the following:
|
||||
|
||||
1. Clean up any stale data from previous runs under ./dist. Note that this directory
|
||||
is used by `poetry build` and `rpmbuild`.
|
||||
2. Create the necessary RPM project structure under the specified build directory
|
||||
(default: ~/rpmbuild), and use symlinks to point to ./dist, so that we don't need
|
||||
to move files explicitly.
|
||||
2. Create the necessary RPM project structure under ./install/linux/rpm-build, and
|
||||
use symlinks to point to ./dist, so that we don't need to move files explicitly.
|
||||
3. Create a Python source distribution using `poetry build`. If we are building a
|
||||
Qubes package and there is a container image under `share/`, stash it temporarily
|
||||
under a different directory.
|
||||
4. Build both binary and source RPMs using rpmbuild. Optionally, pass to the SPEC
|
||||
`_qubes` flag, that denotes we want to build a package for Qubes.
|
||||
"""
|
||||
build_dir = root / "install" / "linux" / "rpm-build"
|
||||
dist_path = root / "dist"
|
||||
specfile_name = "dangerzone.spec"
|
||||
specfile_path = root / "install" / "linux" / specfile_name
|
||||
sdist_name = f"dangerzone-{version}.tar.gz"
|
||||
sdist_path = dist_path / sdist_name
|
||||
|
||||
print("* Deleting old dist")
|
||||
if os.path.exists(dist_path):
|
||||
|
|
@ -64,28 +65,17 @@ def build(build_dir, qubes=False):
|
|||
os.symlink(dist_path, srpm_dir)
|
||||
|
||||
print("* Creating a Python sdist")
|
||||
tessdata = root / "share" / "tessdata"
|
||||
tessdata_bak = root / "tessdata.bak"
|
||||
container_tar = root / "share" / "container.tar"
|
||||
container_tar_bak = root / "container.tar.bak"
|
||||
|
||||
if tessdata.exists():
|
||||
tessdata.rename(tessdata_bak)
|
||||
stash_container = qubes and container_tar.exists()
|
||||
if stash_container and container_tar.exists():
|
||||
container_tar.rename(container_tar_bak)
|
||||
container_tar_gz = root / "share" / "container.tar.gz"
|
||||
container_tar_gz_bak = root / "container.tar.gz.bak"
|
||||
stash_container = qubes and container_tar_gz.exists()
|
||||
if stash_container:
|
||||
container_tar_gz.rename(container_tar_gz_bak)
|
||||
try:
|
||||
subprocess.run(["poetry", "build", "-f", "sdist"], cwd=root, check=True)
|
||||
# Copy and unlink the Dangerzone sdist, instead of just renaming it. If the
|
||||
# build directory is outside the filesystem boundary (e.g., due to a container
|
||||
# mount), then a simple rename will not work.
|
||||
shutil.copy2(sdist_path, build_dir / "SOURCES" / sdist_name)
|
||||
sdist_path.unlink()
|
||||
os.rename(dist_path / sdist_name, build_dir / "SOURCES" / sdist_name)
|
||||
finally:
|
||||
if tessdata_bak.exists():
|
||||
tessdata_bak.rename(tessdata)
|
||||
if stash_container and container_tar_bak.exists():
|
||||
container_tar_bak.rename(container_tar)
|
||||
if stash_container:
|
||||
container_tar_gz_bak.rename(container_tar_gz)
|
||||
|
||||
print("* Building RPM package")
|
||||
cmd = [
|
||||
|
|
@ -103,7 +93,7 @@ def build(build_dir, qubes=False):
|
|||
if qubes:
|
||||
cmd += [
|
||||
"--define",
|
||||
"_qubes 1",
|
||||
f"_qubes 1",
|
||||
]
|
||||
subprocess.run(cmd, check=True)
|
||||
|
||||
|
|
@ -117,14 +107,9 @@ def main():
|
|||
parser.add_argument(
|
||||
"--qubes", action="store_true", help="Build RPM package for a Qubes OS system"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--build-dir",
|
||||
default=Path.home() / "rpmbuild",
|
||||
help="Working directory for rpmbuild command",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
build(args.build_dir, args.qubes)
|
||||
build(args.qubes)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@
|
|||
#
|
||||
# * Qubes packages include some extra files under /etc/qubes-rpc, whereas
|
||||
# regular RPM packages include the container image under
|
||||
# /usr/share/container.tar
|
||||
# /usr/share/container.tar.gz
|
||||
# * Qubes packages have some extra dependencies.
|
||||
# 3. It is best to consume this SPEC file using the `install/linux/build-rpm.py`
|
||||
# script, which handles the necessary scaffolding for building the package.
|
||||
|
|
@ -32,7 +32,7 @@ Name: dangerzone-qubes
|
|||
Name: dangerzone
|
||||
%endif
|
||||
|
||||
Version: 0.9.0
|
||||
Version: 0.6.1
|
||||
Release: 1%{?dist}
|
||||
Summary: Take potentially dangerous PDFs, office documents, or images and convert them to safe PDFs
|
||||
|
||||
|
|
@ -72,12 +72,13 @@ BuildRequires: python3-devel
|
|||
%if 0%{?_qubes}
|
||||
# Qubes-only requirements (server-side)
|
||||
Requires: python3-magic
|
||||
Requires: python3-PyMuPDF
|
||||
Requires: libreoffice
|
||||
%else
|
||||
# Container-only requirements
|
||||
Requires: podman
|
||||
%endif
|
||||
|
||||
# Qubes-only requirements (client-side)
|
||||
Requires: GraphicsMagick
|
||||
Requires: ghostscript
|
||||
Requires: poppler-utils
|
||||
Requires: tesseract
|
||||
# Explicitly require every tesseract model:
|
||||
# See: https://github.com/freedomofpress/dangerzone/issues/431
|
||||
Requires: tesseract-langpack-afr
|
||||
|
|
@ -203,6 +204,10 @@ Requires: tesseract-langpack-uzb_cyrl
|
|||
Requires: tesseract-langpack-vie
|
||||
Requires: tesseract-langpack-yid
|
||||
Requires: tesseract-langpack-yor
|
||||
%else
|
||||
# Container-only requirements
|
||||
Requires: podman
|
||||
%endif
|
||||
|
||||
%description
|
||||
Dangerzone is an open source desktop application that takes potentially
|
||||
|
|
@ -216,6 +221,16 @@ convert the documents within a secure sandbox.
|
|||
%prep
|
||||
%autosetup -p1 -n dangerzone-%{version}
|
||||
|
||||
# XXX: Replace the PySide6 dependency in the pyproject.toml file with PySide2,
|
||||
# since the former did not exist until Fedora 39, where we packaged PySide6 [1].
|
||||
# Once Fedora 38 is no longer supported, we should remove this.
|
||||
#
|
||||
# [1]: https://github.com/freedomofpress/dangerzone/issues/606
|
||||
%if 0%{?fedora} == 38
|
||||
sed -i 's/^PySide6.*$/PySide2 = "*"/' pyproject.toml
|
||||
%endif
|
||||
|
||||
|
||||
%generate_buildrequires
|
||||
%pyproject_buildrequires -R
|
||||
|
||||
|
|
@ -243,17 +258,6 @@ install -m 755 -d %{buildroot}/etc/qubes-rpc
|
|||
install -m 755 qubes/* %{buildroot}/etc/qubes-rpc
|
||||
%endif
|
||||
|
||||
%check
|
||||
# Detect if the filesystem has been affecting our file permissions.
|
||||
bad_files=$(find %{buildroot} -perm 0600)
|
||||
if [ -n "${bad_files}" ]; then
|
||||
echo "Error while building the Dangerzone RPM. Detected the following files with wrong permissions (600):"
|
||||
echo ${bad_files}
|
||||
echo ""
|
||||
echo "For more info about this error, see https://github.com/freedomofpress/dangerzone/issues/727"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
%files -f %{pyproject_files}
|
||||
/usr/bin/dangerzone
|
||||
/usr/bin/dangerzone-cli
|
||||
|
|
|
|||
|
|
@ -1,59 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
DZ_VENDOR_DIR = Path("./dangerzone/vendor")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
"--dest",
|
||||
default=DZ_VENDOR_DIR,
|
||||
help="The destination directory for the vendored packages (default: ./dangerzone/vendor)",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.DEBUG,
|
||||
format="%(asctime)s - %(levelname)s - %(message)s",
|
||||
datefmt="%Y-%m-%d %H:%M:%S",
|
||||
)
|
||||
|
||||
logger.info("Getting PyMuPDF deps as requirements.txt")
|
||||
cmd = ["poetry", "export", "--only", "debian"]
|
||||
container_requirements_txt = subprocess.check_output(cmd)
|
||||
|
||||
logger.info(f"Vendoring PyMuPDF under '{args.dest}'")
|
||||
# We prefer to call the CLI version of `pip`, instead of importing it directly, as
|
||||
# instructed here:
|
||||
# https://pip.pypa.io/en/latest/user_guide/#using-pip-from-your-program
|
||||
cmd = [
|
||||
sys.executable,
|
||||
"-m",
|
||||
"pip",
|
||||
"install",
|
||||
"--no-cache-dir",
|
||||
"--no-compile",
|
||||
"--target",
|
||||
args.dest,
|
||||
"--requirement",
|
||||
"/proc/self/fd/0", # XXX: pip does not read requirements.txt from stdin
|
||||
]
|
||||
subprocess.run(cmd, check=True, input=container_requirements_txt)
|
||||
|
||||
if not os.listdir(args.dest):
|
||||
logger.error(f"Failed to vendor PyMuPDF under '{args.dest}'")
|
||||
|
||||
logger.info(f"Successfully vendored PyMuPDF under '{args.dest}'")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
40
install/linux/install-podman-ubuntu-focal.sh
Executable file
40
install/linux/install-podman-ubuntu-focal.sh
Executable file
|
|
@ -0,0 +1,40 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Development script for installing Podman on Ubuntu Focal. Mainly to be used as
|
||||
# part of our CI pipelines, where we may install Podman on environments that
|
||||
# don't have sudo.
|
||||
|
||||
set -e
|
||||
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
SUDO=sudo
|
||||
else
|
||||
SUDO=
|
||||
fi
|
||||
|
||||
provide() {
|
||||
$SUDO apt-get update
|
||||
$SUDO apt-get install curl wget gnupg2 -y
|
||||
source /etc/os-release
|
||||
$SUDO sh -c "echo 'deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_${VERSION_ID}/ /' \
|
||||
> /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list"
|
||||
wget -nv https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable/xUbuntu_${VERSION_ID}/Release.key -O- \
|
||||
| $SUDO apt-key add -
|
||||
$SUDO apt-get update -qq -y
|
||||
}
|
||||
|
||||
install() {
|
||||
$SUDO apt-get -qq --yes install podman
|
||||
podman --version
|
||||
}
|
||||
|
||||
if [[ "$1" == "--repo-only" ]]; then
|
||||
provide
|
||||
elif [[ "$1" == "" ]]; then
|
||||
provide
|
||||
install
|
||||
else
|
||||
echo "Unexpected argument: $1"
|
||||
echo "Usage: $0 [--repo-only]"
|
||||
exit 1
|
||||
fi
|
||||
|
|
@ -10,5 +10,11 @@
|
|||
<true/>
|
||||
<key>com.apple.security.network.client</key>
|
||||
<true/>
|
||||
<key>com.apple.security.network.server</key>
|
||||
<true/>
|
||||
<key>com.apple.security.hypervisor</key>
|
||||
<true/>
|
||||
<key>com.apple.security.cs.allow-unsigned-executable-memory</key>
|
||||
<true/>
|
||||
</dict>
|
||||
</plist>
|
||||
</plist>
|
||||
|
|
@ -2,38 +2,26 @@ REM delete old dist and build files
|
|||
rmdir /s /q dist
|
||||
rmdir /s /q build
|
||||
|
||||
REM build the gui and cli exe
|
||||
REM build the exe
|
||||
python .\setup-windows.py build
|
||||
|
||||
REM code sign dangerzone.exe
|
||||
signtool.exe sign /v /d "Dangerzone" /a /n "Freedom of the Press Foundation" /fd sha256 /t http://time.certum.pl/ build\exe.win-amd64-3.12\dangerzone.exe
|
||||
signtool.exe sign /v /d "Dangerzone" /a /n "Freedom of the Press Foundation" /fd sha1 /t http://time.certum.pl/ build\exe.win-amd64-3.11\dangerzone.exe
|
||||
signtool.exe sign /v /d "Dangerzone" /a /n "Freedom of the Press Foundation" /fd sha1 /t http://time.certum.pl/ build\exe.win-amd64-3.11\dangerzone-cli.exe
|
||||
|
||||
REM verify the signature of dangerzone.exe
|
||||
signtool.exe verify /pa build\exe.win-amd64-3.12\dangerzone.exe
|
||||
|
||||
REM code sign dangerzone-cli.exe
|
||||
signtool.exe sign /v /d "Dangerzone" /a /n "Freedom of the Press Foundation" /fd sha256 /t http://time.certum.pl/ build\exe.win-amd64-3.12\dangerzone-cli.exe
|
||||
|
||||
REM verify the signature of dangerzone-cli.exe
|
||||
signtool.exe verify /pa build\exe.win-amd64-3.12\dangerzone-cli.exe
|
||||
|
||||
REM build the wxs file
|
||||
python install\windows\build-wxs.py
|
||||
REM build the wix file
|
||||
python install\windows\build-wxs.py > build\Dangerzone.wxs
|
||||
|
||||
REM build the msi package
|
||||
cd build
|
||||
wix build -arch x64 -ext WixToolset.UI.wixext .\Dangerzone.wxs -out Dangerzone.msi
|
||||
candle.exe Dangerzone.wxs
|
||||
light.exe -ext WixUIExtension Dangerzone.wixobj
|
||||
|
||||
REM validate Dangerzone.msi
|
||||
wix msi validate Dangerzone.msi
|
||||
REM code sign dangerzone.msi
|
||||
insignia.exe -im Dangerzone.msi
|
||||
signtool.exe sign /v /d "Dangerzone" /a /n "Freedom of the Press Foundation" /fd sha1 /t http://time.certum.pl/ Dangerzone.msi
|
||||
|
||||
REM code sign Dangerzone.msi
|
||||
signtool.exe sign /v /d "Dangerzone" /a /n "Freedom of the Press Foundation" /fd sha256 /t http://time.certum.pl/ Dangerzone.msi
|
||||
|
||||
REM verify the signature of Dangerzone.msi
|
||||
signtool.exe verify /pa Dangerzone.msi
|
||||
|
||||
REM move Dangerzone.msi to dist
|
||||
REM moving Dangerzone.msi to dist
|
||||
cd ..
|
||||
mkdir dist
|
||||
move build\Dangerzone.msi dist
|
||||
|
|
|
|||
|
|
@ -4,75 +4,114 @@ import uuid
|
|||
import xml.etree.ElementTree as ET
|
||||
|
||||
|
||||
def build_data(base_path, path_prefix, dir_id, dir_name):
|
||||
def build_data(dirname, dir_prefix, id_, name):
|
||||
data = {
|
||||
"directory_name": dir_name,
|
||||
"directory_id": dir_id,
|
||||
"id": id_,
|
||||
"name": name,
|
||||
"files": [],
|
||||
"dirs": [],
|
||||
}
|
||||
|
||||
if dir_id == "INSTALLFOLDER":
|
||||
data["component_id"] = "ApplicationFiles"
|
||||
else:
|
||||
data["component_id"] = "Component" + dir_id
|
||||
data["component_guid"] = str(uuid.uuid4()).upper()
|
||||
|
||||
for entry in os.listdir(base_path):
|
||||
entry_path = os.path.join(base_path, entry)
|
||||
if os.path.isfile(entry_path):
|
||||
data["files"].append(os.path.join(path_prefix, entry))
|
||||
elif os.path.isdir(entry_path):
|
||||
if dir_id == "INSTALLFOLDER":
|
||||
next_dir_prefix = "Folder"
|
||||
for basename in os.listdir(dirname):
|
||||
filename = os.path.join(dirname, basename)
|
||||
if os.path.isfile(filename):
|
||||
data["files"].append(os.path.join(dir_prefix, basename))
|
||||
elif os.path.isdir(filename):
|
||||
if id_ == "INSTALLDIR":
|
||||
id_prefix = "Folder"
|
||||
else:
|
||||
next_dir_prefix = dir_id
|
||||
id_prefix = id_
|
||||
|
||||
# Skip lib/PySide6/examples folder due to ilegal file names
|
||||
if "\\build\\exe.win-amd64-3.12\\lib\\PySide6\\examples" in base_path:
|
||||
if "\\build\\exe.win-amd64-3.11\\lib\\PySide6\\examples" in dirname:
|
||||
continue
|
||||
|
||||
# Skip lib/PySide6/qml/QtQuick folder due to ilegal file names
|
||||
# XXX Since we're not using Qml it should be no problem
|
||||
if "\\build\\exe.win-amd64-3.12\\lib\\PySide6\\qml\\QtQuick" in base_path:
|
||||
if "\\build\\exe.win-amd64-3.11\\lib\\PySide6\\qml\\QtQuick" in dirname:
|
||||
continue
|
||||
|
||||
next_dir_id = next_dir_prefix + entry.capitalize().replace("-", "_")
|
||||
subdata = build_data(
|
||||
os.path.join(base_path, entry),
|
||||
os.path.join(path_prefix, entry),
|
||||
next_dir_id,
|
||||
entry,
|
||||
id_value = f"{id_prefix}{basename.capitalize().replace('-', '_')}"
|
||||
data["dirs"].append(
|
||||
build_data(
|
||||
os.path.join(dirname, basename),
|
||||
os.path.join(dir_prefix, basename),
|
||||
id_value,
|
||||
basename,
|
||||
)
|
||||
)
|
||||
|
||||
# Add the subdirectory only if it contains files or subdirectories
|
||||
if subdata["files"] or subdata["dirs"]:
|
||||
data["dirs"].append(subdata)
|
||||
if len(data["files"]) > 0:
|
||||
if id_ == "INSTALLDIR":
|
||||
data["component_id"] = "ApplicationFiles"
|
||||
else:
|
||||
data["component_id"] = "FolderComponent" + id_[len("Folder") :]
|
||||
data["component_guid"] = str(uuid.uuid4())
|
||||
|
||||
return data
|
||||
|
||||
|
||||
def build_directory_xml(root, data):
|
||||
def build_dir_xml(root, data):
|
||||
attrs = {}
|
||||
attrs["Id"] = data["directory_id"]
|
||||
attrs["Name"] = data["directory_name"]
|
||||
directory_el = ET.SubElement(root, "Directory", attrs)
|
||||
if "id" in data:
|
||||
attrs["Id"] = data["id"]
|
||||
if "name" in data:
|
||||
attrs["Name"] = data["name"]
|
||||
el = ET.SubElement(root, "Directory", attrs)
|
||||
for subdata in data["dirs"]:
|
||||
build_directory_xml(directory_el, subdata)
|
||||
build_dir_xml(el, subdata)
|
||||
|
||||
# If this is the ProgramMenuFolder, add the menu component
|
||||
if "id" in data and data["id"] == "ProgramMenuFolder":
|
||||
component_el = ET.SubElement(
|
||||
el,
|
||||
"Component",
|
||||
Id="ApplicationShortcuts",
|
||||
Guid="539e7de8-a124-4c09-aa55-0dd516aad7bc",
|
||||
)
|
||||
ET.SubElement(
|
||||
component_el,
|
||||
"Shortcut",
|
||||
Id="ApplicationShortcut1",
|
||||
Name="Dangerzone",
|
||||
Description="Dangerzone",
|
||||
Target="[INSTALLDIR]dangerzone.exe",
|
||||
WorkingDirectory="INSTALLDIR",
|
||||
)
|
||||
ET.SubElement(
|
||||
component_el,
|
||||
"RegistryValue",
|
||||
Root="HKCU",
|
||||
Key="Software\Freedom of the Press Foundation\Dangerzone",
|
||||
Name="installed",
|
||||
Type="integer",
|
||||
Value="1",
|
||||
KeyPath="yes",
|
||||
)
|
||||
|
||||
|
||||
def build_components_xml(root, data):
|
||||
component_el = ET.SubElement(
|
||||
root,
|
||||
"Component",
|
||||
Id=data["component_id"],
|
||||
Guid=data["component_guid"],
|
||||
Directory=data["directory_id"],
|
||||
)
|
||||
for filename in data["files"]:
|
||||
ET.SubElement(component_el, "File", Source=filename)
|
||||
component_ids = []
|
||||
if "component_id" in data:
|
||||
component_ids.append(data["component_id"])
|
||||
|
||||
for subdata in data["dirs"]:
|
||||
build_components_xml(root, subdata)
|
||||
if "component_guid" in subdata:
|
||||
dir_ref_el = ET.SubElement(root, "DirectoryRef", Id=subdata["id"])
|
||||
component_el = ET.SubElement(
|
||||
dir_ref_el,
|
||||
"Component",
|
||||
Id=subdata["component_id"],
|
||||
Guid=subdata["component_guid"],
|
||||
)
|
||||
for filename in subdata["files"]:
|
||||
file_el = ET.SubElement(
|
||||
component_el, "File", Source=filename, Id="file_" + uuid.uuid4().hex
|
||||
)
|
||||
|
||||
component_ids += build_components_xml(root, subdata)
|
||||
|
||||
return component_ids
|
||||
|
||||
|
||||
def main():
|
||||
|
|
@ -86,196 +125,114 @@ def main():
|
|||
# -rc markers.
|
||||
version = f.read().strip().split("-")[0]
|
||||
|
||||
build_dir = os.path.join(
|
||||
dist_dir = os.path.join(
|
||||
os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))),
|
||||
"build",
|
||||
"exe.win-amd64-3.11",
|
||||
)
|
||||
|
||||
cx_freeze_dir = "exe.win-amd64-3.12"
|
||||
|
||||
dist_dir = os.path.join(build_dir, cx_freeze_dir)
|
||||
|
||||
if not os.path.exists(dist_dir):
|
||||
print("You must build the dangerzone binary before running this")
|
||||
return
|
||||
|
||||
# Prepare data for WiX file harvesting from the output of cx_Freeze
|
||||
data = build_data(
|
||||
dist_dir,
|
||||
cx_freeze_dir,
|
||||
"INSTALLFOLDER",
|
||||
"Dangerzone",
|
||||
data = {
|
||||
"id": "TARGETDIR",
|
||||
"name": "SourceDir",
|
||||
"dirs": [
|
||||
{
|
||||
"id": "ProgramFilesFolder",
|
||||
"dirs": [],
|
||||
},
|
||||
{
|
||||
"id": "ProgramMenuFolder",
|
||||
"dirs": [],
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
data["dirs"][0]["dirs"].append(
|
||||
build_data(
|
||||
dist_dir,
|
||||
"exe.win-amd64-3.11",
|
||||
"INSTALLDIR",
|
||||
"Dangerzone",
|
||||
)
|
||||
)
|
||||
|
||||
# Add the Wix root element
|
||||
wix_el = ET.Element(
|
||||
"Wix",
|
||||
{
|
||||
"xmlns": "http://wixtoolset.org/schemas/v4/wxs",
|
||||
"xmlns:ui": "http://wixtoolset.org/schemas/v4/wxs/ui",
|
||||
},
|
||||
)
|
||||
|
||||
# Add the Package element
|
||||
package_el = ET.SubElement(
|
||||
wix_el,
|
||||
"Package",
|
||||
root_el = ET.Element("Wix", xmlns="http://schemas.microsoft.com/wix/2006/wi")
|
||||
product_el = ET.SubElement(
|
||||
root_el,
|
||||
"Product",
|
||||
Name="Dangerzone",
|
||||
Manufacturer="Freedom of the Press Foundation",
|
||||
UpgradeCode="12B9695C-965B-4BE0-BC33-21274E809576",
|
||||
Id="*",
|
||||
UpgradeCode="$(var.ProductUpgradeCode)",
|
||||
Language="1033",
|
||||
Compressed="yes",
|
||||
Codepage="1252",
|
||||
Version=version,
|
||||
Version="$(var.ProductVersion)",
|
||||
)
|
||||
ET.SubElement(
|
||||
package_el,
|
||||
"SummaryInformation",
|
||||
product_el,
|
||||
"Package",
|
||||
Id="*",
|
||||
Keywords="Installer",
|
||||
Description="Dangerzone " + version + " Installer",
|
||||
Codepage="1252",
|
||||
Description="Dangerzone $(var.ProductVersion) Installer",
|
||||
Manufacturer="Freedom of the Press Foundation",
|
||||
InstallerVersion="100",
|
||||
Languages="1033",
|
||||
Compressed="yes",
|
||||
SummaryCodepage="1252",
|
||||
)
|
||||
ET.SubElement(package_el, "MediaTemplate", EmbedCab="yes")
|
||||
ET.SubElement(product_el, "Media", Id="1", Cabinet="product.cab", EmbedCab="yes")
|
||||
ET.SubElement(
|
||||
package_el, "Icon", Id="ProductIcon", SourceFile="..\\share\\dangerzone.ico"
|
||||
product_el, "Icon", Id="ProductIcon", SourceFile="..\\share\\dangerzone.ico"
|
||||
)
|
||||
ET.SubElement(package_el, "Property", Id="ARPPRODUCTICON", Value="ProductIcon")
|
||||
ET.SubElement(product_el, "Property", Id="ARPPRODUCTICON", Value="ProductIcon")
|
||||
ET.SubElement(
|
||||
package_el,
|
||||
product_el,
|
||||
"Property",
|
||||
Id="ARPHELPLINK",
|
||||
Value="https://dangerzone.rocks",
|
||||
)
|
||||
ET.SubElement(
|
||||
package_el,
|
||||
product_el,
|
||||
"Property",
|
||||
Id="ARPURLINFOABOUT",
|
||||
Value="https://freedom.press",
|
||||
)
|
||||
ET.SubElement(product_el, "UIRef", Id="WixUI_Minimal")
|
||||
ET.SubElement(product_el, "UIRef", Id="WixUI_ErrorProgressText")
|
||||
ET.SubElement(
|
||||
package_el, "ui:WixUI", Id="WixUI_InstallDir", InstallDirectory="INSTALLFOLDER"
|
||||
)
|
||||
ET.SubElement(package_el, "UIRef", Id="WixUI_ErrorProgressText")
|
||||
ET.SubElement(
|
||||
package_el,
|
||||
product_el,
|
||||
"WixVariable",
|
||||
Id="WixUILicenseRtf",
|
||||
Value="..\\install\\windows\\license.rtf",
|
||||
)
|
||||
ET.SubElement(
|
||||
package_el,
|
||||
product_el,
|
||||
"WixVariable",
|
||||
Id="WixUIDialogBmp",
|
||||
Value="..\\install\\windows\\dialog.bmp",
|
||||
)
|
||||
ET.SubElement(
|
||||
package_el,
|
||||
product_el,
|
||||
"MajorUpgrade",
|
||||
AllowSameVersionUpgrades="yes",
|
||||
DowngradeErrorMessage="A newer version of [ProductName] is already installed. If you are sure you want to downgrade, remove the existing installation via Programs and Features.",
|
||||
)
|
||||
|
||||
# Workaround for an issue after upgrading from WiX Toolset v3 to v5 where the previous
|
||||
# version of Dangerzone is not uninstalled during the upgrade by checking if the older installation
|
||||
# exists in "C:\Program Files (x86)\Dangerzone".
|
||||
#
|
||||
# Also handle a special case for Dangerzone 0.8.0 which allows choosing the install location
|
||||
# during install by checking if the registry key for it exists.
|
||||
#
|
||||
# Note that this seems to allow installing Dangerzone 0.8.0 after installing Dangerzone from this branch.
|
||||
# In this case the installer errors until Dangerzone 0.8.0 is uninstalled again
|
||||
#
|
||||
# TODO: Revert this once we are reasonably certain there aren't too many affected Dangerzone installations.
|
||||
find_old_el = ET.SubElement(package_el, "Property", Id="OLDDANGERZONEFOUND")
|
||||
directory_search_el = ET.SubElement(
|
||||
find_old_el,
|
||||
"DirectorySearch",
|
||||
Id="dangerzone_install_folder",
|
||||
Path="C:\\Program Files (x86)\\Dangerzone",
|
||||
)
|
||||
ET.SubElement(directory_search_el, "FileSearch", Name="dangerzone.exe")
|
||||
registry_search_el = ET.SubElement(package_el, "Property", Id="DANGERZONE08FOUND")
|
||||
ET.SubElement(
|
||||
registry_search_el,
|
||||
"RegistrySearch",
|
||||
Root="HKLM",
|
||||
Key="SOFTWARE\\WOW6432Node\\Microsoft\\Windows\\CurrentVersion\\Uninstall\\{03C2D2B2-9955-4AED-831F-DA4E67FC0FDB}",
|
||||
Name="DisplayName",
|
||||
Type="raw",
|
||||
)
|
||||
ET.SubElement(
|
||||
registry_search_el,
|
||||
"RegistrySearch",
|
||||
Root="HKLM",
|
||||
Key="SOFTWARE\\WOW6432Node\\Microsoft\\Windows\\CurrentVersion\\Uninstall\\{8AAC0808-3556-4164-9D15-6EC1FB673AB2}",
|
||||
Name="DisplayName",
|
||||
Type="raw",
|
||||
)
|
||||
ET.SubElement(
|
||||
package_el,
|
||||
"Launch",
|
||||
Condition="NOT OLDDANGERZONEFOUND AND NOT DANGERZONE08FOUND",
|
||||
Message='A previous version of [ProductName] is already installed. Please uninstall it from "Apps & Features" before proceeding with the installation.',
|
||||
)
|
||||
build_dir_xml(product_el, data)
|
||||
component_ids = build_components_xml(product_el, data)
|
||||
|
||||
# Add the ProgramMenuFolder StandardDirectory
|
||||
programmenufolder_el = ET.SubElement(
|
||||
package_el,
|
||||
"StandardDirectory",
|
||||
Id="ProgramMenuFolder",
|
||||
)
|
||||
# Add a shortcut for Dangerzone in the Start menu
|
||||
shortcut_el = ET.SubElement(
|
||||
programmenufolder_el,
|
||||
"Component",
|
||||
Id="ApplicationShortcuts",
|
||||
Guid="539E7DE8-A124-4C09-AA55-0DD516AAD7BC",
|
||||
)
|
||||
ET.SubElement(
|
||||
shortcut_el,
|
||||
"Shortcut",
|
||||
Id="DangerzoneStartMenuShortcut",
|
||||
Name="Dangerzone",
|
||||
Description="Dangerzone",
|
||||
Target="[INSTALLFOLDER]dangerzone.exe",
|
||||
WorkingDirectory="INSTALLFOLDER",
|
||||
)
|
||||
ET.SubElement(
|
||||
shortcut_el,
|
||||
"RegistryValue",
|
||||
Root="HKCU",
|
||||
Key="Software\\Freedom of the Press Foundation\\Dangerzone",
|
||||
Name="installed",
|
||||
Type="integer",
|
||||
Value="1",
|
||||
KeyPath="yes",
|
||||
)
|
||||
|
||||
# Add the ProgramFilesFolder StandardDirectory
|
||||
programfilesfolder_el = ET.SubElement(
|
||||
package_el,
|
||||
"StandardDirectory",
|
||||
Id="ProgramFiles64Folder",
|
||||
)
|
||||
|
||||
# Create the directory structure for the installed product
|
||||
build_directory_xml(programfilesfolder_el, data)
|
||||
|
||||
# Create a component group for application components
|
||||
applicationcomponents_el = ET.SubElement(
|
||||
package_el, "ComponentGroup", Id="ApplicationComponents"
|
||||
)
|
||||
# Populate the application components group with components for the installed package
|
||||
build_components_xml(applicationcomponents_el, data)
|
||||
|
||||
# Add the Feature element
|
||||
feature_el = ET.SubElement(package_el, "Feature", Id="DefaultFeature", Level="1")
|
||||
ET.SubElement(feature_el, "ComponentGroupRef", Id="ApplicationComponents")
|
||||
feature_el = ET.SubElement(product_el, "Feature", Id="DefaultFeature", Level="1")
|
||||
for component_id in component_ids:
|
||||
ET.SubElement(feature_el, "ComponentRef", Id=component_id)
|
||||
ET.SubElement(feature_el, "ComponentRef", Id="ApplicationShortcuts")
|
||||
|
||||
ET.indent(wix_el, space=" ")
|
||||
|
||||
with open(os.path.join(build_dir, "Dangerzone.wxs"), "w") as wxs_file:
|
||||
wxs_file.write(ET.tostring(wix_el).decode())
|
||||
print('<?xml version="1.0" encoding="windows-1252"?>')
|
||||
print(f'<?define ProductVersion = "{version}"?>')
|
||||
print('<?define ProductUpgradeCode = "12b9695c-965b-4be0-bc33-21274e809576"?>')
|
||||
ET.indent(root_el)
|
||||
print(ET.tostring(root_el).decode())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
|
|||
1540
poetry.lock
generated
1540
poetry.lock
generated
File diff suppressed because it is too large
Load diff
|
|
@ -1,6 +1,6 @@
|
|||
[tool.poetry]
|
||||
name = "dangerzone"
|
||||
version = "0.9.0"
|
||||
version = "0.6.1"
|
||||
description = "Take potentially dangerous PDFs, office documents, or images and convert them to safe PDFs"
|
||||
authors = ["Freedom of the Press Foundation <info@freedom.press>", "Micah Lee <micah.lee@theintercept.com>"]
|
||||
license = "AGPL-3.0"
|
||||
|
|
@ -13,11 +13,10 @@ include = [
|
|||
]
|
||||
|
||||
[tool.poetry.dependencies]
|
||||
python = ">=3.9,<3.14"
|
||||
python = ">=3.8,<3.13"
|
||||
click = "*"
|
||||
platformdirs = "*"
|
||||
PySide6 = "^6.7.1"
|
||||
PyMuPDF = "^1.23.3" # The version in Fedora 39
|
||||
appdirs = "*"
|
||||
PySide6 = "^6.4.1"
|
||||
colorama = "*"
|
||||
pyxdg = {version = "*", platform = "linux"}
|
||||
requests = "*"
|
||||
|
|
@ -31,21 +30,17 @@ dangerzone-cli = 'dangerzone:main'
|
|||
# Dependencies required for packaging the code on various platforms.
|
||||
[tool.poetry.group.package.dependencies]
|
||||
setuptools = "*"
|
||||
cx_freeze = {version = "^7.2.5", platform = "win32"}
|
||||
cx_freeze = {version = "^6.13.1", platform = "win32"}
|
||||
pywin32 = {version = "*", platform = "win32"}
|
||||
pyinstaller = {version = "*", platform = "darwin"}
|
||||
doit = "^0.36.0"
|
||||
jinja2-cli = "^0.8.2"
|
||||
|
||||
# Dependencies required for linting the code.
|
||||
[tool.poetry.group.lint.dependencies]
|
||||
click = "*" # Install click so mypy is able to reason about it.
|
||||
black = "*"
|
||||
isort = "*"
|
||||
mypy = "*"
|
||||
ruff = "*"
|
||||
types-colorama = "*"
|
||||
types-PySide2 = "*"
|
||||
types-Markdown = "*"
|
||||
types-pygments = "*"
|
||||
types-requests = "*"
|
||||
|
||||
# Dependencies required for testing the code.
|
||||
|
|
@ -55,24 +50,18 @@ pytest-mock = "^3.10.0"
|
|||
pytest-qt = "^4.2.0"
|
||||
pytest-cov = "^5.0.0"
|
||||
strip-ansi = "*"
|
||||
pytest-subprocess = "^1.5.2"
|
||||
pytest-rerunfailures = "^14.0"
|
||||
numpy = "2.0" # bump when we remove python 3.9 support
|
||||
|
||||
[tool.poetry.group.debian.dependencies]
|
||||
pymupdf = "^1.24.11"
|
||||
[tool.poetry.group.qubes.dependencies]
|
||||
pymupdf = "^1.23.6"
|
||||
|
||||
[tool.poetry.group.dev.dependencies]
|
||||
httpx = "^0.27.2"
|
||||
[tool.poetry.group.container.dependencies]
|
||||
pymupdf = "^1.24.0"
|
||||
|
||||
[tool.doit]
|
||||
verbosity = 3
|
||||
|
||||
[tool.ruff.lint]
|
||||
select = [
|
||||
# isort
|
||||
"I",
|
||||
]
|
||||
[tool.isort]
|
||||
profile = "black"
|
||||
skip_gitignore = true
|
||||
# This is necessary due to https://github.com/PyCQA/isort/issues/1835
|
||||
follow_links = false
|
||||
|
||||
[build-system]
|
||||
requires = ["poetry-core>=1.2.0"]
|
||||
|
|
|
|||
|
|
@ -1,2 +1,2 @@
|
|||
#!/usr/bin/sh
|
||||
#!/bin/sh
|
||||
python -m dangerzone.conversion.doc_to_pixels
|
||||
|
|
|
|||
|
|
@ -1,8 +1,12 @@
|
|||
#!/usr/bin/python3
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import asyncio
|
||||
import glob
|
||||
import io
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import zipfile
|
||||
|
||||
|
||||
def say(msg):
|
||||
|
|
@ -19,7 +23,7 @@ def main():
|
|||
# Read the zipfile from stdin
|
||||
zf = sys.stdin.buffer.read(size)
|
||||
if len(zf) < size:
|
||||
say("Client closed the connection early")
|
||||
say(f"Client closed the connection early")
|
||||
return 1
|
||||
|
||||
with tempfile.NamedTemporaryFile(suffix=".zip") as t:
|
||||
|
|
@ -27,11 +31,10 @@ def main():
|
|||
t.write(zf)
|
||||
t.flush()
|
||||
|
||||
say("Importing the conversion module")
|
||||
say(f"Importing the conversion module")
|
||||
sys.path.insert(0, t.name)
|
||||
|
||||
from dangerzone.conversion.doc_to_pixels import main
|
||||
|
||||
return asyncio.run(main())
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,9 +1,12 @@
|
|||
#!/usr/bin/env python3
|
||||
import os
|
||||
|
||||
from cx_Freeze import Executable, setup
|
||||
|
||||
with open("share/version.txt") as f:
|
||||
version = f.read().strip()
|
||||
|
||||
packages = ["dangerzone", "dangerzone.gui"]
|
||||
|
||||
setup(
|
||||
name="dangerzone",
|
||||
|
|
@ -11,9 +14,10 @@ setup(
|
|||
# On Windows description will show as the app's name in the "Open With" menu. See:
|
||||
# https://github.com/freedomofpress/dangerzone/issues/283#issuecomment-1365148805
|
||||
description="Dangerzone",
|
||||
packages=packages,
|
||||
options={
|
||||
"build_exe": {
|
||||
"packages": ["dangerzone", "dangerzone.gui", "pymupdf._wxcolors"],
|
||||
"packages": packages,
|
||||
"excludes": ["test", "tkinter"],
|
||||
"include_files": [("share", "share"), ("LICENSE", "LICENSE")],
|
||||
"include_msvcr": True,
|
||||
|
|
|
|||
1
setup.py
1
setup.py
|
|
@ -1,5 +1,6 @@
|
|||
#!/usr/bin/env python3
|
||||
import os
|
||||
import sys
|
||||
|
||||
import setuptools
|
||||
|
||||
|
|
|
|||
|
|
@ -13,11 +13,6 @@ QDialog[OSColorMode="light"] QWidget {
|
|||
color: black;
|
||||
}
|
||||
|
||||
DocSelectionDropFrame{
|
||||
border: 2px dashed rgb(193, 193, 193);
|
||||
border-radius: 5px;
|
||||
margin: 5px;
|
||||
}
|
||||
/*
|
||||
* QLabel left-adjacent to a QLineEdit to give the illusion
|
||||
* that it is part of it, but just not editable
|
||||
|
|
@ -48,27 +43,3 @@ QLabel.version {
|
|||
font-size: 20px;
|
||||
padding-bottom: 5px; /* align with 'dangerzone' font */
|
||||
}
|
||||
|
||||
QTextEdit[style="traceback"] {
|
||||
font-family: Consolas, Monospace;
|
||||
font-size: 12px;
|
||||
background-color: #ffffff;
|
||||
color: #000000;
|
||||
padding: 10px;
|
||||
}
|
||||
|
||||
QLabel[style="warning"] {
|
||||
background-color: #FFF3CD;
|
||||
color: #856404;
|
||||
border: 1px solid #FFEEBA;
|
||||
border-radius: 4px;
|
||||
padding: 10px;
|
||||
margin: 10px;
|
||||
}
|
||||
|
||||
|
||||
MainWindow[OSColorMode="dark"] QLabel[style="warning"] {
|
||||
background-color: #332D00;
|
||||
color: #FFD970;
|
||||
border-color: #665A00;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1 +0,0 @@
|
|||
<svg xmlns:xlink="http://www.w3.org/1999/xlink" width="20.308" xmlns="http://www.w3.org/2000/svg" height="24" id="screenshot-629d13e7-3095-8022-8003-fd3e19dbffbc" viewBox="10023.346 5693.5 20.308 24" style="-webkit-print-color-adjust: exact;" fill="none" version="1.1"><g id="shape-629d13e7-3095-8022-8003-fd3e19dbffbc"><g class="fills" id="fills-629d13e7-3095-8022-8003-fd3e19dbffbc"><path fill="currentColor" fill-rule="evenodd" clip-rule="evenodd" rx="0" ry="0" d="M10026.115,5695.346C10025.606,5695.346,10025.192,5695.759,10025.192,5696.269L10025.192,5714.731C10025.192,5715.240,10025.606,5715.654,10026.115,5715.654L10040.885,5715.654C10041.394,5715.654,10041.808,5715.240,10041.808,5714.731L10041.808,5702.731L10035.346,5702.731C10034.836,5702.731,10034.423,5702.317,10034.423,5701.808L10034.423,5695.346L10026.115,5695.346ZZM10036.269,5696.652L10040.502,5700.885L10036.269,5700.885L10036.269,5696.652ZZM10023.346,5696.269C10023.346,5694.740,10024.586,5693.500,10026.115,5693.500L10035.346,5693.500C10035.591,5693.500,10035.826,5693.597,10035.999,5693.770L10043.384,5701.155C10043.557,5701.328,10043.654,5701.563,10043.654,5701.808L10043.654,5714.731C10043.654,5716.260,10042.414,5717.500,10040.885,5717.500L10026.115,5717.500C10024.586,5717.500,10023.346,5716.260,10023.346,5714.731L10023.346,5696.269ZZ" style="fill: rgb(153, 153, 153); fill-opacity: 1;"/></g></g></svg>
|
||||
|
Before Width: | Height: | Size: 1.3 KiB |
File diff suppressed because it is too large
Load diff
|
|
@ -1 +1 @@
|
|||
0.9.0
|
||||
0.6.1
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue