WIP: Doit

This commit is contained in:
Alex Pyrgiotis 2024-12-03 16:08:39 +02:00
parent 57973b8a2e
commit 1ed1dcee6b
No known key found for this signature in database
GPG key ID: B6C15EBA0357C9AA
5 changed files with 443 additions and 10 deletions

1
.gitignore vendored
View file

@ -149,3 +149,4 @@ share/container.tar
share/container.tar.gz
share/image-id.txt
container/container-pip-requirements.txt
.doit.db.db

377
dodo.py Normal file
View file

@ -0,0 +1,377 @@
import json
import os
import platform
import shutil
from pathlib import Path
from doit import get_var
from doit.action import CmdAction
ARCH = "arm64" if platform.machine() == "arm64" else "i686"
VERSION = open("share/version.txt").read().strip()
FEDORA_VERSIONS = ["40", "41"]
DEBIAN_VERSIONS = ["bullseye", "focal", "jammy", "mantic", "noble", "trixie"]
# In the case of an Apple Silicon machine, there's no need to run anything else than
# building the .dmg.
if ARCH == "arm64":
DOIT_CONFIG = {'default_tasks': ['macos_build_dmg']}
### Task Parameters
PARAM_APPLE_ID = {
"name": "apple_id",
"long": "apple-id",
"default": "fpf@example.com",
"help": "The Apple developer ID that will be used for signing the .dmg",
}
PARAM_USE_CACHE = {
"name": "use_cache",
"long": "use-cache",
"help": (
"Whether to use cached results or not. For reproducibility reasons,"
" it's best to leave it to false"
),
"default": False,
}
### Global parameters
#
# Read more about global parameters in
# https://pydoit.org/task-args.html#command-line-variables-doit-get-var
CONTAINER_RUNTIME = get_var('runtime', 'podman')
RELEASE_DIR = Path(get_var("release_dir", Path.home() / "release-assets" / VERSION))
### File dependencies
#
# Define all the file dependencies we'll see later in tasks here, since some file
# dependencies are shared between tasks.
def list_files(path, recursive=False):
"""List files in a directory, and optionally traverse into subdirectories."""
filepaths = []
for root, _, files in os.walk(path):
for f in files:
if f.endswith(".pyc"):
continue
filepaths.append(Path(root) / f)
if not recursive:
break
return filepaths
def list_language_data():
"""List the expected language data that Dangerzone downloads and stores locally."""
tessdata_dir = Path("share") / "tessdata"
langs = json.loads(open(tessdata_dir.parent / "ocr-languages.json").read()).values()
targets = [tessdata_dir / f"{lang}.traineddata" for lang in langs]
targets.append(tessdata_dir)
return targets
TESSDATA_DEPS = ["install/common/download-tessdata.py", "share/ocr-languages.json"]
TESSDATA_TARGETS = list_language_data()
IMAGE_DEPS = [
"Dockerfile",
"poetry.lock",
*list_files("dangerzone/conversion"),
"dangerzone/gvisor_wrapper/entrypoint.py",
"install/common/build-image.py",
]
IMAGE_TARGETS = ["share/container.tar.gz", "share/image-id.txt"]
SOURCE_DEPS = [
*list_files("assets"),
*list_files("share"),
*list_files("dangerzone", recursive=True),
]
PYTHON_DEPS = ["poetry.lock", "pyproject.toml"]
DMG_DEPS = [
*list_files("install/macos"),
*TESSDATA_TARGETS,
*IMAGE_TARGETS,
*PYTHON_DEPS,
*SOURCE_DEPS,
]
LINUX_DEPS = [
*list_files("install/linux"),
*IMAGE_TARGETS,
*PYTHON_DEPS,
*SOURCE_DEPS,
]
DEB_DEPS = [*LINUX_DEPS, *list_files("debian")]
RPM_DEPS = [*LINUX_DEPS, *list_files("qubes")]
def copy_dir(src, dst):
"""Copy a directory to a destination dir, and overwrite it if it exists."""
shutil.rmtree(dst, ignore_errors=True)
shutil.copytree(src, dst)
def create_release_dir():
RELEASE_DIR.mkdir(parents=True, exist_ok=True)
(RELEASE_DIR / "assets").mkdir(exist_ok=True)
(RELEASE_DIR / "tmp").mkdir(exist_ok=True)
def build_linux_pkg(distro, version, cwd, qubes=False):
"""Generic command for building a .deb/.rpm in a Dangerzone dev environment."""
pkg = "rpm" if distro == "fedora" else "deb"
cmd = [
"python3",
"./dev_scripts/env.py",
"--distro",
distro,
"--version",
version,
"run",
"--no-gui",
"--dev",
f"./dangerzone/install/linux/build-{pkg}.py",
]
if qubes:
cmd += ["--qubes"]
return CmdAction(" ".join(cmd), cwd=cwd)
def build_deb(cwd):
"""Build a .deb package on Debian Bookworm."""
return build_linux_pkg(distro="debian", version="bookworm", cwd=cwd)
def build_rpm(version, cwd, qubes=False):
"""Build an .rpm package on the requested Fedora distro."""
return build_linux_pkg(distro="Fedora", version=version, cwd=cwd, qubes=qubes)
def task_clean_container_runtime():
"""Clean the storage space of the container runtime."""
return {
"actions": None,
"clean": [
[CONTAINER_RUNTIME, "system", "prune", "-a", "-f"],
],
}
def task_check_container_runtime():
"""Test that the container runtime is ready."""
return {
"actions": [
["which", CONTAINER_RUNTIME],
[CONTAINER_RUNTIME, "ps"],
],
}
def task_macos_check_cert():
"""Test that the Apple developer certificate can be used."""
return {
"actions": [
"xcrun notarytool history --apple-id %(apple_id)s --keychain-profile dz-notarytool-release-key"
],
"params": [PARAM_APPLE_ID],
}
def task_macos_check_system():
"""Run macOS specific system checks, as well as the generic ones."""
return {
"actions": None,
"task_dep": [
"check_container_runtime",
"macos_check_cert",
],
}
def task_init_release_dir():
"""Create a directory for release artifacts."""
return {
"actions": [create_release_dir],
"clean": [f"rm -rf {RELEASE_DIR}"],
}
def task_download_tessdata():
"""Download the Tesseract data using ./install/common/download-tessdata.py"""
return {
"actions": ["python install/common/download-tessdata.py"],
"file_dep": TESSDATA_DEPS,
"targets": TESSDATA_TARGETS,
"clean": True,
}
def task_build_image():
"""Build the container image using ./install/common/build-image.py"""
img_src = "share/container.tar.gz"
img_dst = RELEASE_DIR / f"container-{VERSION}-{ARCH}.tar.gz" # FIXME: Add arch
img_id_src = "share/image-id.txt"
img_id_dst = RELEASE_DIR / "image-id.txt" # FIXME: Add arch
return {
"actions": [
f"python install/common/build-image.py --use-cache=%(use_cache)s --runtime={CONTAINER_RUNTIME}",
f"cp {img_src} {img_dst}",
f"cp {img_id_src} {img_id_dst}",
],
"params": [PARAM_USE_CACHE],
"file_dep": IMAGE_DEPS,
"targets": [img_src, img_dst, img_id_src, img_id_dst],
"task_dep": [
"init_release_dir",
"check_container_runtime",
],
"clean": True,
}
def task_poetry_install():
"""Setup the Poetry environment"""
return {
"actions": ["poetry install --sync"],
}
def task_macos_build_dmg():
"""Build the macOS app bundle for Dangerzone."""
dz_dir = RELEASE_DIR / "tmp" / "macos"
dmg_src = dz_dir / "dist" / "Dangerzone.dmg"
dmg_dst = RELEASE_DIR / f"Dangerzone-{VERSION}-{ARCH}.dmg" # FIXME: Add -arch
return {
"actions": [
(copy_dir, [".", dz_dir]),
f"cd {dz_dir} && poetry run install/macos/build-app.py --with-codesign",
(
"xcrun notarytool submit --wait --apple-id %(apple_id)s"
f" --keychain-profile dz-notarytool-release-key {dmg_src}"
),
f"xcrun stapler staple {dmg_src}",
["cp", "-r", dmg_src, dmg_dst],
["rm", "-rf", dz_dir],
],
"params": [PARAM_APPLE_ID],
"file_dep": DMG_DEPS,
"task_dep": [
"init_release_dir",
"poetry_install",
"download_tessdata",
],
"targets": [dmg_dst],
"clean": True,
}
def task_debian_env():
"""Build a Debian Bookworm dev environment."""
return {
"actions": [
[
"python3",
"./dev_scripts/env.py",
"--distro",
"debian",
"--version",
"bookworm",
"build-dev",
]
],
"task_dep": ["check_container_runtime"],
}
def task_debian_deb():
"""Build a Debian package for Debian Bookworm."""
dz_dir = RELEASE_DIR / "tmp" / "debian"
deb_name = f"dangerzone_{VERSION}-1_amd64.deb"
deb_src = dz_dir / "deb_dist" / deb_name
deb_dst = RELEASE_DIR / deb_name
return {
"actions": [
(copy_dir, [".", dz_dir]),
build_deb(cwd=dz_dir),
["cp", deb_src, deb_dst],
["rm", "-rf", dz_dir],
],
"file_dep": DEB_DEPS,
"task_dep": [
"init_release_dir",
"debian_env",
],
"targets": [deb_dst],
"clean": True,
}
def task_fedora_env():
"""Build Fedora dev environments."""
for version in FEDORA_VERSIONS:
yield {
"name": version,
"actions": [
[
"python3",
"./dev_scripts/env.py",
"--distro",
"fedora",
"--version",
version,
"build-dev",
],
],
"task_dep": ["check_container_runtime"],
}
def task_fedora_rpm():
"""Build Fedora packages for every supported version."""
for version in FEDORA_VERSIONS:
for qubes in (True, False):
qubes_ident = "-qubes" if qubes else ""
dz_dir = RELEASE_DIR / "tmp" / f"f{version}{qubes_ident}"
rpm_names = [
f"dangerzone{qubes_ident}-{VERSION}-1.fc{version}.x86_64.rpm",
f"dangerzone{qubes_ident}-{VERSION}-1.fc{version}.src.rpm",
]
rpm_src = [dz_dir / "dist" / rpm_name for rpm_name in rpm_names]
rpm_dst = [RELEASE_DIR / rpm_name for rpm_name in rpm_names]
yield {
"name": version + qubes_ident,
"actions": [
(copy_dir, [".", dz_dir]),
build_rpm(version, cwd=dz_dir, qubes=qubes),
["cp", *rpm_src, RELEASE_DIR],
["rm", "-rf", dz_dir],
],
"file_dep": RPM_DEPS,
"task_dep": [
"init_release_dir",
f"fedora_env:{version}",
],
"targets": rpm_dst,
"clean": True,
}
def task_git_archive():
"""Build a Git archive of the repo."""
target = f"{RELEASE_DIR}/dangerzone-{VERSION}.tar.gz"
return {
"actions": [
f"git archive --format=tar.gz -o {target} --prefix=dangerzone/ v{VERSION}"
],
"targets": [target],
"task_dep": ["init_release_dir"],
}

View file

@ -17,6 +17,16 @@ elif platform.system() == "Linux":
ARCH = platform.machine()
def str2bool(v):
if isinstance(v, bool):
return v
if v.lower() in ('yes', 'true', 't', 'y', '1'):
return True
elif v.lower() in ('no', 'false', 'f', 'n', '0'):
return False
else:
raise argparse.ArgumentTypeError('Boolean value expected.')
def main():
parser = argparse.ArgumentParser()
@ -40,7 +50,10 @@ def main():
)
parser.add_argument(
"--use-cache",
action="store_true",
type=str2bool,
nargs='?',
default=False,
const=True,
help="Use the builder's cache to speed up the builds (not suitable for release builds)",
)
args = parser.parse_args()

39
poetry.lock generated
View file

@ -229,6 +229,17 @@ files = [
[package.dependencies]
colorama = {version = "*", markers = "platform_system == \"Windows\""}
[[package]]
name = "cloudpickle"
version = "3.1.0"
description = "Pickler class to extend the standard pickle.Pickler functionality"
optional = false
python-versions = ">=3.8"
files = [
{file = "cloudpickle-3.1.0-py3-none-any.whl", hash = "sha256:fe11acda67f61aaaec473e3afe030feb131d78a43461b718185363384f1ba12e"},
{file = "cloudpickle-3.1.0.tar.gz", hash = "sha256:81a929b6e3c7335c863c771d673d105f02efdb89dfaba0c90495d1c64796601b"},
]
[[package]]
name = "colorama"
version = "0.4.6"
@ -412,6 +423,24 @@ files = [
{file = "cx_logging-3.2.1.tar.gz", hash = "sha256:812665ae5012680a6fe47095c3772bce638e47cf05b2c3483db3bdbe6b06da44"},
]
[[package]]
name = "doit"
version = "0.36.0"
description = "doit - Automation Tool"
optional = false
python-versions = ">=3.8"
files = [
{file = "doit-0.36.0-py3-none-any.whl", hash = "sha256:ebc285f6666871b5300091c26eafdff3de968a6bd60ea35dd1e3fc6f2e32479a"},
{file = "doit-0.36.0.tar.gz", hash = "sha256:71d07ccc9514cb22fe59d98999577665eaab57e16f644d04336ae0b4bae234bc"},
]
[package.dependencies]
cloudpickle = "*"
importlib-metadata = ">=4.4"
[package.extras]
toml = ["tomli"]
[[package]]
name = "exceptiongroup"
version = "1.2.2"
@ -554,7 +583,6 @@ python-versions = ">=3.8"
files = [
{file = "lief-0.15.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a80246b96501b2b1d4927ceb3cb817eda9333ffa9e07101358929a6cffca5dae"},
{file = "lief-0.15.1-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:84bf310710369544e2bb82f83d7fdab5b5ac422651184fde8bf9e35f14439691"},
{file = "lief-0.15.1-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:517dc5dad31c754720a80a87ad9e6cb1e48223d4505980c2fd86072bd4f69001"},
{file = "lief-0.15.1-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:8fb58efb77358291109d2675d5459399c0794475b497992d0ecee18a4a46a207"},
{file = "lief-0.15.1-cp310-cp310-manylinux_2_33_aarch64.whl", hash = "sha256:d5852a246361bbefa4c1d5930741765a2337638d65cfe30de1b7d61f9a54b865"},
{file = "lief-0.15.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:12e53dc0253c303df386ae45487a2f0078026602b36d0e09e838ae1d4dbef958"},
@ -562,7 +590,6 @@ files = [
{file = "lief-0.15.1-cp310-cp310-win_amd64.whl", hash = "sha256:ddf2ebd73766169594d631b35f84c49ef42871de552ad49f36002c60164d0aca"},
{file = "lief-0.15.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:20508c52de0dffcee3242253541609590167a3e56150cbacb506fdbb822206ef"},
{file = "lief-0.15.1-cp311-cp311-macosx_11_0_x86_64.whl", hash = "sha256:0750c892fd3b7161a3c2279f25fe1844427610c3a5a4ae23f65674ced6f93ea5"},
{file = "lief-0.15.1-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:3e49bd595a8548683bead982bc15b064257fea3110fd15e22fb3feb17d97ad1c"},
{file = "lief-0.15.1-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:a8634ea79d6d9862297fadce025519ab25ff01fcadb333cf42967c6295f0d057"},
{file = "lief-0.15.1-cp311-cp311-manylinux_2_33_aarch64.whl", hash = "sha256:1e11e046ad71fe8c81e1a8d1d207fe2b99c967d33ce79c3d3915cb8f5ecacf52"},
{file = "lief-0.15.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:674b620cdf1d686f52450fd97c1056d4c92e55af8217ce85a1b2efaf5b32140b"},
@ -570,15 +597,11 @@ files = [
{file = "lief-0.15.1-cp311-cp311-win_amd64.whl", hash = "sha256:e9b96a37bf11ca777ff305d85d957eabad2a92a6e577b6e2fb3ab79514e5a12e"},
{file = "lief-0.15.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1a96f17c2085ef38d12ad81427ae8a5d6ad76f0bc62a1e1f5fe384255cd2cc94"},
{file = "lief-0.15.1-cp312-cp312-macosx_11_0_x86_64.whl", hash = "sha256:d780af1762022b8e01b613253af490afea3864fbd6b5a49c6de7cea8fde0443d"},
{file = "lief-0.15.1-cp312-cp312-manylinux2014_aarch64.whl", hash = "sha256:536a4ecd46b295b3acac0d60a68d1646480b7761ade862c6c87ccbb41229fae3"},
{file = "lief-0.15.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:d0f10d80202de9634a16786b53ba3a8f54ae8b9a9e124a964d83212444486087"},
{file = "lief-0.15.1-cp312-cp312-manylinux_2_33_aarch64.whl", hash = "sha256:864f17ecf1736296e6d5fc38b11983f9d19a5e799f094e21e20d58bfb1b95b80"},
{file = "lief-0.15.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c2ec738bcafee8a569741f4a749f0596823b12f10713306c7d0cbbf85759f51c"},
{file = "lief-0.15.1-cp312-cp312-win32.whl", hash = "sha256:db38619edf70e27fb3686b8c0f0bec63ad494ac88ab51660c5ecd2720b506e41"},
{file = "lief-0.15.1-cp312-cp312-win_amd64.whl", hash = "sha256:28bf0922de5fb74502a29cc47930d3a052df58dc23ab6519fa590e564f194a60"},
{file = "lief-0.15.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0805301e8fef9b13da00c33c831fb0c05ea892309230f3a35551c2dfaf69b11d"},
{file = "lief-0.15.1-cp313-cp313-macosx_11_0_x86_64.whl", hash = "sha256:7580defe140e921bc4f210e8a6cb115fcf2923f00d37800b1626168cbca95108"},
{file = "lief-0.15.1-cp313-cp313-manylinux2014_aarch64.whl", hash = "sha256:c0119306b6a38759483136de7242b7c2e0a23f1de1d4ae53f12792c279607410"},
{file = "lief-0.15.1-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:0616e6048f269d262ff93d67c497ebff3c1d3965ffb9427b0f2b474764fd2e8c"},
{file = "lief-0.15.1-cp313-cp313-manylinux_2_33_aarch64.whl", hash = "sha256:6a08b2e512a80040429febddc777768c949bcd53f6f580e902e41ec0d9d936b8"},
{file = "lief-0.15.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:fcd489ff80860bcc2b2689faa330a46b6d66f0ee3e0f6ef9e643e2b996128a06"},
@ -586,7 +609,6 @@ files = [
{file = "lief-0.15.1-cp313-cp313-win_amd64.whl", hash = "sha256:5af7dcb9c3f44baaf60875df6ba9af6777db94776cc577ee86143bcce105ba2f"},
{file = "lief-0.15.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:f9757ff0c7c3d6f66e5fdcc6a9df69680fad0dc2707d64a3428f0825dfce1a85"},
{file = "lief-0.15.1-cp38-cp38-macosx_11_0_x86_64.whl", hash = "sha256:8ac3cd099be2580d0e15150b1d2f5095c38f150af89993ddf390d7897ee8135f"},
{file = "lief-0.15.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:e732619acc34943b504c867258fc0196f1931f72c2a627219d4f116a7acc726d"},
{file = "lief-0.15.1-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:4dedeab498c312a29b58f16b739895f65fa54b2a21b8d98b111e99ad3f7e30a8"},
{file = "lief-0.15.1-cp38-cp38-manylinux_2_33_aarch64.whl", hash = "sha256:b9217578f7a45f667503b271da8481207fb4edda8d4a53e869fb922df6030484"},
{file = "lief-0.15.1-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:82e6308ad8bd4bc7eadee3502ede13a5bb398725f25513a0396c8dba850f58a1"},
@ -594,7 +616,6 @@ files = [
{file = "lief-0.15.1-cp38-cp38-win_amd64.whl", hash = "sha256:a079a76bca23aa73c850ab5beb7598871a1bf44662658b952cead2b5ddd31bee"},
{file = "lief-0.15.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:785a3aa14575f046ed9c8d44ea222ea14c697cd03b5331d1717b5b0cf4f72466"},
{file = "lief-0.15.1-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:d7044553cf07c8a2ab6e21874f07585610d996ff911b9af71dc6085a89f59daa"},
{file = "lief-0.15.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:fa020f3ed6e95bb110a4316af544021b74027d18bf4671339d4cffec27aa5884"},
{file = "lief-0.15.1-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:13285c3ff5ef6de2421d85684c954905af909db0ad3472e33c475e5f0f657dcf"},
{file = "lief-0.15.1-cp39-cp39-manylinux_2_33_aarch64.whl", hash = "sha256:932f880ee8a130d663a97a9099516d8570b1b303af7816e70a02f9931d5ef4c2"},
{file = "lief-0.15.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:de9453f94866e0f2c36b6bd878625880080e7e5800788f5cbc06a76debf283b9"},
@ -1189,4 +1210,4 @@ type = ["pytest-mypy"]
[metadata]
lock-version = "2.0"
python-versions = ">=3.9,<3.13"
content-hash = "5d1ff28aa04c3a814280e55c0b2a307efe5ca953cd4cb281056c35fd2e53fdf0"
content-hash = "8367cee9a978ba6df32f44e902d223156e321b2cb4ea8a9d7a4bf1f88392a8c0"

View file

@ -23,6 +23,7 @@ pyxdg = {version = "*", platform = "linux"}
requests = "*"
markdown = "*"
packaging = "*"
doit = "^0.36.0"
[tool.poetry.scripts]
dangerzone = 'dangerzone:main'
@ -66,6 +67,26 @@ skip_gitignore = true
# This is necessary due to https://github.com/PyCQA/isort/issues/1835
follow_links = false
[tool.doit.commands.clean]
# XXX: Change this to false if you REALLY want to clean your environment. Note
# that this command will:
# * prune container images,
# * clean the Git repo, and
# * remove all tasks output
#
# Else, the `doit clean` comamnd will print the commands that would run instead.
dryrun = true
[tool.doit.tasks.macos_check_cert]
apple_id = "fpf@example.com"
[tool.doit.tasks.macos_codesign]
apple_id = "fpf@example.com"
[tool.doit.tasks.build_image]
use_cache = false
force_tag = ""
[build-system]
requires = ["poetry-core>=1.2.0"]
build-backend = "poetry.core.masonry.api"