Merge b63c1f4f5e into d722800a4b

This reverts commit 68f8338d20.

Fixes #982

.github/workflows/build.yml

@@ -1,6 +1,10 @@
 name: Build dev environments
 on:
+  pull_request:
   push:
+    branches:
+      - main
+      - "test/**"
   schedule:
     - cron: "0 0 * * *" # Run every day at 00:00 UTC.
 

.github/workflows/check_push.yml

@@ -1,6 +1,6 @@
 name: Check branch conformity
 on:
-  push:
+  pull_request:
 
 jobs:
   prevent-fixup-commits:

.github/workflows/ci.yml

@@ -1,8 +1,10 @@
 name: Tests
 on:
-  push:
   pull_request:
-    branches: [main]
+  push:
+    branches:
+      - main
+      - "test/**"
   schedule:
     - cron: "2 0 * * *" # Run every day at 02:00 UTC.
   workflow_dispatch:
@@ -91,7 +93,8 @@ jobs:
 
   windows:
     runs-on: windows-latest
-    needs: download-tessdata
+    needs:
+      - download-tessdata
     env:
       DUMMY_CONVERSION: 1
     steps:
@@ -121,7 +124,8 @@ jobs:
   macOS:
     name: "macOS (${{ matrix.arch }})"
     runs-on: ${{ matrix.runner }}
-    needs: download-tessdata
+    needs:
+      - download-tessdata
     strategy:
       matrix:
         include:
@@ -149,9 +153,10 @@ jobs:
         run: poetry run make test
 
   build-deb:
+    needs:
+      - build-container-image
     name: "build-deb (${{ matrix.distro }} ${{ matrix.version }})"
     runs-on: ubuntu-latest
-    needs: build-container-image
     strategy:
       matrix:
         include:
@@ -219,7 +224,8 @@ jobs:
   install-deb:
     name: "install-deb (${{ matrix.distro }} ${{ matrix.version }})"
     runs-on: ubuntu-latest
-    needs: build-deb
+    needs: 
+      - build-deb
     strategy:
       matrix:
         include:
@@ -273,7 +279,8 @@ jobs:
   build-install-rpm:
     name: "build-install-rpm (${{ matrix.distro }} ${{matrix.version}})"
     runs-on: ubuntu-latest
-    needs: build-container-image
+    needs:
+      - build-container-image
     strategy:
       matrix:
         distro: ["fedora"]

.github/workflows/scan.yml

@@ -1,8 +1,9 @@
 name: Scan latest app and container
 on:
   push:
+    branches:
+      - main
   pull_request:
-    branches: [ main ]
   schedule:
     - cron: '0 0 * * *' # Run every day at 00:00 UTC.
   workflow_dispatch:

.github/workflows/scan_released.yml

@@ -6,14 +6,21 @@ on:
 
 jobs:
   security-scan-container:
-    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - runs-on: ubuntu-latest
+            arch: i686
+          - runs-on: macos-latest
+            arch: arm64
+    runs-on: ${{ matrix.runs-on }}
     steps:
       - name: Checkout
         uses: actions/checkout@v4
       - name: Download container image for the latest release and load it
         run: |
           VERSION=$(curl https://api.github.com/repos/freedomofpress/dangerzone/releases/latest | jq -r '.tag_name')
-          CONTAINER_FILENAME=container-${VERSION:1}-i686.tar.gz
+          CONTAINER_FILENAME=container-${VERSION:1}-${{ matrix.arch }}.tar.gz
           wget https://github.com/freedomofpress/dangerzone/releases/download/${VERSION}/${CONTAINER_FILENAME} -O ${CONTAINER_FILENAME}
           docker load -i ${CONTAINER_FILENAME}
       # NOTE: Scan first without failing, else we won't be able to read the scan
@@ -30,7 +37,7 @@ jobs:
         uses: github/codeql-action/upload-sarif@v3
         with:
           sarif_file: ${{ steps.scan_container.outputs.sarif }}
-          category: container
+          category: container-${{ matrix.arch }}
       - name: Inspect container scan report
         run: cat ${{ steps.scan_container.outputs.sarif }}
       - name: Scan container image

CHANGELOG.md

@@ -7,6 +7,11 @@ since 0.4.1, and this project adheres to [Semantic Versioning](https://semver.or
 
 ## [Unreleased](https://github.com/freedomofpress/dangerzone/compare/v0.8.0...HEAD)
 
+### Added
+
+- Disable gVisor's DirectFS feature ([#226](https://github.com/freedomofpress/dangerzone/issues/226)).
+  Thanks [EtiennePerot](https://github.com/EtiennePerot) for the contribution.
+
 ## [0.8.0](https://github.com/freedomofpress/dangerzone/compare/v0.8.0...0.7.1)
 
 ### Added

Dockerfile

@@ -74,9 +74,7 @@ FROM alpine:latest
 RUN apk --no-cache -U upgrade && \
     apk --no-cache add python3
 
-# Temporarily pin gVisor to the latest working version (release-20240826.0).
-# See: https://github.com/freedomofpress/dangerzone/issues/928
-RUN GVISOR_URL="https://storage.googleapis.com/gvisor/releases/release/20240826/$(uname -m)"; \
+RUN GVISOR_URL="https://storage.googleapis.com/gvisor/releases/release/latest/$(uname -m)"; \
     wget "${GVISOR_URL}/runsc" "${GVISOR_URL}/runsc.sha512" && \
     sha512sum -c runsc.sha512 && \
     rm -f runsc.sha512 && \

dangerzone/gvisor_wrapper/entrypoint.py

@@ -142,6 +142,9 @@ runsc_argv = [
     "--rootless=true",
     "--network=none",
     "--root=/home/dangerzone/.containers",
+    # Disable DirectFS for to make the seccomp filter even stricter,
+    # at some performance cost.
+    "--directfs=false",
 ]
 if os.environ.get("RUNSC_DEBUG"):
     runsc_argv += ["--debug=true", "--alsologtostderr=true"]

dev_scripts/generate-release-notes.py

@@ -0,0 +1,254 @@
+#!/usr/bin/env python3
+
+import argparse
+import asyncio
+import re
+import sys
+from datetime import datetime
+from typing import Dict, List, Optional, Tuple
+
+import httpx
+
+REPOSITORY = "https://github.com/freedomofpress/dangerzone/"
+TEMPLATE = "- {title} ([#{number}]({url}))"
+
+
+def parse_version(version: str) -> Tuple[int, int]:
+    """Extract major.minor from version string, ignoring patch"""
+    match = re.match(r"v?(\d+)\.(\d+)", version)
+    if not match:
+        raise ValueError(f"Invalid version format: {version}")
+    return (int(match.group(1)), int(match.group(2)))
+
+
+async def get_last_minor_release(
+    client: httpx.AsyncClient, owner: str, repo: str
+) -> Optional[str]:
+    """Get the latest minor release date (ignoring patches)"""
+    response = await client.get(f"https://api.github.com/repos/{owner}/{repo}/releases")
+    response.raise_for_status()
+    releases = response.json()
+
+    if not releases:
+        return None
+
+    # Get the latest minor version by comparing major.minor numbers
+    current_version = parse_version(releases[0]["tag_name"])
+    latest_date = None
+
+    for release in releases:
+        try:
+            version = parse_version(release["tag_name"])
+            if version < current_version:
+                latest_date = release["published_at"]
+                break
+        except ValueError:
+            continue
+
+    return latest_date
+
+
+async def get_issue_details(
+    client: httpx.AsyncClient, owner: str, repo: str, issue_number: int
+) -> Optional[dict]:
+    """Get issue title and number if it exists"""
+    response = await client.get(
+        f"https://api.github.com/repos/{owner}/{repo}/issues/{issue_number}"
+    )
+    if response.is_success:
+        data = response.json()
+        return {
+            "title": data["title"],
+            "number": data["number"],
+            "url": data["html_url"],
+        }
+    return None
+
+
+def extract_issue_number(pr_body: Optional[str]) -> Optional[int]:
+    """Extract issue number from PR body looking for common formats like 'Fixes #123' or 'Closes #123'"""
+    if not pr_body:
+        return None
+
+    patterns = [
+        r"(?:closes|fixes|resolves)\s*#(\d+)",
+        r"(?:close|fix|resolve)\s*#(\d+)",
+    ]
+
+    for pattern in patterns:
+        match = re.search(pattern, pr_body.lower())
+        if match:
+            return int(match.group(1))
+
+    return None
+
+
+async def verify_commit_in_master(
+    client: httpx.AsyncClient, owner: str, repo: str, commit_id: str
+) -> bool:
+    """Verify if a commit exists in master"""
+    response = await client.get(
+        f"https://api.github.com/repos/{owner}/{repo}/commits/{commit_id}"
+    )
+    return response.is_success and response.json().get("commit") is not None
+
+
+async def process_issue_events(
+    client: httpx.AsyncClient, owner: str, repo: str, issue: Dict
+) -> Optional[Dict]:
+    """Process events for a single issue"""
+    events_response = await client.get(f"{issue['url']}/events")
+    if not events_response.is_success:
+        return None
+
+    for event in events_response.json():
+        if event["event"] == "closed" and event.get("commit_id"):
+            if await verify_commit_in_master(client, owner, repo, event["commit_id"]):
+                return {
+                    "title": issue["title"],
+                    "number": issue["number"],
+                    "url": issue["html_url"],
+                }
+    return None
+
+
+async def get_closed_issues(
+    client: httpx.AsyncClient, owner: str, repo: str, since: str
+) -> List[Dict]:
+    """Get issues closed by commits to master since the given date"""
+    response = await client.get(
+        f"https://api.github.com/repos/{owner}/{repo}/issues",
+        params={
+            "state": "closed",
+            "sort": "updated",
+            "direction": "desc",
+            "since": since,
+            "per_page": 100,
+        },
+    )
+    response.raise_for_status()
+
+    tasks = []
+    since_date = datetime.strptime(since, "%Y-%m-%dT%H:%M:%SZ")
+
+    for issue in response.json():
+        if "pull_request" in issue:
+            continue
+
+        closed_at = datetime.strptime(issue["closed_at"], "%Y-%m-%dT%H:%M:%SZ")
+        if closed_at <= since_date:
+            continue
+
+        tasks.append(process_issue_events(client, owner, repo, issue))
+
+    results = await asyncio.gather(*tasks)
+    return [r for r in results if r is not None]
+
+
+async def process_pull_request(
+    client: httpx.AsyncClient,
+    owner: str,
+    repo: str,
+    pr: Dict,
+    closed_issues: List[Dict],
+) -> Optional[str]:
+    """Process a single pull request"""
+    issue_number = extract_issue_number(pr.get("body"))
+    if issue_number:
+        issue = await get_issue_details(client, owner, repo, issue_number)
+        if issue:
+            if not any(i["number"] == issue["number"] for i in closed_issues):
+                return TEMPLATE.format(**issue)
+            return None
+
+    return TEMPLATE.format(title=pr["title"], number=pr["number"], url=pr["html_url"])
+
+
+async def get_changes_since_last_release(
+    owner: str, repo: str, token: Optional[str] = None
+) -> List[str]:
+    headers = {
+        "Accept": "application/vnd.github.v3+json",
+    }
+    if token:
+        headers["Authorization"] = f"token {token}"
+    else:
+        print(
+            "Warning: No token provided. API rate limiting may occur.", file=sys.stderr
+        )
+
+    async with httpx.AsyncClient(headers=headers, timeout=30.0) as client:
+        # Get the date of last minor release
+        since = await get_last_minor_release(client, owner, repo)
+        if not since:
+            return []
+
+        changes = []
+
+        # Get issues closed by commits to master
+        closed_issues = await get_closed_issues(client, owner, repo, since)
+        changes.extend([TEMPLATE.format(**issue) for issue in closed_issues])
+
+        # Get merged PRs
+        response = await client.get(
+            f"https://api.github.com/repos/{owner}/{repo}/pulls",
+            params={
+                "state": "closed",
+                "sort": "updated",
+                "direction": "desc",
+                "per_page": 100,
+            },
+        )
+        response.raise_for_status()
+
+        # Process PRs in parallel
+        pr_tasks = []
+        for pr in response.json():
+            if not pr["merged_at"]:
+                continue
+            if since and pr["merged_at"] <= since:
+                break
+
+            pr_tasks.append(
+                process_pull_request(client, owner, repo, pr, closed_issues)
+            )
+
+        pr_results = await asyncio.gather(*pr_tasks)
+        changes.extend([r for r in pr_results if r is not None])
+
+        return changes
+
+
+async def main_async():
+    parser = argparse.ArgumentParser(description="Generate release notes from GitHub")
+    parser.add_argument("--token", "-t", help="the file path to the GitHub API token")
+    args = parser.parse_args()
+
+    token = None
+    if args.token:
+        with open(args.token) as f:
+            token = f.read().strip()
+    try:
+        url_path = REPOSITORY.rstrip("/").split("github.com/")[1]
+        owner, repo = url_path.split("/")[-2:]
+    except (ValueError, IndexError):
+        print("Error: Invalid GitHub URL", file=sys.stderr)
+        sys.exit(1)
+
+    try:
+        notes = await get_changes_since_last_release(owner, repo, token)
+        print("\n".join(notes))
+    except httpx.HTTPError as e:
+        print(f"Error: {e}", file=sys.stderr)
+        sys.exit(1)
+    except Exception as e:
+        print(f"Error: {e}", file=sys.stderr)
+        sys.exit(1)
+
+
+def main():
+    asyncio.run(main_async())
+
+
+if __name__ == "__main__":
+    main()

dev_scripts/qa.py

@@ -813,30 +813,39 @@ class QAWindows(QABase):
         while msvcrt.kbhit():
             msvcrt.getch()
 
-    @QABase.task(
-        f"Install the latest version of Python {PYTHON_VERSION_STR}", ref=REF_BUILD
-    )
-    def install_python(self):
-        cur_version = list(sys.version_info[:3])
-
-        logger.info("Getting latest Python release")
+    def get_latest_python_release(self):
         with urllib.request.urlopen(EOL_PYTHON_URL) as f:
             resp = f.read()
             releases = json.loads(resp)
             for release in releases:
                 if release["cycle"] == PYTHON_VERSION_STR:
-                    latest_version = [int(num) for num in release["latest"].split(".")]
+                    # Transform the Python version string (e.g., "3.12.7") into a list
+                    # (e.g., [3, 12, 7]), and return it
+                    return [int(num) for num in release["latest"].split(".")]
+
+            raise RuntimeError(
+                f"Could not find a Python release for version {PYTHON_VERSION_STR}"
+            )
+
+    @QABase.task(
+        f"Install the latest version of Python {PYTHON_VERSION_STR}", ref=REF_BUILD
+    )
+    def install_python(self):
+        logger.info("Getting latest Python release")
+        try:
+            latest_version = self.get_latest_python_release()
+        except Exception:
+            logger.error("Could not verify that the latest Python version is installed")
+
+        cur_version = list(sys.version_info[:3])
         if latest_version > cur_version:
             self.prompt(
-                            f"You need to install the latest Python version ({release['latest']})"
+                f"You need to install the latest Python version ({latest_version})"
             )
         elif latest_version == cur_version:
             logger.info(
-                            f"Verified that the latest Python version ({release['latest']}) is installed"
+                f"Verified that the latest Python version ({latest_version}) is installed"
             )
-                        return
-
-        logger.error("Could not verify that the latest Python version is installed")
 
     @QABase.task("Install and Run Docker Desktop", ref=REF_BUILD)
     def install_docker(self):

poetry.lock

@@ -11,6 +11,28 @@ files = [
     {file = "altgraph-0.17.4.tar.gz", hash = "sha256:1b5afbb98f6c4dcadb2e2ae6ab9fa994bbb8c1d75f4fa96d340f9437ae454406"},
 ]
 
+[[package]]
+name = "anyio"
+version = "4.6.2.post1"
+description = "High level compatibility layer for multiple asynchronous event loop implementations"
+optional = false
+python-versions = ">=3.9"
+files = [
+    {file = "anyio-4.6.2.post1-py3-none-any.whl", hash = "sha256:6d170c36fba3bdd840c73d3868c1e777e33676a69c3a72cf0a0d5d6d8009b61d"},
+    {file = "anyio-4.6.2.post1.tar.gz", hash = "sha256:4c8bc31ccdb51c7f7bd251f51c609e038d63e34219b44aa86e47576389880b4c"},
+]
+
+[package.dependencies]
+exceptiongroup = {version = ">=1.0.2", markers = "python_version < \"3.11\""}
+idna = ">=2.8"
+sniffio = ">=1.1"
+typing-extensions = {version = ">=4.1", markers = "python_version < \"3.11\""}
+
+[package.extras]
+doc = ["Sphinx (>=7.4,<8.0)", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme"]
+test = ["anyio[trio]", "coverage[toml] (>=7)", "exceptiongroup (>=1.2.0)", "hypothesis (>=4.0)", "psutil (>=5.9)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "truststore (>=0.9.1)", "uvloop (>=0.21.0b1)"]
+trio = ["trio (>=0.26.1)"]
+
 [[package]]
 name = "appdirs"
 version = "1.4.4"
@@ -404,6 +426,63 @@ files = [
 [package.extras]
 test = ["pytest (>=6)"]
 
+[[package]]
+name = "h11"
+version = "0.14.0"
+description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "h11-0.14.0-py3-none-any.whl", hash = "sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761"},
+    {file = "h11-0.14.0.tar.gz", hash = "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d"},
+]
+
+[[package]]
+name = "httpcore"
+version = "1.0.7"
+description = "A minimal low-level HTTP client."
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "httpcore-1.0.7-py3-none-any.whl", hash = "sha256:a3fff8f43dc260d5bd363d9f9cf1830fa3a458b332856f34282de498ed420edd"},
+    {file = "httpcore-1.0.7.tar.gz", hash = "sha256:8551cb62a169ec7162ac7be8d4817d561f60e08eaa485234898414bb5a8a0b4c"},
+]
+
+[package.dependencies]
+certifi = "*"
+h11 = ">=0.13,<0.15"
+
+[package.extras]
+asyncio = ["anyio (>=4.0,<5.0)"]
+http2 = ["h2 (>=3,<5)"]
+socks = ["socksio (==1.*)"]
+trio = ["trio (>=0.22.0,<1.0)"]
+
+[[package]]
+name = "httpx"
+version = "0.27.2"
+description = "The next generation HTTP client."
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "httpx-0.27.2-py3-none-any.whl", hash = "sha256:7bb2708e112d8fdd7829cd4243970f0c223274051cb35ee80c03301ee29a3df0"},
+    {file = "httpx-0.27.2.tar.gz", hash = "sha256:f7c2be1d2f3c3c3160d441802406b206c2b76f5947b11115e6df10c6c65e66c2"},
+]
+
+[package.dependencies]
+anyio = "*"
+certifi = "*"
+httpcore = "==1.*"
+idna = "*"
+sniffio = "*"
+
+[package.extras]
+brotli = ["brotli", "brotlicffi"]
+cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"]
+http2 = ["h2 (>=3,<5)"]
+socks = ["socksio (==1.*)"]
+zstd = ["zstandard (>=0.18.0)"]
+
 [[package]]
 name = "idna"
 version = "3.10"
@@ -991,6 +1070,17 @@ files = [
     {file = "shiboken6-6.8.0.2-cp39-abi3-win_amd64.whl", hash = "sha256:b11e750e696bb565d897e0f5836710edfb86bd355f87b09988bd31b2aad404d3"},
 ]
 
+[[package]]
+name = "sniffio"
+version = "1.3.1"
+description = "Sniff out which async library your code is running under"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2"},
+    {file = "sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc"},
+]
+
 [[package]]
 name = "strip-ansi"
 version = "0.1.1"
@@ -1099,4 +1189,4 @@ type = ["pytest-mypy"]
 [metadata]
 lock-version = "2.0"
 python-versions = ">=3.9,<3.13"
-content-hash = "44356eaeeb3dbe23b922413ee68f8c73c6ce8ebbdee8a80afecb410e060d0382"
+content-hash = "5d1ff28aa04c3a814280e55c0b2a307efe5ca953cd4cb281056c35fd2e53fdf0"

pyproject.toml

@@ -57,6 +57,9 @@ pytest-rerunfailures = "^14.0"
 [tool.poetry.group.container.dependencies]
 pymupdf = "1.24.11" # Last version to support python 3.8 (needed for Ubuntu Focal support)
 
+[tool.poetry.group.dev.dependencies]
+httpx = "^0.27.2"
+
 [tool.isort]
 profile = "black"
 skip_gitignore = true