tests: test for regressions when converting PDFs when running the tests

This stores a reference version of the converted PDFs and diffs them when
the newly converted document during the tests.

CHANGELOG.md

@@ -10,7 +10,11 @@ since 0.4.1, and this project adheres to [Semantic Versioning](https://semver.or
 - Platform support: Drop support for Ubuntu Focal, since it's nearing end-of-life ([#1018](https://github.com/freedomofpress/dangerzone/issues/1018))
 - Platform support: Add support for Fedora 42 ([#1091](https://github.com/freedomofpress/dangerzone/issues/1091))
 - Platform support: Add support for Ubuntu 25.04 (Plucky Puffin)([#1090](https://github.com/freedomofpress/dangerzone/issues/1090))
+
+### Added
+
 - Document Operating System support [#986](https://github.com/freedomofpress/dangerzone/issues/986)
+- Tests: Look for regressions when converting PDFs [#321](https://github.com/freedomofpress/dangerzone/issues/321)
 
 ## [0.8.1](https://github.com/freedomofpress/dangerzone/compare/v0.8.1...0.8.0)
 

tests/conftest.py

@@ -122,7 +122,7 @@ test_docs_compressed_dir = Path(__file__).parent.joinpath(SAMPLE_COMPRESSED_DIRE
 
 test_docs = [
     p
-    for p in test_docs_dir.rglob("*")
+    for p in test_docs_dir.glob("*")
     if p.is_file()
     and not (p.name.endswith(SAFE_EXTENSION) or p.name.startswith("sample_bad"))
 ]

tests/test_cli.py

@@ -11,6 +11,7 @@ import traceback
 from pathlib import Path
 from typing import Optional, Sequence
 
+import fitz
 import pytest
 from click.testing import CliRunner, Result
 from pytest_mock import MockerFixture
@@ -191,9 +192,26 @@ class TestCliConversion(TestCliBasic):
         result.assert_failure()
 
     @for_each_doc
-    def test_formats(self, doc: Path) -> None:
+    def test_formats(self, doc: Path, tmp_path_factory: pytest.TempPathFactory) -> None:
-        result = self.run_cli(str(doc))
+        reference = (doc.parent / "reference" / doc.stem).with_suffix(".pdf")
+        destination = tmp_path_factory.mktemp(doc.stem).with_suffix(".pdf")
+
+        result = self.run_cli([str(doc), "--output-filename", str(destination)])
         result.assert_success()
+        # When needed, regenerate the reference PDFs by uncommenting the following line:
+        # reference.parent.mkdir(parents=True, exist_ok=True)
+        # shutil.copy(destination, reference)
+
+        converted = fitz.open(destination)
+        ref = fitz.open(reference)
+        assert len(converted) == len(ref), "different number of pages"
+
+        for page, ref_page in zip(converted, ref):
+            page.get_pixmap(dpi=150)
+            ref_page.get_pixmap(dpi=150)
+            assert page.get_pixmap().tobytes() == ref_page.get_pixmap().tobytes(), (
+                f"different page content for page {page.number}"
+            )
 
     def test_output_filename(self, sample_pdf: str) -> None:
         temp_dir = tempfile.mkdtemp(prefix="dangerzone-")