Add a makefile target to regenerate reference PDFs

This leverages a new flag that can be passed during the tests to
regenerate the PDFs if needed.

Makefile

@@ -66,6 +66,9 @@ build-macos-arm: build-clean
 build-linux: build-clean
 	doit -n 8 fedora_rpm debian_deb
 
+.PHONY: regenerate-reference-pdfs
+regenerate-reference-pdfs:
+	pytest tests/test_cli.py -k regenerate --generate-reference-pdfs
 # Makefile self-help borrowed from the securedrop-client project
 # Explaination of the below shell command should it ever break.
 # 1. Set the field separator to ": ##" and any make targets that might appear between : and ##

tests/conftest.py

@@ -160,3 +160,31 @@ def for_each_external_doc(glob_pattern: str = "*") -> Callable:
 
 class TestBase:
     sample_doc = str(test_docs_dir.joinpath(BASIC_SAMPLE_PDF))
+
+
+def pytest_configure(config: pytest.Config) -> None:
+    config.addinivalue_line(
+        "markers",
+        "reference_generator: Used to mark the test cases that regenerate reference documents",
+    )
+
+
+def pytest_addoption(parser: pytest.Parser) -> None:
+    parser.addoption(
+        "--generate-reference-pdfs",
+        action="store_true",
+        default=False,
+        help="Regenerate reference PDFs",
+    )
+
+
+def pytest_collection_modifyitems(
+    config: pytest.Config, items: List[pytest.Item]
+) -> None:
+    if not config.getoption("--generate-reference-pdfs"):
+        skip_generator = pytest.mark.skip(
+            reason="Only run when --generate-reference-pdfs is provided"
+        )
+        for item in items:
+            if "reference_generator" in item.keywords:
+                item.add_marker(skip_generator)

tests/test_cli.py

@@ -193,6 +193,14 @@ class TestCliConversion(TestCliBasic):
         result = self.run_cli([sample_pdf, "--ocr-lang", "piglatin"])
         result.assert_failure()
 
+    @pytest.mark.reference_generator
+    @for_each_doc
+    def test_regenerate_reference(self, doc: Path) -> None:
+        reference = (doc.parent / "reference" / doc.stem).with_suffix(".pdf")
+
+        result = self.run_cli([str(doc), "--output-filename", str(reference)])
+        result.assert_success()
+
     @for_each_doc
     def test_formats(self, doc: Path, tmp_path_factory: pytest.TempPathFactory) -> None:
         reference = (doc.parent / "reference" / doc.stem).with_suffix(".pdf")