Adapt code so it works for reporting script

Reporting script now parses JunitXML instead of a series of
".container_log" files. The script in in changed submodule.

Additionally it makes failed tests actually fail so that this is
recorded in the JunitXML report.

Makefile

@@ -23,7 +23,8 @@ MYPY_ARGS := --ignore-missing-imports \
 			 --disallow-untyped-defs \
 			 --show-error-codes \
 			 --warn-unreachable \
-			 --warn-unused-ignores
+			 --warn-unused-ignores \
+			 --exclude $(LARGE_TEST_REPO_DIR)/*.py
 
 mypy-host:
 	mypy $(MYPY_ARGS) dangerzone
@@ -57,12 +58,13 @@ test-large-init: test-large-requirements
 	@echo "initializing 'test_docs_large' submodule"
 	git submodule init $(LARGE_TEST_REPO_DIR)
 	git submodule update $(LARGE_TEST_REPO_DIR)
-	git lfs pull $(LARGE_TEST_REPO_DIR)
+	cd $(LARGE_TEST_REPO_DIR) && $(MAKE) clone-docs
 
 TEST_LARGE_RESULTS:=$(LARGE_TEST_REPO_DIR)/results/junit/commit_$(GIT_DESC).junit.xml
 .PHONY: tests-large
 test-large: test-large-init  ## Run large test set
 	python -m pytest tests/test_large_set.py::TestLargeSet -v $(JUNIT_FLAGS) --junitxml=$(TEST_LARGE_RESULTS)
+	python $(TEST_LARGE_RESULTS)/report.py $(TEST_LARGE_RESULTS)
 
 # Makefile self-help borrowed from the securedrop-client project
 # Explaination of the below shell command should it ever break.

dangerzone/conversion/doc_to_pixels.py

@@ -388,5 +388,6 @@ async def main() -> int:
             container_log.write(converter.captured_output)
     return error_code
 
+
 if __name__ == "__main__":
     sys.exit(asyncio.run(main()))

dangerzone/isolation_provider/base.py

@@ -11,6 +11,8 @@ from ..util import replace_control_chars
 log = logging.getLogger(__name__)
 
 MAX_CONVERSION_LOG_CHARS = 150 * 50  # up to ~150 lines of 50 characters
+CONVERSION_LOG_START = "-----CONVERSION LOG START-----"
+CONVERSION_LOG_END = "-----CONVERSION LOG END-----"
 
 
 class IsolationProvider(ABC):
@@ -88,8 +90,8 @@ class IsolationProvider(ABC):
         conversion_string = replace_control_chars(untrusted_conversion_str)
 
         # Add armor (gpg-style)
-        armor_start = "-----CONVERSION LOG START-----\n"
-        armor_end = "-----CONVERSION LOG END-----"
+        armor_start = f"{CONVERSION_LOG_START}\n"
+        armor_end = CONVERSION_LOG_END
         return armor_start + conversion_string + armor_end
 
 

tests/test_docs_large

@@ -1 +1 @@
-Subproject commit 4cbf14ac31ac986ced60e83867aac8a6d2d4a81b
+Subproject commit 51419aa36bd4701927d1610fb544e18a4692f1db

tests/test_large_set.py

@@ -10,6 +10,7 @@ from _pytest.fixtures import FixtureRequest
 
 from dangerzone.document import SAFE_EXTENSION
 
+from .test_cli import TestCli
 
 test_docs_repo_dir = Path(__file__).parent / "test_docs_large"
 test_docs_dir = test_docs_repo_dir / "all_documents"
@@ -59,16 +60,27 @@ for_each_100M_doc = pytest.mark.parametrize(
 )
 
 
-
-class TestLargeSet():
+class TestLargeSet(TestCli):
     def run_doc_test(self, doc: Path, tmp_path: Path) -> None:
         output_file_path = str(tmp_path / "output.pdf")
-        p = subprocess.Popen([
-            "python", "dev_scripts/dangerzone-cli", "--output-filename", output_file_path, "--ocr-lang", "eng", str(doc)
-        ], stdout=subprocess.PIPE,stderr=subprocess.STDOUT)
+        p = subprocess.Popen(
+            [
+                "python",
+                "dev_scripts/dangerzone-cli",
+                "--output-filename",
+                output_file_path,
+                "--ocr-lang",
+                "eng",
+                str(doc),
+            ],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+        )
         out, _ = p.communicate()
         from strip_ansi import strip_ansi
+
         print(strip_ansi(out.decode()))
+        assert p.returncode == 0
 
     @for_each_10K_doc
     def test_10K_docs(self, doc: Path, tmp_path: Path) -> None: