REMOVE_ME: Bypass group 13 because Dangerzone hangs

.github/workflows/large-tests.yml

@@ -48,8 +48,9 @@ jobs:
       - build-container-image
     strategy:
       matrix:
-        group: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]
+        #group: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]
-
+        #Bypass the 13th group, since a file in it makes Dangerzone hang.
+        group: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 15, 16, 17, 18, 19, 20]
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -83,6 +84,7 @@ jobs:
           poetry run ./dev_scripts/dangerzone-cli tests/test_docs/sample-pdf.pdf
 
       - name: Run large tests
+        continue-on-error: true  # We expect test failures
         run: |-
           export TEST_GROUP_COUNT=20
           export TEST_GROUP=${{ matrix.group }}
@@ -95,5 +97,60 @@ jobs:
       - name: Upload results
         uses: actions/upload-artifact@v4
         with:
-          name: results_${{ matrix.group }}
+          name: results-${{ matrix.group }}
           path: tests/test_docs_large/results/junit
+          if-no-files-found: error
+          retention-days: 1
+
+  report:
+    runs-on: ubuntu-latest
+    needs:
+      - run-large-tests
+    steps:
+      - name: Download results
+        uses: actions/download-artifact@v4
+        with:
+          path: ${{ runner.temp }}/results
+          pattern: results-*
+          merge-multiple: true
+
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Install requirements
+        run: |-
+          sudo apt update -y
+          sudo apt install -y \
+            podman git-lfs libxml2-utils \
+            python3 python3-poetry make
+          poetry install
+
+      - name: Download the large test repo
+        run: |-
+          make test-large-init
+
+      - name: Combine the results
+        run: |-
+          mkdir -p ${{ runner.temp }}/final_results
+          dev_scripts/merge_large_tests_results.py \
+             ${{ runner.temp }}/results \
+             ${{ runner.temp }}/final_results/results.xml
+
+      - name: Generate a report
+        run: |-
+          poetry run python tests/test_docs_large/report.py \
+              ${{ runner.temp }}/final_results/results.xml \
+              | tee ${{ runner.temp }}/final_results/report.txt
+
+      - name: Upload final results and report
+        uses: actions/upload-artifact@v4
+        with:
+          name: final-results
+          path: ${{ runner.temp }}/final_results
+          if-no-files-found: error

dev_scripts/merge_large_tests_results.py

@@ -0,0 +1,76 @@
+#!/usr/bin/env python3
+
+import glob
+import sys
+import xml.etree.ElementTree as ET
+
+
+def combine_xmls(xml_files, output_file):
+    # Initialize accumulators for summary numbers
+    total_errors = 0
+    total_failures = 0
+    total_skipped = 0
+    total_tests = 0
+    total_time = 0.0
+
+    # Create the root element for the output XML
+    testsuites_elem = ET.Element("testsuites")
+
+    # Prepare a base testsuite element that will contain all testcases
+    combined_testsuite = ET.Element("testsuite", name="combined")
+
+    for xml_file in xml_files:
+        print(f"Parsing '{xml_file}'")
+
+        try:
+            tree = ET.parse(xml_file)
+        except ET.ParseError as e:
+            print(f"Error parsing {xml_file}: {e}")
+            continue
+
+        root = tree.getroot()
+        # Assuming structure: <testsuites><testsuite ...>
+        testsuite_elem = root.find("testsuite")
+        if testsuite_elem is None:
+            print(f"No <testsuite> element found in {xml_file}")
+            continue
+
+        # Sum up the attributes from each testsuite.
+        total_errors += int(testsuite_elem.attrib.get("errors", "0"))
+        total_failures += int(testsuite_elem.attrib.get("failures", "0"))
+        total_skipped += int(testsuite_elem.attrib.get("skipped", "0"))
+        total_tests += int(testsuite_elem.attrib.get("tests", "0"))
+        total_time += float(testsuite_elem.attrib.get("time", "0.0"))
+
+        # Move all <testcase> subelements to our combined testsuite.
+        for testcase in testsuite_elem.findall("testcase"):
+            combined_testsuite.append(testcase)
+
+    # Update the attributes of the combined testsuite
+    combined_testsuite.attrib["errors"] = str(total_errors)
+    combined_testsuite.attrib["failures"] = str(total_failures)
+    combined_testsuite.attrib["skipped"] = str(total_skipped)
+    combined_testsuite.attrib["tests"] = str(total_tests)
+    combined_testsuite.attrib["time"] = str(total_time)
+    # Optionally add a timestamp or hostname you want to combine from the originals.
+
+    # Append the combined testsuite to the testsuites root element.
+    testsuites_elem.append(combined_testsuite)
+
+    # Write out the combined XML file.
+    tree_out = ET.ElementTree(testsuites_elem)
+    tree_out.write(output_file, encoding="utf-8", xml_declaration=True)
+    print(f"Combined XML written to {output_file}")
+
+
+if __name__ == "__main__":
+    # For instance, if all XML files are stored in the "xml_results" directory
+    folder = sys.argv[1]
+    output = sys.argv[2]
+    print(
+        f"Will search for XML files in '{folder}' and create a combined XML in"
+        f" '{output}'"
+    )
+    xml_files = glob.glob(f"{folder}/*.xml")
+    print("Found len(list(xml_files)) XML file(s)")
+    combine_xmls(xml_files, output)

tests/test_docs_large

@@ -1 +1 @@
-Subproject commit 1891f586b3b627c1631b7425711c571cdda005d2
+Subproject commit 0faa21eb4e33ec1a3212468dcb6db3a668cf8fc8