From 84b8212e5d85de7a1daf3d522d4f48402a1cbe00 Mon Sep 17 00:00:00 2001
From: deeplow <deeplower@protonmail.com>
Date: Mon, 12 Dec 2022 15:10:07 +0000
Subject: [PATCH] Fix test instability: pytest in seq. podman<4.3.0

Instability in the automated tests sometimes would sometimes fail when
running "podman images --format {{.ID}}". It turns out that in versions
prior to podman 4.3.0, podman volumes (stored in
~/.local/share/contaiers) would get corrupted when multiple tests were
run in parallel.

The current solution is to wrap the test command to run sequentially in
versions prior to the fix and in parallel for versions after that.

Fixes #217
---
 Makefile                      |  4 +--
 dev_scripts/pytest-wrapper.py | 55 +++++++++++++++++++++++++++++++++++
 2 files changed, 57 insertions(+), 2 deletions(-)
 create mode 100755 dev_scripts/pytest-wrapper.py

diff --git a/Makefile b/Makefile
index 49ae113..534d1e7 100644
--- a/Makefile
+++ b/Makefile
@@ -39,8 +39,8 @@ lint: lint-black lint-isort mypy ## check the code with various linters
 lint-apply: lint-black-apply lint-isort-apply ## apply all the linter's suggestions
 
 .PHONY: test
-test:  ## run tests in parallel
-	pytest -v -n 4 --cov
+test:
+	./dev_scripts/pytest-wrapper.py -v --cov
 
 # Makefile self-help borrowed from the securedrop-client project
 # Explaination of the below shell command should it ever break.
diff --git a/dev_scripts/pytest-wrapper.py b/dev_scripts/pytest-wrapper.py
new file mode 100755
index 0000000..d1fe733
--- /dev/null
+++ b/dev_scripts/pytest-wrapper.py
@@ -0,0 +1,55 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+# PyTest Wrapper script - temporary solution to tests failing non-deterministically [1]
+# This is only fixed in podman v4.3.0. The wrapper essentially runs the tests in sequence
+# when the podman version is lower.
+#
+# [1]: https://github.com/freedomofpress/dangerzone/issues/217
+
+# FIXME this whole script should be removed and the respective Makefile calling line
+# replaced once all supported platforms have a podman version >= v4.3.0.
+
+import re
+import subprocess
+import sys
+
+import pytest
+from pkg_resources import parse_version
+
+from dangerzone.container import get_runtime_name
+
+PODMAN_MIN_VERSION = "4.3.0"
+
+
+def get_podman_version():
+    result = subprocess.run(
+        ["podman", "version", "--format", "'{{.Client.Version}}'"], capture_output=True
+    )
+    version = result.stdout.decode()[:-1]  # trim trailing \n
+    return version.split("-dev")[0]  # exclude "-dev" suffix from version
+
+
+def run_tests_in_parallel(pytest_args):
+    args = pytest_args + ["-n", "4"]
+    exit_code = pytest.main(args)
+
+
+def run_tests_in_sequence(pytest_args):
+    print("running tests sequentially")
+    exit_code = pytest.main(pytest_args)
+
+
+if __name__ == "__main__":
+
+    pytest_args = sys.argv[1:]  # exclude program names
+
+    if get_runtime_name() == "docker":
+        run_tests_in_parallel(pytest_args)
+    else:
+        podman_ver_minimum_parallel = parse_version(PODMAN_MIN_VERSION)
+        podman_ver_current = parse_version(get_podman_version())
+        if podman_ver_current >= podman_ver_minimum_parallel:
+            run_tests_in_parallel(pytest_args)
+        else:
+            run_tests_in_sequence(pytest_args)