Merge pull request mendersoftware#1276 from oleorhagen/flakytests

Flaky PoC

tests/requirements/python-requirements.txt

@@ -60,3 +60,4 @@ urllib3==1.26.4
 webencodings==0.5.1
 websocket-client==0.58.0
 websockets==8.1
+flaky==3.7.0

tests/tests/test_demo_artifact.py

@@ -18,6 +18,7 @@
 import time
 
 import pytest
+from flaky import flaky
 
 from ..common_setup import running_custom_production_setup
 from ..MenderAPI import authentication, deployments, DeviceAuthV2, logger
@@ -102,6 +103,7 @@ def run_demo_script_up(exit_cond=exit_cond):
 
     # Give the test a timeframe, as the script might run forever,
     # if something goes awry, or the script is not brought down properly.
+    @flaky(max_runs=3)  # https://tracker.mender.io/browse/MEN-4495
     @pytest.mark.timeout(3000)
     def test_demo_artifact(self, run_demo_script):
         """Tests that the demo script does indeed upload the demo Artifact to the server."""

tests/tests/test_mtls.py

@@ -19,6 +19,7 @@
 import tempfile
 import time
 import re
+from flaky import flaky
 
 from testutils.common import create_org
 from testutils.infra.container_manager import factory
@@ -236,6 +237,7 @@ def common_test_mtls_enterprise(self, env, algorithm=None, use_hsm=False):
 
     @MenderTesting.fast
     @pytest.mark.parametrize("algorithm", ["rsa", "ec256", "ed25519"])
+    @flaky(max_runs=3)  # https://tracker.mender.io/browse/QA-243
     def test_mtls_enterprise(self, setup_ent_mtls, algorithm):
         self.common_test_mtls_enterprise(setup_ent_mtls, algorithm, use_hsm=False)
 
@@ -267,6 +269,7 @@ def test_mtls_enterprise(self, setup_ent_mtls, algorithm):
 
     @MenderTesting.fast
     @pytest.mark.parametrize("algorithm", ["rsa"])
+    @flaky(max_runs=3)  # https://tracker.mender.io/browse/QA-243
     def test_mtls_enterprise_hsm(self, setup_ent_mtls, algorithm):
 
         # Check if the client has has SoftHSM (from yocto dunfell forward)