Merge pull request mendersoftware#2670 from alfrunes/pytest-parallel-…

…script-fix-v2

More fixes to the pytest parallelization script

tests/ci-parallel-pytest-plugin.py

@@ -16,32 +16,31 @@
 # pytest classes into a Gitlab Parallel matrix.
 
 import os
-import re
 import subprocess
 import sys
 
-INDEX = int(os.environ.get("CI_NODE_INDEX", "1"))
-TOTAL = int(os.environ.get("CI_NODE_TOTAL", "1"))
+if __name__ == "__main__":
+    INDEX = int(os.environ.get("CI_NODE_INDEX", "1"))
+    TOTAL = int(os.environ.get("CI_NODE_TOTAL", "1"))
 
-output = subprocess.check_output(["pytest", "--co", "tests"] + sys.argv[1:])
+    output = subprocess.check_output(["pytest", "--co", "tests", "-q"] + sys.argv[1:])
 
-classes = []
-expr = re.compile("<Class ([^>]+)>")
-for line in output.decode("UTF-8").splitlines():
-    match = expr.search(line)
-    if match:
-        classes.append(match.group(1))
-classes.sort()
+    node_ids = []
+    for line in output.decode("UTF-8").splitlines():
+        if line:
+            node_ids.append(line)
+        else:
+            break
+    node_ids.sort()
 
+    n_batch = len(node_ids) // TOTAL
+    n_rest = len(node_ids) % TOTAL
 
-n_batch = len(classes) // TOTAL
-n_rest = len(classes) % TOTAL
+    offset = n_batch * (INDEX - 1)
+    if INDEX <= n_rest:
+        n_batch += 1
+        offset += INDEX - 1
+    else:
+        offset += n_rest
 
-offset = n_batch * (INDEX - 1)
-if INDEX <= n_rest:
-    n_batch += 1
-    offset += INDEX - 1
-else:
-    offset += n_rest
-
-print(" or ".join(classes[offset : offset + n_batch]), end="")
+    print("\n".join(node_ids[offset : offset + n_batch]), end="")

tests/run.sh

@@ -160,12 +160,12 @@ if [ -n "$K8S" ]; then
     aws eks update-kubeconfig --region $AWS_DEFAULT_REGION --name $AWS_EKS_CLUSTER_NAME --kubeconfig ${HOME}/kubeconfig.${K8S}
 fi
 if test ${CI_NODE_TOTAL:-1} -gt 1; then
-  PYTEST_FILTER="$(python ci-parallel-pytest-plugin.py $@)"
-  if test -z "$PYTEST_FILTER"; then
+  PYTEST_NODES=$(python ci-parallel-pytest-plugin.py | tr '\n' ' ')
+  if test -z "$PYTEST_NODES"; then
     echo "No tests to run for current node"
     exit 0
   fi
-  export PYTEST_ADDOPTS="$PYTEST_ADDOPTS -k '$PYTEST_FILTER'"
+  export PYTEST_ADDOPTS="$PYTEST_ADDOPTS $PYTEST_NODES"
 fi
 python3 -m pytest \
     $EXTRA_TEST_ARGS \