QE-14110 improve CI stability (#541)

Fix - improve CI stability (increase workers, add envvar tweaks)
Fix - parse --workers correctly
Fix - Ctrl-C to terminate for multi-workers
Fix - ensure testing webserver is online
Change - use fork when not MacOS for multi-worker
Chore - bump jellyfish to 1.1.3

.circleci/config.yml

@@ -51,9 +51,14 @@ jobs:
         default: "chrome"
     docker:
       - image: cimg/python:3.12
-      - image: selenium/standalone-<<parameters.browser>>:124.0
         environment:
+          # Starting with python 3.12 coverage could be 2x slower so use experimental COVERAGE_CORE=sysmon
+          COVERAGE_CORE: sysmon
+      - image: selenium/standalone-<<parameters.browser>>:125.0
+        environment:
+          SE_ENABLE_TRACING: false
           SE_NODE_MAX_SESSIONS: 12
+          SE_NODE_OVERRIDE_MAX_SESSIONS: true
           SE_NODE_SESSION_TIMEOUT: 300
           SCREEN_WIDTH: 1366
           SCREEN_HEIGHT: 768
@@ -77,7 +82,7 @@ jobs:
           command: curl --retry 60 --retry-delay 5 --retry-connrefused http://localhost:4444
       - run:
           name: run_functional_tests
-          command: uv run cucu run features --workers 6 --selenium-remote-url http://localhost:4444 --generate-report --junit junit_results --browser "<<parameters.browser>>"
+          command: uv run cucu run features --workers 8 --selenium-remote-url http://localhost:4444 --generate-report --junit junit_results --browser "<<parameters.browser>>"
           environment:
             COVERAGE_PROCESS_START: pyproject.toml  # set to config file
       - run:

.github/workflows/build.yml

@@ -70,7 +70,7 @@ jobs:
       pull-requests: write
     services:
       webserver:
-        image: selenium/standalone-${{ matrix.browser }}:124.0
+        image: selenium/standalone-${{ matrix.browser }}:125.0
         ports:
           - 4444:4444
         options: --shm-size=4gb

CHANGELOG.md

@@ -5,6 +5,14 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project closely adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## 1.0.7
+- Fix - stablize CI runs (increase workers, add envvar tweaks)
+- Fix - parse --workers correctly
+- Fix - Ctrl-C to terminate for multi-workers
+- Fix - ensure testing webserver is online
+- Change - use fork when not MacOS for multi-worker
+- Chore - bump jellyfish to 1.1.3
+
 ## 1.0.6
 - Chore - relax required versions for requests library
 

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "cucu"
-version = "1.0.6"
+version = "1.0.7"
 description = "Easy BDD web testing"
 readme = "README.md"
 license = { text = "The Clear BSD License" }
@@ -51,7 +51,7 @@ dependencies = [
     "humanize~=4.8.0",
     "importlib-metadata~=8.0.0",
     "ipdb~=0.13.13",
-    "jellyfish~=1.0.1",
+    "jellyfish~=1.1.3",
     "jinja2~=3.1.3",
     "lsprotocol~=2023.0.1",
     "mpire~=2.10.2",

src/cucu/cli/core.py

@@ -4,6 +4,7 @@
 import os
 import shutil
 import signal
+import sys
 import time
 import xml.etree.ElementTree as ET
 from importlib.metadata import version
@@ -189,6 +190,7 @@ def main():
     "--workers",
     default=None,
     help="Specifies the number of workers to use to run tests in parallel",
+    type=int,
 )
 @click.option(
     "--verbose/--no-verbose",
@@ -338,11 +340,20 @@ def cancel_timer(_):
             if os.path.isdir(filepath):
                 basepath = os.path.join(filepath, "**/*.feature")
                 feature_filepaths = list(glob.iglob(basepath, recursive=True))
-
             else:
                 feature_filepaths = [filepath]
 
-            with WorkerPool(n_jobs=int(workers), start_method="spawn") as pool:
+            if sys.platform == "darwin":
+                logger.info(
+                    "MAC OS detected, using 'forkserver' start method since 'fork' is unstable"
+                )
+                start_method = "forkserver"
+            else:
+                start_method = "fork"
+
+            with WorkerPool(
+                n_jobs=int(workers), start_method=start_method
+            ) as pool:
                 # Each feature file is applied to the pool as an async task.
                 # It then polls the async result of each task. It the result
                 # is ready, it removes the result from the list of results that
@@ -364,6 +375,34 @@ def runtime_exit():
                     timer = Timer(runtime_timeout, runtime_exit)
                     timer.start()
 
+                def kill_workers():
+                    for worker in pool._workers:
+                        try:
+                            worker_proc = psutil.Process(worker.pid)
+                            for child in worker_proc.children():
+                                child.kill()
+
+                            worker_proc.kill()
+                        except psutil.NoSuchProcess:
+                            pass
+
+                def handle_kill_signal(signum, frame):
+                    signal.signal(
+                        signum, signal.SIG_IGN
+                    )  # ignore additional signals
+                    logger.warn(
+                        f"received signal {signum}, sending kill signal to workers"
+                    )
+                    kill_workers()
+                    if timer:
+                        timer.cancel()
+
+                    os.kill(os.getpid(), signal.SIGINT)
+
+                # This is for local runs where you want to cancel the run with a ctrl+c or SIGTERM
+                signal.signal(signal.SIGINT, handle_kill_signal)
+                signal.signal(signal.SIGTERM, handle_kill_signal)
+
                 async_results = {}
                 for feature_filepath in feature_filepaths:
                     async_results[feature_filepath] = pool.apply_async(
@@ -430,15 +469,7 @@ def runtime_exit():
 
                 if timeout_reached:
                     logger.warn("Timeout reached, send kill signal to workers")
-                    for worker in pool._workers:
-                        try:
-                            worker_proc = psutil.Process(worker.pid)
-                            for child in worker_proc.children():
-                                child.kill()
-
-                            worker_proc.kill()
-                        except psutil.NoSuchProcess:
-                            pass
+                    kill_workers()
 
                 task_failed.update(async_results)
 

src/cucu/steps/webserver_steps.py

@@ -1,10 +1,11 @@
+import socket
 from functools import partial
 from http.server import HTTPServer, SimpleHTTPRequestHandler
 from threading import Thread
 
 from behave import step
 
-from cucu import register_after_this_scenario_hook
+from cucu import logger, register_after_this_scenario_hook
 from cucu.config import CONFIG
 
 
@@ -33,6 +34,9 @@ def run_webserver_for_scenario(ctx, directory, variable):
     _, port = httpd.server_address
     CONFIG[variable] = str(port)
 
+    with socket.create_connection(("localhost", port), timeout=5):
+        logger.debug(f"Webserver is running at {port=}port")
+
     def shutdown_webserver(_):
         httpd.shutdown()
         thread.join()