From b5a72255d09d7183d456164a5ef62a4fe74fd694 Mon Sep 17 00:00:00 2001 From: "Paul J. Dorn" Date: Sun, 12 May 2024 23:49:37 +0200 Subject: [PATCH 01/17] fcntl(fd, FD_CLOEXEC) => os.set_inheritable(fd, False) --- gunicorn/util.py | 7 ++++--- gunicorn/workers/base.py | 2 +- gunicorn/workers/sync.py | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/gunicorn/util.py b/gunicorn/util.py index e66dbebf3..70c6c5463 100644 --- a/gunicorn/util.py +++ b/gunicorn/util.py @@ -251,9 +251,10 @@ def parse_address(netloc, default_port='8000'): def close_on_exec(fd): - flags = fcntl.fcntl(fd, fcntl.F_GETFD) - flags |= fcntl.FD_CLOEXEC - fcntl.fcntl(fd, fcntl.F_SETFD, flags) + # available since Python 3.4, equivalent to either: + # ioctl(fd, FIOCLEX) + # fcntl(fd, F_SETFD, fcntl(fd, F_GETFD) | FD_CLOEXEC) + os.set_inheritable(fd, False) def set_non_blocking(fd): diff --git a/gunicorn/workers/base.py b/gunicorn/workers/base.py index 93c465c98..58609297c 100644 --- a/gunicorn/workers/base.py +++ b/gunicorn/workers/base.py @@ -109,7 +109,7 @@ def init_process(self): # Prevent fd inheritance for s in self.sockets: - util.close_on_exec(s) + util.close_on_exec(s.fileno()) util.close_on_exec(self.tmp.fileno()) self.wait_fds = self.sockets + [self.PIPE[0]] diff --git a/gunicorn/workers/sync.py b/gunicorn/workers/sync.py index 4c029f912..705366297 100644 --- a/gunicorn/workers/sync.py +++ b/gunicorn/workers/sync.py @@ -27,7 +27,7 @@ class SyncWorker(base.Worker): def accept(self, listener): client, addr = listener.accept() client.setblocking(1) - util.close_on_exec(client) + util.close_on_exec(client.fileno()) self.handle(listener, client, addr) def wait(self, timeout): From 9dda62eb01f9ae58a64e00ba269424a5c268b2cd Mon Sep 17 00:00:00 2001 From: "Paul J. Dorn" Date: Sun, 12 May 2024 23:54:38 +0200 Subject: [PATCH 02/17] fcntl(fd, O_NONBLOCK) => os.set_blocking(fd, False) --- .github/workflows/tox.yml | 2 +- appveyor.yml | 2 +- gunicorn/util.py | 7 ++++--- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/tox.yml b/.github/workflows/tox.yml index 759800eb1..8fcf95b24 100644 --- a/.github/workflows/tox.yml +++ b/.github/workflows/tox.yml @@ -19,7 +19,7 @@ jobs: - ubuntu-latest # not defaulting to macos-latest: Python <= 3.9 was missing from macos-14 @ arm64 - macos-13 - # Not testing Windows, because tests need Unix-only fcntl, grp, pwd, etc. + # Not testing Windows, because tests need Unix-only non-blocking pipes, grp, pwd, etc. python-version: # CPython <= 3.7 is EoL since 2023-06-27 - "3.7" diff --git a/appveyor.yml b/appveyor.yml index 3cf11f0e9..505193911 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -13,7 +13,7 @@ environment: #- TOXENV: run-entrypoint # PYTHON: "C:\\Python38-x64" # Windows is not ready for testing!!! - # Python's fcntl, grp, pwd, os.geteuid(), and socket.AF_UNIX are all Unix-only. + # Python's non-blocking pipes, grp, pwd, os.geteuid(), and socket.AF_UNIX are all Unix-only. #- TOXENV: py35 # PYTHON: "C:\\Python35-x64" #- TOXENV: py36 diff --git a/gunicorn/util.py b/gunicorn/util.py index 70c6c5463..fdb6a2aa0 100644 --- a/gunicorn/util.py +++ b/gunicorn/util.py @@ -4,7 +4,6 @@ import ast import email.utils import errno -import fcntl import html import importlib import inspect @@ -258,8 +257,10 @@ def close_on_exec(fd): def set_non_blocking(fd): - flags = fcntl.fcntl(fd, fcntl.F_GETFL) | os.O_NONBLOCK - fcntl.fcntl(fd, fcntl.F_SETFL, flags) + # available since Python 3.5, equivalent to either: + # ioctl(fd, FIONBIO) + # fcntl(fd, fcntl.F_SETFL, fcntl(fd, F_GETFL) | O_NONBLOCK) + os.set_blocking(fd, False) def close(sock): From 9d13f7ad466d5aa24770deeb5602bf89c8b3b88f Mon Sep 17 00:00:00 2001 From: "Paul J. Dorn" Date: Fri, 16 Aug 2024 23:13:55 +0200 Subject: [PATCH 03/17] test: setup nginx proxy --- .github/workflows/tox.yml | 5 + tests/test_nginx.py | 457 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 462 insertions(+) create mode 100644 tests/test_nginx.py diff --git a/.github/workflows/tox.yml b/.github/workflows/tox.yml index 8fcf95b24..c4e0ebb0d 100644 --- a/.github/workflows/tox.yml +++ b/.github/workflows/tox.yml @@ -54,6 +54,11 @@ jobs: cache-dependency-path: requirements_test.txt check-latest: true allow-prereleases: ${{ matrix.unsupported }} + - name: Add test utils + if: matrix.os == 'ubuntu-latest' + run: | + sudo systemctl mask nginx.service + sudo apt install nginx openssl - name: Install Dependencies run: | python -m pip install --upgrade pip diff --git a/tests/test_nginx.py b/tests/test_nginx.py new file mode 100644 index 000000000..69742edb7 --- /dev/null +++ b/tests/test_nginx.py @@ -0,0 +1,457 @@ +# +# This file is part of gunicorn released under the MIT license. +# See the NOTICE for more information. + +# hint: can see stdout as the (complex) test progresses using: +# python -B -m pytest -s -vvvv --ff \ +# --override-ini=addopts=--strict-markers --exitfirst \ +# -- tests/test_nginx.py + +import importlib +import os +import secrets +import signal +import subprocess +import sys +import time +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import TYPE_CHECKING +from filelock import FileLock + +import pytest + +if TYPE_CHECKING: + import http.client + from typing import Any, NamedTuple, Self + +CMD_OPENSSL = Path("/usr/bin/openssl") +CMD_NGINX = Path("/usr/sbin/nginx") + +pytestmark = pytest.mark.skipif( + not CMD_OPENSSL.is_file() or not CMD_NGINX.is_file(), + reason="need %s and %s" % (CMD_OPENSSL, CMD_NGINX), +) + +STDOUT = 0 +STDERR = 1 + +TEST_SIMPLE = [ + pytest.param("sync"), + "eventlet", + "gevent", + "gevent_wsgi", + "gevent_pywsgi", + # "tornado", + "gthread", + # "aiohttp.GunicornWebWorker", # different app signature + # "aiohttp.GunicornUVLoopWebWorker", # " +] # type: list[str|NamedTuple] + +WORKER_DEPENDS = { + "aiohttp.GunicornWebWorker": ["aiohttp"], + "aiohttp.GunicornUVLoopWebWorker": ["aiohttp", "uvloop"], + "uvicorn.workers.UvicornWorker": ["uvicorn"], # deprecated + "uvicorn.workers.UvicornH11Worker": ["uvicorn"], # deprecated + "uvicorn_worker.UvicornWorker": ["uvicorn_worker"], + "uvicorn_worker.UvicornH11Worker": ["uvicorn_worker"], + "eventlet": ["eventlet"], + "gevent": ["gevent"], + "gevent_wsgi": ["gevent"], + "gevent_pywsgi": ["gevent"], + "tornado": ["tornado"], +} +DEP_WANTED = set(sum(WORKER_DEPENDS.values(), start=[])) # type: set[str] +DEP_INSTALLED = set() # type: set[str] + +for dependency in DEP_WANTED: + try: + importlib.import_module(dependency) + DEP_INSTALLED.add(dependency) + except ImportError: + pass + +for worker_name, worker_needs in WORKER_DEPENDS.items(): + missing = list(pkg for pkg in worker_needs if pkg not in DEP_INSTALLED) + if missing: + for T in (TEST_SIMPLE,): + if worker_name not in T: + continue + T.remove(worker_name) + skipped_worker = pytest.param( + worker_name, marks=pytest.mark.skip("%s not installed" % (missing[0])) + ) + T.append(skipped_worker) + +WORKER_COUNT = 2 +GRACEFUL_TIMEOUT = 3 +APP_IMPORT_NAME = "testsyntax" +APP_FUNC_NAME = "myapp" +HTTP_HOST = "local.test" + +PY_APPLICATION = f""" +import time +def {APP_FUNC_NAME}(environ, start_response): + body = b"response body from app" + response_head = [ + ("Content-Type", "text/plain"), + ("Content-Length", "%d" % len(body)), + ] + start_response("200 OK", response_head) + time.sleep(0.02) + return iter([body]) +""" + +# used in string.format() - duplicate {{ and }} +NGINX_CONFIG_TEMPLATE = """ +pid {pid_path}; +worker_processes 1; +error_log stderr notice; +events {{ + worker_connections 1024; +}} +worker_shutdown_timeout 1; +http {{ + default_type application/octet-stream; + access_log /dev/stdout combined; + upstream upstream_gunicorn {{ + server {gunicorn_upstream} fail_timeout=0; + }} + + server {{ listen {server_bind} default_server; return 400; }} + server {{ + listen {server_bind}; client_max_body_size 4G; + server_name {server_name}; + root {static_dir}; + location / {{ try_files $uri @proxy_to_app; }} + + location @proxy_to_app {{ + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Host $http_host; + proxy_http_version 1.1; + proxy_redirect off; + proxy_pass {proxy_method}://upstream_gunicorn; + }} + }} +}} +""" + + +class SubProcess: + GRACEFUL_SIGNAL = signal.SIGTERM + + def __enter__(self): + # type: () -> Self + self.run() + return self + + def __exit__(self, *exc): + # type: (*Any) -> None + if self.p is None: + return + self.p.send_signal(signal.SIGKILL) + stdout, stderr = self.p.communicate(timeout=1 + GRACEFUL_TIMEOUT) + ret = self.p.returncode + assert stdout == b"", stdout + assert ret == 0, (ret, stdout, stderr) + + def read_stdio(self, *, key, timeout_sec, wait_for_keyword, expect=None): + # type: (int, int, str, set[str]|None) -> str + # try: + # stdout, stderr = self.p.communicate(timeout=timeout) + # except subprocess.TimeoutExpired: + buf = ["", ""] + seen_keyword = 0 + unseen_keywords = list(expect or []) + poll_per_second = 20 + assert key in {0, 1}, key + assert self.p is not None # this helps static type checkers + assert self.p.stdout is not None # this helps static type checkers + assert self.p.stderr is not None # this helps static type checkers + for _ in range(timeout_sec * poll_per_second): + print("parsing", buf, "waiting for", wait_for_keyword, unseen_keywords) + for fd, file in enumerate([self.p.stdout, self.p.stderr]): + read = file.read(64 * 1024) + if read is not None: + buf[fd] += read.decode("utf-8", "surrogateescape") + if seen_keyword or wait_for_keyword in buf[key]: + seen_keyword += 1 + for additional_keyword in tuple(unseen_keywords): + for somewhere in buf: + if additional_keyword in somewhere: + unseen_keywords.remove(additional_keyword) + # gathered all the context we wanted + if seen_keyword and not unseen_keywords: + break + # not seen expected output? wait for % of original timeout + # .. maybe we will still see better error context that way + if seen_keyword > (0.5 * timeout_sec * poll_per_second): + break + # retcode = self.p.poll() + # if retcode is not None: + # break + time.sleep(1.0 / poll_per_second) + # assert buf[abs(key - 1)] == "" + assert wait_for_keyword in buf[key], (wait_for_keyword, *buf) + assert not unseen_keywords, (unseen_keywords, *buf) + return buf[key] + + def run(self): + # type: () -> None + self.p = subprocess.Popen( + self._argv, + bufsize=0, # allow read to return short + cwd=self.temp_path, + shell=False, + close_fds=True, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + # creationflags=subprocess.CREATE_NEW_PROCESS_GROUP, + ) + os.set_blocking(self.p.stdout.fileno(), False) + os.set_blocking(self.p.stderr.fileno(), False) + assert self.p.stdout is not None # this helps static type checkers + + def graceful_quit(self, expect=None): + # type: (set[str]|None) -> str + if self.p is None: + raise AssertionError("called graceful_quit() when not running") + self.p.send_signal(self.GRACEFUL_SIGNAL) + # self.p.kill() + stdout = self.p.stdout.read(64 * 1024) or b"" + stderr = self.p.stderr.read(64 * 1024) or b"" + try: + o, e = self.p.communicate(timeout=GRACEFUL_TIMEOUT) + stdout += o + stderr += e + except subprocess.TimeoutExpired: + pass + assert stdout == b"" + self.p.stdout.close() + self.p.stderr.close() + exitcode = self.p.poll() # will return None if running + assert exitcode == 0, (exitcode, stdout, stderr) + print("output after signal: ", stdout, stderr, exitcode) + self.p = None + ret = stderr.decode("utf-8", "surrogateescape") + for keyword in expect or (): + assert keyword in ret, (keyword, ret) + return ret + + +class NginxProcess(SubProcess): + GRACEFUL_SIGNAL = signal.SIGQUIT + + def __init__( + self, + *, + temp_path, + config, + ): + assert isinstance(temp_path, Path) + self.conf_path = (temp_path / ("%s.nginx" % APP_IMPORT_NAME)).absolute() + self.p = None # type: subprocess.Popen[bytes] | None + self.temp_path = temp_path + with open(self.conf_path, "w+") as f: + f.write(config) + self._argv = [ + CMD_NGINX, + "-e", + "stderr", + "-c", + "%s" % self.conf_path, + ] + + +def generate_dummy_ssl_cert(cert_path, key_path): + # dummy self-signed cert + subprocess.check_output( + [ + CMD_OPENSSL, + "req", + "-new", + "-newkey", + "ed25519", + "-outform", + "PEM", + "-subj", + "/C=DE", + "-addext", + "subjectAltName=DNS:%s" % (HTTP_HOST), + "-days", + "1", + "-nodes", + "-x509", + "-keyout", + "%s" % (key_path), + "-out", + "%s" % (cert_path), + ], + shell=False, + ) + + +@pytest.fixture(scope="session") +def dummy_ssl_cert(tmp_path_factory): + base_tmp_dir = tmp_path_factory.getbasetemp().parent + crt = base_tmp_dir / "dummy.crt" + key = base_tmp_dir / "dummy.key" + print(crt, key) + # generate once, reuse for all tests + with FileLock("%s.lock" % crt): + if not crt.is_file(): + generate_dummy_ssl_cert(crt, key) + return crt, key + + +class GunicornProcess(SubProcess): + def __init__( + self, + *, + temp_path, + server_bind, + read_size=1024, + ssl_files=None, + worker_class="sync", + ): + self.conf_path = Path(os.devnull) + self.p = None # type: subprocess.Popen[bytes] | None + assert isinstance(temp_path, Path) + self.temp_path = temp_path + self.py_path = (temp_path / ("%s.py" % APP_IMPORT_NAME)).absolute() + with open(self.py_path, "w+") as f: + f.write(PY_APPLICATION) + + ssl_opt = [] + if ssl_files is not None: + cert_path, key_path = ssl_files + ssl_opt = [ + "--do-handshake-on-connect", + "--certfile=%s" % cert_path, + "--keyfile=%s" % key_path, + ] + + self._argv = [ + sys.executable, + "-m", + "gunicorn", + "--config=%s" % self.conf_path, + "--log-level=debug", + "--worker-class=%s" % worker_class, + "--workers=%d" % WORKER_COUNT, + # unsupported at the time this test was submitted + # "--buf-read-size=%d" % read_size, + "--enable-stdio-inheritance", + "--access-logfile=-", + "--disable-redirect-access-to-syslog", + "--graceful-timeout=%d" % (GRACEFUL_TIMEOUT,), + "--bind=%s" % server_bind, + "--reuse-port", + *ssl_opt, + "--", + f"{APP_IMPORT_NAME}:{APP_FUNC_NAME}", + ] + + +class Client: + def __init__(self, host_port): + # type: (str) -> None + self._host_port = host_port + + def __enter__(self): + # type: () -> Self + import http.client + + self.conn = http.client.HTTPConnection(self._host_port, timeout=2) + return self + + def __exit__(self, *exc): + self.conn.close() + + def get(self, path): + # type: () -> http.client.HTTPResponse + self.conn.request("GET", path, headers={"Host": HTTP_HOST}, body="GETBODY!") + return self.conn.getresponse() + + +# @pytest.mark.parametrize("read_size", [50+secrets.randbelow(2048)]) +@pytest.mark.parametrize("ssl", [False, True], ids=["plain", "ssl"]) +@pytest.mark.parametrize("worker_class", TEST_SIMPLE) +def test_nginx_proxy(*, ssl, worker_class, dummy_ssl_cert, read_size=1024): + # avoid ports <= 6144 which may be in use by CI runner + fixed_port = 1024 * 6 + secrets.randbelow(1024 * 9) + # FIXME: should also test inherited socket (LISTEN_FDS) + # FIXME: should also test non-inherited (named) UNIX socket + gunicorn_bind = "[::1]:%d" % fixed_port + + # syntax matches between nginx conf and http client + nginx_bind = "[::1]:%d" % (fixed_port + 1) + + static_dir = "/run/gunicorn/nonexist" + # gunicorn_upstream = "unix:/run/gunicorn/for-nginx.sock" + # syntax "[ipv6]:port" matches between gunicorn and nginx + gunicorn_upstream = gunicorn_bind + + with TemporaryDirectory(suffix="_temp_py") as tempdir_name, Client( + nginx_bind + ) as client: + temp_path = Path(tempdir_name) + nginx_config = NGINX_CONFIG_TEMPLATE.format( + server_bind=nginx_bind, + pid_path="%s" % (temp_path / "nginx.pid"), + gunicorn_upstream=gunicorn_upstream, + server_name=HTTP_HOST, + static_dir=static_dir, + proxy_method="https" if ssl else "http", + ) + + with GunicornProcess( + server_bind=gunicorn_bind, + worker_class=worker_class, + read_size=read_size, + ssl_files=dummy_ssl_cert if ssl else None, + temp_path=temp_path, + ) as server, NginxProcess( + config=nginx_config, + temp_path=temp_path, + ) as proxy: + proxy.read_stdio( + key=STDERR, + timeout_sec=4, + wait_for_keyword="start worker processes", + ) + + server.read_stdio( + key=STDERR, + wait_for_keyword="Arbiter booted", + timeout_sec=4, + expect={ + "Booting worker", + }, + ) + + for num_request in range(5): + path = "/pytest/%d" % (num_request) + response = client.get(path) + assert response.status == 200 + assert response.read() == b"response body from app" + + # using 1.1 to not fail on tornado reporting for 1.0 + # nginx sees our HTTP/1.1 request + proxy.read_stdio( + key=STDOUT, timeout_sec=2, wait_for_keyword="GET %s HTTP/1.1" % path + ) + # gunicorn sees the HTTP/1.1 request from nginx + server.read_stdio( + key=STDOUT, timeout_sec=2, wait_for_keyword="GET %s HTTP/1.1" % path + ) + + server.graceful_quit( + expect={ + "Handling signal: term", + "Shutting down: Master", + }, + ) + proxy.graceful_quit() From 5c9dccde29ae8f9bc557585a855854e23296fa1d Mon Sep 17 00:00:00 2001 From: "Paul J. Dorn" Date: Fri, 23 Aug 2024 23:13:51 +0200 Subject: [PATCH 04/17] CI: workaround OpenBSD / Python 3.10 / ed25510 x509 --- tests/test_nginx.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_nginx.py b/tests/test_nginx.py index 69742edb7..aed05b921 100644 --- a/tests/test_nginx.py +++ b/tests/test_nginx.py @@ -273,7 +273,11 @@ def generate_dummy_ssl_cert(cert_path, key_path): "req", "-new", "-newkey", - "ed25519", + # "ed25519", + # OpenBSD 7.5 / LibreSSL 3.9.0 / Python 3.10.13 + # ssl.SSLError: [SSL: UNKNOWN_CERTIFICATE_TYPE] unknown certificate type (_ssl.c:3900) + # workaround: use RSA keys for testing + "rsa", "-outform", "PEM", "-subj", From 371b39642540478c595f484d53fb2d1ff10f4b5d Mon Sep 17 00:00:00 2001 From: "Paul J. Dorn" Date: Fri, 23 Aug 2024 23:18:10 +0200 Subject: [PATCH 05/17] Py<=3.7: sum() takes no keyword arguments --- tests/test_nginx.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_nginx.py b/tests/test_nginx.py index aed05b921..35fae3254 100644 --- a/tests/test_nginx.py +++ b/tests/test_nginx.py @@ -10,10 +10,12 @@ import importlib import os import secrets +import shutil import signal import subprocess import sys import time +from itertools import chain from pathlib import Path from tempfile import TemporaryDirectory from typing import TYPE_CHECKING @@ -61,7 +63,7 @@ "gevent_pywsgi": ["gevent"], "tornado": ["tornado"], } -DEP_WANTED = set(sum(WORKER_DEPENDS.values(), start=[])) # type: set[str] +DEP_WANTED = set(chain(*WORKER_DEPENDS.values())) # type: set[str] DEP_INSTALLED = set() # type: set[str] for dependency in DEP_WANTED: From 7462bf50a6277dfcba3c05d6556711b2619be1af Mon Sep 17 00:00:00 2001 From: "Paul J. Dorn" Date: Fri, 23 Aug 2024 23:24:43 +0200 Subject: [PATCH 06/17] CI: nginx 1.18 compat --- tests/test_nginx.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_nginx.py b/tests/test_nginx.py index 35fae3254..52cb4ff96 100644 --- a/tests/test_nginx.py +++ b/tests/test_nginx.py @@ -260,8 +260,8 @@ def __init__( f.write(config) self._argv = [ CMD_NGINX, - "-e", - "stderr", + # nginx 1.19.5+ added the -e cmdline flag - may be testing earlier + # "-e", "stderr", "-c", "%s" % self.conf_path, ] From a9fa3d3df82bfc0714d5f731040371f388eb526a Mon Sep 17 00:00:00 2001 From: "Paul J. Dorn" Date: Wed, 27 Nov 2024 22:21:19 +0100 Subject: [PATCH 07/17] CI: nginx timeout & error handling --- tests/test_nginx.py | 66 ++++++++++++++++++++++++++++----------------- 1 file changed, 42 insertions(+), 24 deletions(-) diff --git a/tests/test_nginx.py b/tests/test_nginx.py index 52cb4ff96..e2a80af7a 100644 --- a/tests/test_nginx.py +++ b/tests/test_nginx.py @@ -19,7 +19,6 @@ from pathlib import Path from tempfile import TemporaryDirectory from typing import TYPE_CHECKING -from filelock import FileLock import pytest @@ -27,12 +26,14 @@ import http.client from typing import Any, NamedTuple, Self -CMD_OPENSSL = Path("/usr/bin/openssl") -CMD_NGINX = Path("/usr/sbin/nginx") +# test runner may not be system administrator. not needed here, to run nginx +PATH = "/usr/sbin:/usr/local/sbin:" + os.environ.get("PATH", "/usr/local/bin:/usr/bin") +CMD_OPENSSL = shutil.which("openssl", path=PATH) +CMD_NGINX = shutil.which("nginx", path=PATH) pytestmark = pytest.mark.skipif( - not CMD_OPENSSL.is_file() or not CMD_NGINX.is_file(), - reason="need %s and %s" % (CMD_OPENSSL, CMD_NGINX), + CMD_OPENSSL is None or CMD_NGINX is None, + reason="need nginx and openssl binaries", ) STDOUT = 0 @@ -51,6 +52,8 @@ ] # type: list[str|NamedTuple] WORKER_DEPENDS = { + "sync": [], + "gthread": [], "aiohttp.GunicornWebWorker": ["aiohttp"], "aiohttp.GunicornUVLoopWebWorker": ["aiohttp", "uvloop"], "uvicorn.workers.UvicornWorker": ["uvicorn"], # deprecated @@ -65,6 +68,7 @@ } DEP_WANTED = set(chain(*WORKER_DEPENDS.values())) # type: set[str] DEP_INSTALLED = set() # type: set[str] +WORKER_ORDER = list(WORKER_DEPENDS.keys()) for dependency in DEP_WANTED: try: @@ -86,7 +90,7 @@ T.append(skipped_worker) WORKER_COUNT = 2 -GRACEFUL_TIMEOUT = 3 +GRACEFUL_TIMEOUT = 2 APP_IMPORT_NAME = "testsyntax" APP_FUNC_NAME = "myapp" HTTP_HOST = "local.test" @@ -153,9 +157,9 @@ def __exit__(self, *exc): if self.p is None: return self.p.send_signal(signal.SIGKILL) - stdout, stderr = self.p.communicate(timeout=1 + GRACEFUL_TIMEOUT) + stdout, stderr = self.p.communicate(timeout=2 + GRACEFUL_TIMEOUT) ret = self.p.returncode - assert stdout == b"", stdout + assert stdout[-512:] == b"", stdout assert ret == 0, (ret, stdout, stderr) def read_stdio(self, *, key, timeout_sec, wait_for_keyword, expect=None): @@ -172,11 +176,13 @@ def read_stdio(self, *, key, timeout_sec, wait_for_keyword, expect=None): assert self.p.stdout is not None # this helps static type checkers assert self.p.stderr is not None # this helps static type checkers for _ in range(timeout_sec * poll_per_second): - print("parsing", buf, "waiting for", wait_for_keyword, unseen_keywords) + keep_reading = False + # print(f"parsing {buf!r} waiting for {wait_for_keyword!r} + {unseen_keywords!r}") for fd, file in enumerate([self.p.stdout, self.p.stderr]): read = file.read(64 * 1024) if read is not None: buf[fd] += read.decode("utf-8", "surrogateescape") + keep_reading = True if seen_keyword or wait_for_keyword in buf[key]: seen_keyword += 1 for additional_keyword in tuple(unseen_keywords): @@ -185,7 +191,8 @@ def read_stdio(self, *, key, timeout_sec, wait_for_keyword, expect=None): unseen_keywords.remove(additional_keyword) # gathered all the context we wanted if seen_keyword and not unseen_keywords: - break + if not keep_reading: + break # not seen expected output? wait for % of original timeout # .. maybe we will still see better error context that way if seen_keyword > (0.5 * timeout_sec * poll_per_second): @@ -216,7 +223,7 @@ def run(self): os.set_blocking(self.p.stderr.fileno(), False) assert self.p.stdout is not None # this helps static type checkers - def graceful_quit(self, expect=None): + def graceful_quit(self, expect=None, ignore=None): # type: (set[str]|None) -> str if self.p is None: raise AssertionError("called graceful_quit() when not running") @@ -225,17 +232,21 @@ def graceful_quit(self, expect=None): stdout = self.p.stdout.read(64 * 1024) or b"" stderr = self.p.stderr.read(64 * 1024) or b"" try: - o, e = self.p.communicate(timeout=GRACEFUL_TIMEOUT) + o, e = self.p.communicate(timeout=2 + GRACEFUL_TIMEOUT) stdout += o stderr += e except subprocess.TimeoutExpired: pass - assert stdout == b"" + out = stdout.decode("utf-8", "surrogateescape") + for line in out.split("\n"): + if any(i in line for i in (ignore or ())): + continue + assert line == "" + exitcode = self.p.poll() # will return None if running self.p.stdout.close() self.p.stderr.close() - exitcode = self.p.poll() # will return None if running assert exitcode == 0, (exitcode, stdout, stderr) - print("output after signal: ", stdout, stderr, exitcode) + # print("output after signal: ", stdout, stderr, exitcode) self.p = None ret = stderr.decode("utf-8", "surrogateescape") for keyword in expect or (): @@ -306,9 +317,9 @@ def dummy_ssl_cert(tmp_path_factory): key = base_tmp_dir / "dummy.key" print(crt, key) # generate once, reuse for all tests - with FileLock("%s.lock" % crt): - if not crt.is_file(): - generate_dummy_ssl_cert(crt, key) + # with FileLock("%s.lock" % crt): + if not crt.is_file(): + generate_dummy_ssl_cert(crt, key) return crt, key @@ -339,13 +350,17 @@ def __init__( "--keyfile=%s" % key_path, ] + thread_opt = [] + if worker_class != "sync": + thread_opt = ["--threads=50"] + self._argv = [ sys.executable, "-m", "gunicorn", "--config=%s" % self.conf_path, "--log-level=debug", - "--worker-class=%s" % worker_class, + "--worker-class=%s" % (worker_class,), "--workers=%d" % WORKER_COUNT, # unsupported at the time this test was submitted # "--buf-read-size=%d" % read_size, @@ -355,6 +370,7 @@ def __init__( "--graceful-timeout=%d" % (GRACEFUL_TIMEOUT,), "--bind=%s" % server_bind, "--reuse-port", + *thread_opt, *ssl_opt, "--", f"{APP_IMPORT_NAME}:{APP_FUNC_NAME}", @@ -387,7 +403,9 @@ def get(self, path): @pytest.mark.parametrize("worker_class", TEST_SIMPLE) def test_nginx_proxy(*, ssl, worker_class, dummy_ssl_cert, read_size=1024): # avoid ports <= 6144 which may be in use by CI runner - fixed_port = 1024 * 6 + secrets.randbelow(1024 * 9) + # avoid quickly reusing ports as they might not be cleared immediately on BSD + worker_index = WORKER_ORDER.index(worker_class) + fixed_port = 1024 * 6 + (2 if ssl else 0) + (4 * worker_index) # FIXME: should also test inherited socket (LISTEN_FDS) # FIXME: should also test non-inherited (named) UNIX socket gunicorn_bind = "[::1]:%d" % fixed_port @@ -425,14 +443,14 @@ def test_nginx_proxy(*, ssl, worker_class, dummy_ssl_cert, read_size=1024): ) as proxy: proxy.read_stdio( key=STDERR, - timeout_sec=4, + timeout_sec=8, wait_for_keyword="start worker processes", ) server.read_stdio( key=STDERR, wait_for_keyword="Arbiter booted", - timeout_sec=4, + timeout_sec=8, expect={ "Booting worker", }, @@ -447,11 +465,11 @@ def test_nginx_proxy(*, ssl, worker_class, dummy_ssl_cert, read_size=1024): # using 1.1 to not fail on tornado reporting for 1.0 # nginx sees our HTTP/1.1 request proxy.read_stdio( - key=STDOUT, timeout_sec=2, wait_for_keyword="GET %s HTTP/1.1" % path + key=STDOUT, timeout_sec=4, wait_for_keyword="GET %s HTTP/1.1" % path ) # gunicorn sees the HTTP/1.1 request from nginx server.read_stdio( - key=STDOUT, timeout_sec=2, wait_for_keyword="GET %s HTTP/1.1" % path + key=STDOUT, timeout_sec=4, wait_for_keyword="GET %s HTTP/1.1" % path ) server.graceful_quit( From 29019824aa9e7030ffa50cc34b4c06720e2713da Mon Sep 17 00:00:00 2001 From: "Paul J. Dorn" Date: Wed, 27 Nov 2024 22:21:33 +0100 Subject: [PATCH 08/17] CI: setup wrk benchmark --- tests/test_wrk.py | 432 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 432 insertions(+) create mode 100644 tests/test_wrk.py diff --git a/tests/test_wrk.py b/tests/test_wrk.py new file mode 100644 index 000000000..efa4b5468 --- /dev/null +++ b/tests/test_wrk.py @@ -0,0 +1,432 @@ +# +# This file is part of gunicorn released under the MIT license. +# See the NOTICE for more information. + +# hint: can see stdout as the (complex) test progresses using: +# python -B -m pytest -s -vvvv --ff \ +# --override-ini=addopts=--strict-markers --exitfirst \ +# -- tests/test_nginx.py + +import importlib +import os +import re +import secrets +import shutil +import signal +import subprocess +import sys +import time +from itertools import chain +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import TYPE_CHECKING + +import pytest + +if TYPE_CHECKING: + import http.client + from typing import Any, NamedTuple, Self + +# path may be /usr/local/bin for packages ported from other OS +CMD_OPENSSL = shutil.which("openssl") +CMD_WRK = shutil.which("wrk") + +RATE = re.compile(r"^Requests/sec: *([0-9]+(?:\.[0-9]+)?)$", re.MULTILINE) + +pytestmark = pytest.mark.skipif( + CMD_OPENSSL is None or CMD_WRK is None, + reason="need openssl and wrk binaries", +) + +STDOUT = 0 +STDERR = 1 + +TEST_SIMPLE = [ + pytest.param("sync"), + "eventlet", + "gevent", + "gevent_wsgi", + "gevent_pywsgi", + # "tornado", + "gthread", + # "aiohttp.GunicornWebWorker", # different app signature + # "aiohttp.GunicornUVLoopWebWorker", # " +] # type: list[str|NamedTuple] + +WORKER_DEPENDS = { + "sync": [], + "gthread": [], + "aiohttp.GunicornWebWorker": ["aiohttp"], + "aiohttp.GunicornUVLoopWebWorker": ["aiohttp", "uvloop"], + "uvicorn.workers.UvicornWorker": ["uvicorn"], # deprecated + "uvicorn.workers.UvicornH11Worker": ["uvicorn"], # deprecated + "uvicorn_worker.UvicornWorker": ["uvicorn_worker"], + "uvicorn_worker.UvicornH11Worker": ["uvicorn_worker"], + "eventlet": ["eventlet"], + "gevent": ["gevent"], + "gevent_wsgi": ["gevent"], + "gevent_pywsgi": ["gevent"], + "tornado": ["tornado"], +} + +DEP_WANTED = set(chain(*WORKER_DEPENDS.values())) # type: set[str] +DEP_INSTALLED = set() # type: set[str] +WORKER_ORDER = list(WORKER_DEPENDS.keys()) + +for dependency in DEP_WANTED: + try: + importlib.import_module(dependency) + DEP_INSTALLED.add(dependency) + except ImportError: + pass + +for worker_name, worker_needs in WORKER_DEPENDS.items(): + missing = list(pkg for pkg in worker_needs if pkg not in DEP_INSTALLED) + if missing: + for T in (TEST_SIMPLE,): + if worker_name not in T: + continue + T.remove(worker_name) + skipped_worker = pytest.param( + worker_name, marks=pytest.mark.skip("%s not installed" % (missing[0])) + ) + T.append(skipped_worker) + +WORKER_COUNT = 2 +GRACEFUL_TIMEOUT = 2 +APP_IMPORT_NAME = "testsyntax" +APP_FUNC_NAME = "myapp" +HTTP_HOST = "local.test" + +PY_APPLICATION = f""" +import time +def {APP_FUNC_NAME}(environ, start_response): + body = b"response body from app" + response_head = [ + ("Content-Type", "text/plain"), + ("Content-Length", "%d" % len(body)), + ] + start_response("200 OK", response_head) + time.sleep(0.02) + return iter([body]) +""" + + +class SubProcess: + GRACEFUL_SIGNAL = signal.SIGTERM + + def __enter__(self): + # type: () -> Self + self.run() + return self + + def __exit__(self, *exc): + # type: (*Any) -> None + if self.p is None: + return + self.p.send_signal(signal.SIGKILL) + stdout, stderr = self.p.communicate(timeout=2 + GRACEFUL_TIMEOUT) + ret = self.p.returncode + assert stdout[-512:] == b"", stdout + assert ret == 0, (ret, stdout, stderr) + + def read_stdio(self, *, key, timeout_sec, wait_for_keyword, expect=None): + # type: (int, int, str, set[str]|None) -> str + # try: + # stdout, stderr = self.p.communicate(timeout=timeout) + # except subprocess.TimeoutExpired: + buf = ["", ""] + seen_keyword = 0 + unseen_keywords = list(expect or []) + poll_per_second = 20 + assert key in {0, 1}, key + assert self.p is not None # this helps static type checkers + assert self.p.stdout is not None # this helps static type checkers + assert self.p.stderr is not None # this helps static type checkers + for _ in range(timeout_sec * poll_per_second): + keep_reading = False + # print(f"parsing {buf!r} waiting for {wait_for_keyword!r} + {unseen_keywords!r}") + for fd, file in enumerate([self.p.stdout, self.p.stderr]): + read = file.read(64 * 1024) + if read is not None: + buf[fd] += read.decode("utf-8", "surrogateescape") + keep_reading = True + if seen_keyword or wait_for_keyword in buf[key]: + seen_keyword += 1 + for additional_keyword in tuple(unseen_keywords): + for somewhere in buf: + if additional_keyword in somewhere: + unseen_keywords.remove(additional_keyword) + # gathered all the context we wanted + if seen_keyword and not unseen_keywords: + if not keep_reading: + break + # not seen expected output? wait for % of original timeout + # .. maybe we will still see better error context that way + if seen_keyword > (0.5 * timeout_sec * poll_per_second): + break + # retcode = self.p.poll() + # if retcode is not None: + # break + time.sleep(1.0 / poll_per_second) + # assert buf[abs(key - 1)] == "" + assert wait_for_keyword in buf[key], (wait_for_keyword, *buf) + assert not unseen_keywords, (unseen_keywords, *buf) + return buf[key] + + def run(self): + # type: () -> None + self.p = subprocess.Popen( + self._argv, + bufsize=0, # allow read to return short + cwd=self.temp_path, + shell=False, + close_fds=True, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + # creationflags=subprocess.CREATE_NEW_PROCESS_GROUP, + ) + os.set_blocking(self.p.stdout.fileno(), False) + os.set_blocking(self.p.stderr.fileno(), False) + assert self.p.stdout is not None # this helps static type checkers + + def graceful_quit(self, expect=None, ignore=None): + # type: (set[str]|None) -> str + if self.p is None: + raise AssertionError("called graceful_quit() when not running") + self.p.send_signal(self.GRACEFUL_SIGNAL) + # self.p.kill() + stdout = self.p.stdout.read(64 * 1024) or b"" + stderr = self.p.stderr.read(64 * 1024) or b"" + try: + o, e = self.p.communicate(timeout=2 + GRACEFUL_TIMEOUT) + stdout += o + stderr += e + except subprocess.TimeoutExpired: + pass + out = stdout.decode("utf-8", "surrogateescape") + for line in out.split("\n"): + if any(i in line for i in (ignore or ())): + continue + assert line == "" + exitcode = self.p.poll() # will return None if running + self.p.stdout.close() + self.p.stderr.close() + assert exitcode == 0, (exitcode, stdout, stderr) + # print("output after signal: ", stdout, stderr, exitcode) + self.p = None + ret = stderr.decode("utf-8", "surrogateescape") + for keyword in expect or (): + assert keyword in ret, (keyword, ret) + return ret + + +def generate_dummy_ssl_cert(cert_path, key_path): + # dummy self-signed cert + subprocess.check_output( + [ + CMD_OPENSSL, + "req", + "-new", + "-newkey", + # "ed25519", + # OpenBSD 7.5 / LibreSSL 3.9.0 / Python 3.10.13 + # ssl.SSLError: [SSL: UNKNOWN_CERTIFICATE_TYPE] unknown certificate type (_ssl.c:3900) + # workaround: use RSA keys for testing + "rsa", + "-outform", + "PEM", + "-subj", + "/C=DE", + "-addext", + "subjectAltName=DNS:%s" % (HTTP_HOST), + "-days", + "1", + "-nodes", + "-x509", + "-keyout", + "%s" % (key_path), + "-out", + "%s" % (cert_path), + ], + shell=False, + ) + + +@pytest.fixture(scope="session") +def dummy_ssl_cert(tmp_path_factory): + base_tmp_dir = tmp_path_factory.getbasetemp().parent + crt = base_tmp_dir / "dummy.crt" + key = base_tmp_dir / "dummy.key" + print(crt, key) + # generate once, reuse for all tests + # with FileLock("%s.lock" % crt): + if not crt.is_file(): + generate_dummy_ssl_cert(crt, key) + return crt, key + + +class GunicornProcess(SubProcess): + def __init__( + self, + *, + temp_path, + server_bind, + read_size=1024, + ssl_files=None, + worker_class="sync", + ): + self.conf_path = Path(os.devnull) + self.p = None # type: subprocess.Popen[bytes] | None + assert isinstance(temp_path, Path) + self.temp_path = temp_path + self.py_path = (temp_path / ("%s.py" % APP_IMPORT_NAME)).absolute() + with open(self.py_path, "w+") as f: + f.write(PY_APPLICATION) + + ssl_opt = [] + if ssl_files is not None: + cert_path, key_path = ssl_files + ssl_opt = [ + "--do-handshake-on-connect", + "--certfile=%s" % cert_path, + "--keyfile=%s" % key_path, + ] + thread_opt = [] + if worker_class != "sync": + thread_opt = ["--threads=50"] + + self._argv = [ + sys.executable, + "-m", + "gunicorn", + "--config=%s" % self.conf_path, + "--log-level=info", + "--worker-class=%s" % (worker_class,), + "--workers=%d" % WORKER_COUNT, + # unsupported at the time this test was submitted + # "--buf-read-size=%d" % read_size, + "--enable-stdio-inheritance", + "--access-logfile=-", + "--disable-redirect-access-to-syslog", + "--graceful-timeout=%d" % (GRACEFUL_TIMEOUT,), + "--bind=%s" % server_bind, + "--reuse-port", + *thread_opt, + *ssl_opt, + "--", + f"{APP_IMPORT_NAME}:{APP_FUNC_NAME}", + ] + + +class Client: + def __init__(self, url_base): + # type: (str) -> None + self._url_base = url_base + self._env = os.environ.copy() + self._env["LC_ALL"] = "C" + + def __enter__(self): + # type: () -> Self + return self + + def __exit__(self, *exc): + pass + + def get(self, path): + # type: () -> http.client.HTTPResponse + assert path.startswith("/") + threads = 10 + connections = 100 + out = subprocess.check_output( + [ + CMD_WRK, + "-t", + "%d" % threads, + "-c", + "%d" % connections, + "-d5s", + "%s%s" + % ( + self._url_base, + path, + ), + ], + shell=False, + env=self._env, + ) + + return out.decode("utf-8", "replace") + + +# @pytest.mark.parametrize("read_size", [50+secrets.randbelow(2048)]) +@pytest.mark.parametrize("ssl", [False, True], ids=["plain", "ssl"]) +@pytest.mark.parametrize("worker_class", TEST_SIMPLE) +def test_wrk(*, ssl, worker_class, dummy_ssl_cert, read_size=1024): + if worker_class == "eventlet" and ssl: + pytest.skip("eventlet worker does not catch errors in ssl.wrap_socket") + + # avoid ports <= 6144 which may be in use by CI runne + worker_index = WORKER_ORDER.index(worker_class) + fixed_port = 1024 * 6 + 1024 + (2 if ssl else 0) + (4 * worker_index) + # FIXME: should also test inherited socket (LISTEN_FDS) + # FIXME: should also test non-inherited (named) UNIX socket + gunicorn_bind = "[::1]:%d" % fixed_port + + proxy_method = "https" if ssl else "http" + + with TemporaryDirectory(suffix="_temp_py") as tempdir_name, Client( + proxy_method + "://" + gunicorn_bind + ) as client: + temp_path = Path(tempdir_name) + + with GunicornProcess( + server_bind=gunicorn_bind, + worker_class=worker_class, + read_size=read_size, + ssl_files=dummy_ssl_cert if ssl else None, + temp_path=temp_path, + ) as server: + server.read_stdio( + key=STDERR, + wait_for_keyword="[INFO] Starting gunicorn", + timeout_sec=6, + expect={ + "[INFO] Booting worker", + }, + ) + + path = "/pytest/basic" + out = client.get(path) + print("##############\n" + out) + + extract = RATE.search(out) + assert extract is not None, out + rate = float(extract.groups()[0]) + if worker_class == "sync": + assert rate > 5 + else: + assert rate > 50 + + server.read_stdio( + key=STDOUT, timeout_sec=2, wait_for_keyword="GET %s HTTP/1.1" % path + ) + if ssl: + pass + # server.read_stdio( + # key=STDERR, + # wait_for_keyword="[DEBUG] ssl connection closed", + # timeout_sec=4, + # ) + + server.graceful_quit( + ignore={ + "GET %s HTTP/1.1" % path, + "Ignoring connection epipe", + "Ignoring connection reset", + }, + expect={ + "[INFO] Handling signal: term", + }, + ) From 7660f78c5b8a27d678740176aff19ac850307ae4 Mon Sep 17 00:00:00 2001 From: "Paul J. Dorn" Date: Wed, 27 Nov 2024 22:22:25 +0100 Subject: [PATCH 09/17] CI: limit tox to master+PR reduces duplicate workflow runs in forks --- .github/workflows/tox.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tox.yml b/.github/workflows/tox.yml index c4e0ebb0d..7ba184787 100644 --- a/.github/workflows/tox.yml +++ b/.github/workflows/tox.yml @@ -1,5 +1,11 @@ name: tox -on: [push, pull_request] +on: + push: + branches: + - master + pull_request: + branches: + - master permissions: contents: read # to fetch code (actions/checkout) env: From 5aa6eda09881092312260ab4692fcac5ce80c4a7 Mon Sep 17 00:00:00 2001 From: "Paul J. Dorn" Date: Wed, 27 Nov 2024 22:23:24 +0100 Subject: [PATCH 10/17] CI: test OpenBSD + FreeBSD + illumos in Linux VM using https://github.com/vmactions workaround greenlet installation on OmniOS v11: libev assumes inotify.h must be Linux. make autoconf stop offering it. --- .github/workflows/bsd.yml | 70 +++++++++++++++++++++++++++++++++++ .github/workflows/illumos.yml | 56 ++++++++++++++++++++++++++++ 2 files changed, 126 insertions(+) create mode 100644 .github/workflows/bsd.yml create mode 100644 .github/workflows/illumos.yml diff --git a/.github/workflows/bsd.yml b/.github/workflows/bsd.yml new file mode 100644 index 000000000..963c55dd4 --- /dev/null +++ b/.github/workflows/bsd.yml @@ -0,0 +1,70 @@ +name: bsd +on: + push: + branches: + - master + paths: + - '*.py' + - 'tox.ini' + - '.github/workflows/bsd.yml' + pull_request: + branches: + - master + workflow_dispatch: + # allow manual trigger +permissions: + # BOLD WARNING: do not add permissions, this workflow executes remote code + contents: read +env: + FORCE_COLOR: 1 +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true +jobs: + freebsd: + name: freebsd + timeout-minutes: 20 + runs-on: ubuntu-latest + strategy: + fail-fast: true + steps: + - uses: actions/checkout@v4 + - uses: vmactions/freebsd-vm@v1 + with: + # TODO: replace pyMN with version-agnostic alias + prepare: pkg install -y nginx python311 py311-pip py311-tox py311-sqlite3 + usesh: true + copyback: false + # not a typo: "openssl --version" != "openssl version" + run: | + uname -a \ + && python3.11 --version \ + && python3.11 -m tox --version \ + && openssl version \ + && pkg info nginx \ + && python3.11 -m tox -e run-module \ + && python3.11 -m tox -e run-entrypoint \ + && python3.11 -m tox -e py + + openbsd: + name: openbsd + timeout-minutes: 20 + runs-on: ubuntu-latest + strategy: + fail-fast: true + steps: + - uses: actions/checkout@v4 + - uses: vmactions/openbsd-vm@v1 + with: + prepare: pkg_add python py3-pip py3-tox py3-sqlite3 nginx + usesh: true + copyback: false + run: | + uname -a \ + && python3 --version \ + && python3 -m tox --version \ + && openssl version \ + && pkg_info nginx \ + && python3 -m tox -e run-module \ + && python3 -m tox -e run-entrypoint \ + && python3 -m tox -e py diff --git a/.github/workflows/illumos.yml b/.github/workflows/illumos.yml new file mode 100644 index 000000000..2bd853440 --- /dev/null +++ b/.github/workflows/illumos.yml @@ -0,0 +1,56 @@ +name: illumos +on: + push: + branches: + - master + paths: + - '*.py' + - 'tox.ini' + - '.github/workflows/illumos.yml' + pull_request: + branches: + - master + workflow_dispatch: + # allow manual trigger +permissions: + # BOLD WARNING: do not add permissions, this workflow executes remote code + contents: read +env: + FORCE_COLOR: 1 +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true +jobs: + omnios: + name: illumos + timeout-minutes: 20 + runs-on: ubuntu-latest + strategy: + fail-fast: true + steps: + - uses: actions/checkout@v4 + - uses: vmactions/omnios-vm@v1 + with: + # need gcc: compile greenlet from source + # autoconf must pretend inotify unavail: libev FTBFS + # /tmp/.nginx must exist because nginx will not create configured tmp + # build-essential shall point to suitable gcc13/gcc14/.. + # TODO: replace python-MN with version-agnostic alias + prepare: | + pkg install pip-312 python-312 sqlite-3 nginx build-essential + usesh: true + copyback: false + run: | + cat /etc/release \ + && uname -a \ + && python3 --version \ + && openssl version \ + && pkg info nginx \ + && gcc -dM -E - Date: Wed, 27 Nov 2024 22:36:14 +0100 Subject: [PATCH 11/17] CI: deconflict fixed test ports --- tests/test_nginx.py | 4 ++-- tests/test_wrk.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_nginx.py b/tests/test_nginx.py index e2a80af7a..bcdea02f8 100644 --- a/tests/test_nginx.py +++ b/tests/test_nginx.py @@ -402,10 +402,10 @@ def get(self, path): @pytest.mark.parametrize("ssl", [False, True], ids=["plain", "ssl"]) @pytest.mark.parametrize("worker_class", TEST_SIMPLE) def test_nginx_proxy(*, ssl, worker_class, dummy_ssl_cert, read_size=1024): - # avoid ports <= 6144 which may be in use by CI runner + # avoid ports <= 6178 which may be in use by CI runner # avoid quickly reusing ports as they might not be cleared immediately on BSD worker_index = WORKER_ORDER.index(worker_class) - fixed_port = 1024 * 6 + (2 if ssl else 0) + (4 * worker_index) + fixed_port = 6178 + 512 + (2 if ssl else 0) + (4 * worker_index) # FIXME: should also test inherited socket (LISTEN_FDS) # FIXME: should also test non-inherited (named) UNIX socket gunicorn_bind = "[::1]:%d" % fixed_port diff --git a/tests/test_wrk.py b/tests/test_wrk.py index efa4b5468..08ccf71d9 100644 --- a/tests/test_wrk.py +++ b/tests/test_wrk.py @@ -367,9 +367,9 @@ def test_wrk(*, ssl, worker_class, dummy_ssl_cert, read_size=1024): if worker_class == "eventlet" and ssl: pytest.skip("eventlet worker does not catch errors in ssl.wrap_socket") - # avoid ports <= 6144 which may be in use by CI runne + # avoid ports <= 6178 which may be in use by CI runne worker_index = WORKER_ORDER.index(worker_class) - fixed_port = 1024 * 6 + 1024 + (2 if ssl else 0) + (4 * worker_index) + fixed_port = 6178 + 1024 + (2 if ssl else 0) + (4 * worker_index) # FIXME: should also test inherited socket (LISTEN_FDS) # FIXME: should also test non-inherited (named) UNIX socket gunicorn_bind = "[::1]:%d" % fixed_port From 6258262e8eea4b9f24c0b56997398d3e5b73e433 Mon Sep 17 00:00:00 2001 From: "Paul J. Dorn" Date: Thu, 28 Nov 2024 21:34:17 +0100 Subject: [PATCH 12/17] CI: deduplicate tests: nginx wrk gunicorn invocation now matches between the two similar tests nginx needs worker=off to shut down when signaled --- tests/support_subprocess.py | 481 ++++++++++++++++++++++++++++++++++++ tests/test_nginx.py | 416 ++----------------------------- tests/test_wrk.py | 373 ++-------------------------- 3 files changed, 520 insertions(+), 750 deletions(-) create mode 100644 tests/support_subprocess.py diff --git a/tests/support_subprocess.py b/tests/support_subprocess.py new file mode 100644 index 000000000..144795481 --- /dev/null +++ b/tests/support_subprocess.py @@ -0,0 +1,481 @@ +import importlib +import logging +import os +import re +import secrets +import shutil +import signal +import subprocess +import sys +import time +from itertools import chain +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import TYPE_CHECKING + +import pytest + +if TYPE_CHECKING: + import http.client + from typing import Any, NamedTuple, Self + + +logger = logging.getLogger(__name__) + +# note: BSD path may be /usr/local/bin for ported packages +CMD_OPENSSL = shutil.which("openssl") +CMD_WRK = shutil.which("wrk") + +STDOUT = 0 +STDERR = 1 +WORKER_COUNT = 2 +# shared between gunicorn and nginx proxy +GRACEFUL_TIMEOUT = 1 +APP_IMPORT_NAME = "testsyntax" +APP_FUNC_NAME = "myapp" +HTTP_HOST = "local.test" + + +PY_APPLICATION = f""" +import time +def {APP_FUNC_NAME}(environ, start_response): + body = b"response body from app" + response_head = [ + ("Content-Type", "text/plain"), + ("Content-Length", "%d" % len(body)), + ] + start_response("200 OK", response_head) + time.sleep(0.02) + return iter([body]) +""" + +# used in string.format() - duplicate {{ and }} +NGINX_CONFIG_TEMPLATE = """ +pid {pid_path}; +daemon off; +worker_processes 1; +error_log stderr notice; +events {{ + worker_connections 1024; +}} +worker_shutdown_timeout {graceful_timeout}; +http {{ + default_type application/octet-stream; + access_log /dev/stdout combined; + upstream upstream_gunicorn {{ + server {gunicorn_upstream} fail_timeout=0; + }} + + server {{ listen {server_bind} default_server; return 400; }} + server {{ + listen {server_bind}; client_max_body_size 4G; + server_name {server_name}; + root {static_dir}; + location / {{ try_files $uri @proxy_to_app; }} + + location @proxy_to_app {{ + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Host $http_host; + proxy_http_version 1.1; + proxy_redirect off; + proxy_pass {proxy_method}://upstream_gunicorn; + }} + }} +}} +""" + +WORKER_PYTEST_LIST = [ + pytest.param("sync"), + "eventlet", + "gevent", + "gevent_wsgi", + "gevent_pywsgi", + # "tornado", + "gthread", + # "aiohttp.GunicornWebWorker", # different app signature + # "aiohttp.GunicornUVLoopWebWorker", # " +] # type: list[str|NamedTuple] + +WORKER_DEPENDS = { + "sync": [], + "gthread": [], + "aiohttp.GunicornWebWorker": ["aiohttp"], + "aiohttp.GunicornUVLoopWebWorker": ["aiohttp", "uvloop"], + "uvicorn.workers.UvicornWorker": ["uvicorn"], # deprecated + "uvicorn.workers.UvicornH11Worker": ["uvicorn"], # deprecated + "uvicorn_worker.UvicornWorker": ["uvicorn_worker"], + "uvicorn_worker.UvicornH11Worker": ["uvicorn_worker"], + "eventlet": ["eventlet"], + "gevent": ["gevent"], + "gevent_wsgi": ["gevent"], + "gevent_pywsgi": ["gevent"], + "tornado": ["tornado"], +} +DEP_WANTED = set(chain(*WORKER_DEPENDS.values())) # type: set[str] +DEP_INSTALLED = set() # type: set[str] +WORKER_ORDER = list(WORKER_DEPENDS.keys()) + +for dependency in DEP_WANTED: + try: + importlib.import_module(dependency) + DEP_INSTALLED.add(dependency) + except ImportError: + pass + +for worker_name, worker_needs in WORKER_DEPENDS.items(): + missing = list(pkg for pkg in worker_needs if pkg not in DEP_INSTALLED) + if missing: + for T in (WORKER_PYTEST_LIST,): + if worker_name not in T: + continue + T.remove(worker_name) + skipped_worker = pytest.param( + worker_name, marks=pytest.mark.skip("%s not installed" % (missing[0])) + ) + T.append(skipped_worker) + + +class SubProcess(subprocess.Popen): + GRACEFUL_SIGNAL = signal.SIGQUIT + EXIT_SIGNAL = signal.SIGINT + + def __exit__(self, *exc): + # type: (*Any) -> None + if self.returncode is None: + self.send_signal(self.EXIT_SIGNAL) + try: + stdout, stderr = self.communicate(timeout=1) + # assert stdout[-512:] == b"", stdout + logger.debug(f"stdout not empty on shutdown, sample: {stdout[-512:]!r}") + except subprocess.TimeoutExpired: + pass + # only helpful for diagnostics. we are shutting down unexpected + # assert self.returncode == 0, (ret, stdout, stderr) + logger.debug(f"exit code {self.returncode}") + if self.returncode is None: + self.kill() # no need to wait, Popen.__exit__ does that + super().__exit__(*exc) + + def read_stdio(self, *, timeout_sec, wait_for_keyword, expect=None, stderr=False): + # type: (int, int, str, set[str]|None) -> str + # try: + # stdout, stderr = self.communicate(timeout=timeout) + # except subprocess.TimeoutExpired: + key = STDERR if stderr else STDOUT + buf = ["", ""] + seen_keyword = 0 + unseen_keywords = list(expect or []) + poll_per_second = 20 + assert key in {0, 1}, key + assert self.stdout is not None # this helps static type checkers + assert self.stderr is not None # this helps static type checkers + for _ in range(timeout_sec * poll_per_second): + keep_reading = False + logger.debug( + f"parsing {buf!r} waiting for {wait_for_keyword!r} + {unseen_keywords!r}" + ) + for fd, file in enumerate([self.stdout, self.stderr]): + read = file.read(64 * 1024) + if read is not None: + buf[fd] += read.decode("utf-8", "surrogateescape") + keep_reading = True + if seen_keyword or wait_for_keyword in buf[key]: + seen_keyword += 1 + for additional_keyword in tuple(unseen_keywords): + for somewhere in buf: + if additional_keyword in somewhere: + unseen_keywords.remove(additional_keyword) + # gathered all the context we wanted + if seen_keyword and not unseen_keywords: + if not keep_reading: + break + # not seen expected output? wait for % of original timeout + # .. maybe we will still see better error context that way + if seen_keyword > (0.5 * timeout_sec * poll_per_second): + break + # retcode = self.poll() + # if retcode is not None: + # break + time.sleep(1.0 / poll_per_second) + # assert buf[abs(key - 1)] == "" + assert wait_for_keyword in buf[key], (wait_for_keyword, *buf) + assert not unseen_keywords, (unseen_keywords, *buf) + return buf[key] + + def __init__(self): + # type: () -> None + super().__init__( + self._argv, + bufsize=0, # allow read to return short + cwd=self.temp_path, + shell=False, + close_fds=True, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + # creationflags=subprocess.CREATE_NEW_PROCESS_GROUP, + ) + os.set_blocking(self.stdout.fileno(), False) + os.set_blocking(self.stderr.fileno(), False) + assert self.stdout is not None # this helps static type checkers + + def graceful_quit(self, expect=None, ignore=None): + # type: (set[str]|None) -> str + stdout = self.stdout.read(64 * 1024) or b"" + stderr = self.stderr.read(64 * 1024) or b"" + if self.returncode is None: + self.send_signal(self.GRACEFUL_SIGNAL) + try: + o, e = self.communicate(timeout=2 + GRACEFUL_TIMEOUT) + stdout += o + stderr += e + except subprocess.TimeoutExpired: + pass + out = stdout.decode("utf-8", "surrogateescape") + for line in out.split("\n"): + if any(i in line for i in (ignore or ())): + continue + assert line == "" + assert self.stdin is None + # no need to crash still running here, Popen.__exit__ will close + # self.stdout.close() + # self.stderr.close() + exitcode = self.poll() # will return None if running + assert exitcode == 0, (self._argv[0], exitcode, stdout, stderr) + logger.debug("output after signal: ", stdout, stderr, exitcode) + ret = stderr.decode("utf-8", "surrogateescape") + for keyword in expect or (): + assert keyword in ret, (keyword, ret) + return ret + + +class NginxProcess(SubProcess): + # SIGQUIT = drain, SIGTERM = fast shutdown + GRACEFUL_SIGNAL = signal.SIGQUIT + EXIT_SIGNAL = signal.SIGTERM + + # test runner may not be system administrator, with PATH lacking /sbin/ + # .. since we know we do not need root for our tests, disregard that + __default = "/usr/local/bin:/usr/bin" + _PATH = os.environ.get("PATH", __default) + ":/usr/sbin:/usr/local/sbin" + CMD_NGINX = shutil.which("nginx", path=_PATH) + + @classmethod + def gen_config(cls, *, bind, temp_path, upstream, static_dir, ssl): + return NGINX_CONFIG_TEMPLATE.format( + server_bind=bind, + pid_path="%s" % (temp_path / "nginx.pid"), + gunicorn_upstream=upstream, + server_name=HTTP_HOST, + static_dir=static_dir, + graceful_timeout=GRACEFUL_TIMEOUT, + proxy_method="https" if ssl else "http", + ) + + @classmethod + def pytest_supported(cls): + return pytest.mark.skipif( + CMD_OPENSSL is None or cls.CMD_NGINX is None, + reason="need nginx and openssl binaries", + ) + + def __init__( + self, + *, + temp_path, + config, + ): + assert isinstance(temp_path, Path) + self.conf_path = (temp_path / ("%s.nginx" % APP_IMPORT_NAME)).absolute() + self.temp_path = temp_path + with open(self.conf_path, "w+") as f: + f.write(config) + self._argv = [ + self.CMD_NGINX, + # nginx 1.19.5+ added the -e cmdline flag - may be testing earlier + # "-e", "stderr", + "-c", + "%s" % self.conf_path, + ] + super().__init__() + + +def generate_dummy_ssl_cert(cert_path, key_path): + # dummy self-signed cert + subprocess.check_output( + [ + CMD_OPENSSL, + "req", + "-new", + "-newkey", + # "ed25519", + # OpenBSD 7.5 / LibreSSL 3.9.0 / Python 3.10.13 + # ssl.SSLError: [SSL: UNKNOWN_CERTIFICATE_TYPE] unknown certificate type (_ssl.c:3900) + # workaround: use RSA keys for testing + "rsa", + "-outform", + "PEM", + "-subj", + "/C=DE", + "-addext", + "subjectAltName=DNS:%s" % (HTTP_HOST), + "-days", + "1", + "-nodes", + "-x509", + "-keyout", + "%s" % (key_path), + "-out", + "%s" % (cert_path), + ], + shell=False, + ) + + +@pytest.fixture(scope="session") +def dummy_ssl_cert(tmp_path_factory): + base_tmp_dir = tmp_path_factory.getbasetemp().parent + crt = base_tmp_dir / "pytest-dummy.crt" + key = base_tmp_dir / "pytest-dummy.key" + logger.debug(f"pytest dummy certificate: {crt}, {key}") + # generate once, reuse for all tests + # with FileLock("%s.lock" % crt): + if not crt.is_file(): + generate_dummy_ssl_cert(crt, key) + return crt, key + + +class GunicornProcess(SubProcess): + # QUIT = fast shutdown, TERM = graceful shutdown + GRACEFUL_SIGNAL = signal.SIGTERM + EXIT_SIGNAL = signal.SIGQUIT + + def __init__( + self, + *, + temp_path, + server_bind, + read_size=1024, + ssl_files=None, + worker_class="sync", + log_level="debug", + ): + self.conf_path = Path(os.devnull) + assert isinstance(temp_path, Path) + self.temp_path = temp_path + self.py_path = (temp_path / ("%s.py" % APP_IMPORT_NAME)).absolute() + with open(self.py_path, "w+") as f: + f.write(PY_APPLICATION) + + ssl_opt = [] + if ssl_files is not None: + cert_path, key_path = ssl_files + ssl_opt = [ + "--do-handshake-on-connect", + "--certfile=%s" % cert_path, + "--keyfile=%s" % key_path, + ] + + thread_opt = [] + if worker_class != "sync": + thread_opt = ["--threads=50"] + + self._argv = [ + sys.executable, + "-m", + "gunicorn", + "--config=%s" % self.conf_path, + "--log-level=%s" % (log_level,), + "--worker-class=%s" % (worker_class,), + "--workers=%d" % WORKER_COUNT, + # unsupported at the time this test was submitted + # "--buf-read-size=%d" % read_size, + "--enable-stdio-inheritance", + "--access-logfile=-", + "--disable-redirect-access-to-syslog", + "--graceful-timeout=%d" % (GRACEFUL_TIMEOUT,), + "--bind=%s" % server_bind, + "--reuse-port", + *thread_opt, + *ssl_opt, + "--", + f"{APP_IMPORT_NAME}:{APP_FUNC_NAME}", + ] + super().__init__() + + +class StdlibClient: + def __init__(self, host_port): + # type: (str) -> None + self._host_port = host_port + + def __enter__(self): + # type: () -> Self + import http.client + + self.conn = http.client.HTTPConnection(self._host_port, timeout=2) + return self + + def __exit__(self, *exc): + self.conn.close() + + def get(self, path): + # type: () -> http.client.HTTPResponse + self.conn.request("GET", path, headers={"Host": HTTP_HOST}, body="GETBODY!") + return self.conn.getresponse() + + +class WrkClient(subprocess.Popen): + RE_RATE = re.compile(r"^Requests/sec: *([0-9]+(?:\.[0-9]+)?)$", re.MULTILINE) + + @classmethod + def pytest_supported(cls): + return pytest.mark.skipif( + CMD_OPENSSL is None or CMD_WRK is None, + reason="need openssl and wrk binaries", + ) + + def __init__(self, url_base, path): + # type: (str, str) -> None + assert path.startswith("/") + threads = 10 + connections = 100 + self._env = os.environ.copy() + self._env["LC_ALL"] = "C" + super().__init__( + [ + CMD_WRK, + "-t", + "%d" % threads, + "-c", + "%d" % connections, + "-d5s", + "%s%s" + % ( + url_base, + path, + ), + ], + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + shell=False, + env=self._env, + ) + + def get(self): + out = self.stdout.read(1024 * 4) + ret = self.wait() + assert ret == 0, ret + return out.decode("utf-8", "replace") + + +__all__ = [ + WORKER_PYTEST_LIST, + WORKER_ORDER, + NginxProcess, + GunicornProcess, + StdlibClient, + WrkClient, +] diff --git a/tests/test_nginx.py b/tests/test_nginx.py index bcdea02f8..acb965bfc 100644 --- a/tests/test_nginx.py +++ b/tests/test_nginx.py @@ -7,428 +7,57 @@ # --override-ini=addopts=--strict-markers --exitfirst \ # -- tests/test_nginx.py -import importlib import os -import secrets -import shutil -import signal -import subprocess -import sys -import time -from itertools import chain from pathlib import Path from tempfile import TemporaryDirectory from typing import TYPE_CHECKING import pytest +from support_subprocess import ( + WORKER_ORDER, + WORKER_PYTEST_LIST, + GunicornProcess, + NginxProcess, + StdlibClient, + dummy_ssl_cert, +) if TYPE_CHECKING: import http.client from typing import Any, NamedTuple, Self -# test runner may not be system administrator. not needed here, to run nginx -PATH = "/usr/sbin:/usr/local/sbin:" + os.environ.get("PATH", "/usr/local/bin:/usr/bin") -CMD_OPENSSL = shutil.which("openssl", path=PATH) -CMD_NGINX = shutil.which("nginx", path=PATH) - -pytestmark = pytest.mark.skipif( - CMD_OPENSSL is None or CMD_NGINX is None, - reason="need nginx and openssl binaries", -) - -STDOUT = 0 -STDERR = 1 - -TEST_SIMPLE = [ - pytest.param("sync"), - "eventlet", - "gevent", - "gevent_wsgi", - "gevent_pywsgi", - # "tornado", - "gthread", - # "aiohttp.GunicornWebWorker", # different app signature - # "aiohttp.GunicornUVLoopWebWorker", # " -] # type: list[str|NamedTuple] - -WORKER_DEPENDS = { - "sync": [], - "gthread": [], - "aiohttp.GunicornWebWorker": ["aiohttp"], - "aiohttp.GunicornUVLoopWebWorker": ["aiohttp", "uvloop"], - "uvicorn.workers.UvicornWorker": ["uvicorn"], # deprecated - "uvicorn.workers.UvicornH11Worker": ["uvicorn"], # deprecated - "uvicorn_worker.UvicornWorker": ["uvicorn_worker"], - "uvicorn_worker.UvicornH11Worker": ["uvicorn_worker"], - "eventlet": ["eventlet"], - "gevent": ["gevent"], - "gevent_wsgi": ["gevent"], - "gevent_pywsgi": ["gevent"], - "tornado": ["tornado"], -} -DEP_WANTED = set(chain(*WORKER_DEPENDS.values())) # type: set[str] -DEP_INSTALLED = set() # type: set[str] -WORKER_ORDER = list(WORKER_DEPENDS.keys()) - -for dependency in DEP_WANTED: - try: - importlib.import_module(dependency) - DEP_INSTALLED.add(dependency) - except ImportError: - pass - -for worker_name, worker_needs in WORKER_DEPENDS.items(): - missing = list(pkg for pkg in worker_needs if pkg not in DEP_INSTALLED) - if missing: - for T in (TEST_SIMPLE,): - if worker_name not in T: - continue - T.remove(worker_name) - skipped_worker = pytest.param( - worker_name, marks=pytest.mark.skip("%s not installed" % (missing[0])) - ) - T.append(skipped_worker) - -WORKER_COUNT = 2 -GRACEFUL_TIMEOUT = 2 -APP_IMPORT_NAME = "testsyntax" -APP_FUNC_NAME = "myapp" -HTTP_HOST = "local.test" - -PY_APPLICATION = f""" -import time -def {APP_FUNC_NAME}(environ, start_response): - body = b"response body from app" - response_head = [ - ("Content-Type", "text/plain"), - ("Content-Length", "%d" % len(body)), - ] - start_response("200 OK", response_head) - time.sleep(0.02) - return iter([body]) -""" - -# used in string.format() - duplicate {{ and }} -NGINX_CONFIG_TEMPLATE = """ -pid {pid_path}; -worker_processes 1; -error_log stderr notice; -events {{ - worker_connections 1024; -}} -worker_shutdown_timeout 1; -http {{ - default_type application/octet-stream; - access_log /dev/stdout combined; - upstream upstream_gunicorn {{ - server {gunicorn_upstream} fail_timeout=0; - }} - - server {{ listen {server_bind} default_server; return 400; }} - server {{ - listen {server_bind}; client_max_body_size 4G; - server_name {server_name}; - root {static_dir}; - location / {{ try_files $uri @proxy_to_app; }} - - location @proxy_to_app {{ - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header Host $http_host; - proxy_http_version 1.1; - proxy_redirect off; - proxy_pass {proxy_method}://upstream_gunicorn; - }} - }} -}} -""" - - -class SubProcess: - GRACEFUL_SIGNAL = signal.SIGTERM - - def __enter__(self): - # type: () -> Self - self.run() - return self - - def __exit__(self, *exc): - # type: (*Any) -> None - if self.p is None: - return - self.p.send_signal(signal.SIGKILL) - stdout, stderr = self.p.communicate(timeout=2 + GRACEFUL_TIMEOUT) - ret = self.p.returncode - assert stdout[-512:] == b"", stdout - assert ret == 0, (ret, stdout, stderr) - - def read_stdio(self, *, key, timeout_sec, wait_for_keyword, expect=None): - # type: (int, int, str, set[str]|None) -> str - # try: - # stdout, stderr = self.p.communicate(timeout=timeout) - # except subprocess.TimeoutExpired: - buf = ["", ""] - seen_keyword = 0 - unseen_keywords = list(expect or []) - poll_per_second = 20 - assert key in {0, 1}, key - assert self.p is not None # this helps static type checkers - assert self.p.stdout is not None # this helps static type checkers - assert self.p.stderr is not None # this helps static type checkers - for _ in range(timeout_sec * poll_per_second): - keep_reading = False - # print(f"parsing {buf!r} waiting for {wait_for_keyword!r} + {unseen_keywords!r}") - for fd, file in enumerate([self.p.stdout, self.p.stderr]): - read = file.read(64 * 1024) - if read is not None: - buf[fd] += read.decode("utf-8", "surrogateescape") - keep_reading = True - if seen_keyword or wait_for_keyword in buf[key]: - seen_keyword += 1 - for additional_keyword in tuple(unseen_keywords): - for somewhere in buf: - if additional_keyword in somewhere: - unseen_keywords.remove(additional_keyword) - # gathered all the context we wanted - if seen_keyword and not unseen_keywords: - if not keep_reading: - break - # not seen expected output? wait for % of original timeout - # .. maybe we will still see better error context that way - if seen_keyword > (0.5 * timeout_sec * poll_per_second): - break - # retcode = self.p.poll() - # if retcode is not None: - # break - time.sleep(1.0 / poll_per_second) - # assert buf[abs(key - 1)] == "" - assert wait_for_keyword in buf[key], (wait_for_keyword, *buf) - assert not unseen_keywords, (unseen_keywords, *buf) - return buf[key] - - def run(self): - # type: () -> None - self.p = subprocess.Popen( - self._argv, - bufsize=0, # allow read to return short - cwd=self.temp_path, - shell=False, - close_fds=True, - stdin=subprocess.DEVNULL, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - # creationflags=subprocess.CREATE_NEW_PROCESS_GROUP, - ) - os.set_blocking(self.p.stdout.fileno(), False) - os.set_blocking(self.p.stderr.fileno(), False) - assert self.p.stdout is not None # this helps static type checkers - - def graceful_quit(self, expect=None, ignore=None): - # type: (set[str]|None) -> str - if self.p is None: - raise AssertionError("called graceful_quit() when not running") - self.p.send_signal(self.GRACEFUL_SIGNAL) - # self.p.kill() - stdout = self.p.stdout.read(64 * 1024) or b"" - stderr = self.p.stderr.read(64 * 1024) or b"" - try: - o, e = self.p.communicate(timeout=2 + GRACEFUL_TIMEOUT) - stdout += o - stderr += e - except subprocess.TimeoutExpired: - pass - out = stdout.decode("utf-8", "surrogateescape") - for line in out.split("\n"): - if any(i in line for i in (ignore or ())): - continue - assert line == "" - exitcode = self.p.poll() # will return None if running - self.p.stdout.close() - self.p.stderr.close() - assert exitcode == 0, (exitcode, stdout, stderr) - # print("output after signal: ", stdout, stderr, exitcode) - self.p = None - ret = stderr.decode("utf-8", "surrogateescape") - for keyword in expect or (): - assert keyword in ret, (keyword, ret) - return ret - - -class NginxProcess(SubProcess): - GRACEFUL_SIGNAL = signal.SIGQUIT - - def __init__( - self, - *, - temp_path, - config, - ): - assert isinstance(temp_path, Path) - self.conf_path = (temp_path / ("%s.nginx" % APP_IMPORT_NAME)).absolute() - self.p = None # type: subprocess.Popen[bytes] | None - self.temp_path = temp_path - with open(self.conf_path, "w+") as f: - f.write(config) - self._argv = [ - CMD_NGINX, - # nginx 1.19.5+ added the -e cmdline flag - may be testing earlier - # "-e", "stderr", - "-c", - "%s" % self.conf_path, - ] - - -def generate_dummy_ssl_cert(cert_path, key_path): - # dummy self-signed cert - subprocess.check_output( - [ - CMD_OPENSSL, - "req", - "-new", - "-newkey", - # "ed25519", - # OpenBSD 7.5 / LibreSSL 3.9.0 / Python 3.10.13 - # ssl.SSLError: [SSL: UNKNOWN_CERTIFICATE_TYPE] unknown certificate type (_ssl.c:3900) - # workaround: use RSA keys for testing - "rsa", - "-outform", - "PEM", - "-subj", - "/C=DE", - "-addext", - "subjectAltName=DNS:%s" % (HTTP_HOST), - "-days", - "1", - "-nodes", - "-x509", - "-keyout", - "%s" % (key_path), - "-out", - "%s" % (cert_path), - ], - shell=False, - ) - - -@pytest.fixture(scope="session") -def dummy_ssl_cert(tmp_path_factory): - base_tmp_dir = tmp_path_factory.getbasetemp().parent - crt = base_tmp_dir / "dummy.crt" - key = base_tmp_dir / "dummy.key" - print(crt, key) - # generate once, reuse for all tests - # with FileLock("%s.lock" % crt): - if not crt.is_file(): - generate_dummy_ssl_cert(crt, key) - return crt, key - - -class GunicornProcess(SubProcess): - def __init__( - self, - *, - temp_path, - server_bind, - read_size=1024, - ssl_files=None, - worker_class="sync", - ): - self.conf_path = Path(os.devnull) - self.p = None # type: subprocess.Popen[bytes] | None - assert isinstance(temp_path, Path) - self.temp_path = temp_path - self.py_path = (temp_path / ("%s.py" % APP_IMPORT_NAME)).absolute() - with open(self.py_path, "w+") as f: - f.write(PY_APPLICATION) - - ssl_opt = [] - if ssl_files is not None: - cert_path, key_path = ssl_files - ssl_opt = [ - "--do-handshake-on-connect", - "--certfile=%s" % cert_path, - "--keyfile=%s" % key_path, - ] - - thread_opt = [] - if worker_class != "sync": - thread_opt = ["--threads=50"] - - self._argv = [ - sys.executable, - "-m", - "gunicorn", - "--config=%s" % self.conf_path, - "--log-level=debug", - "--worker-class=%s" % (worker_class,), - "--workers=%d" % WORKER_COUNT, - # unsupported at the time this test was submitted - # "--buf-read-size=%d" % read_size, - "--enable-stdio-inheritance", - "--access-logfile=-", - "--disable-redirect-access-to-syslog", - "--graceful-timeout=%d" % (GRACEFUL_TIMEOUT,), - "--bind=%s" % server_bind, - "--reuse-port", - *thread_opt, - *ssl_opt, - "--", - f"{APP_IMPORT_NAME}:{APP_FUNC_NAME}", - ] - - -class Client: - def __init__(self, host_port): - # type: (str) -> None - self._host_port = host_port - - def __enter__(self): - # type: () -> Self - import http.client - - self.conn = http.client.HTTPConnection(self._host_port, timeout=2) - return self - - def __exit__(self, *exc): - self.conn.close() - - def get(self, path): - # type: () -> http.client.HTTPResponse - self.conn.request("GET", path, headers={"Host": HTTP_HOST}, body="GETBODY!") - return self.conn.getresponse() - # @pytest.mark.parametrize("read_size", [50+secrets.randbelow(2048)]) +@NginxProcess.pytest_supported() @pytest.mark.parametrize("ssl", [False, True], ids=["plain", "ssl"]) -@pytest.mark.parametrize("worker_class", TEST_SIMPLE) +@pytest.mark.parametrize("worker_class", WORKER_PYTEST_LIST) def test_nginx_proxy(*, ssl, worker_class, dummy_ssl_cert, read_size=1024): # avoid ports <= 6178 which may be in use by CI runner # avoid quickly reusing ports as they might not be cleared immediately on BSD worker_index = WORKER_ORDER.index(worker_class) - fixed_port = 6178 + 512 + (2 if ssl else 0) + (4 * worker_index) + fixed_port = 6178 + 512 + (4 if ssl else 0) + (8 * worker_index) # FIXME: should also test inherited socket (LISTEN_FDS) # FIXME: should also test non-inherited (named) UNIX socket gunicorn_bind = "[::1]:%d" % fixed_port # syntax matches between nginx conf and http client - nginx_bind = "[::1]:%d" % (fixed_port + 1) + nginx_bind = "[::1]:%d" % (fixed_port + 2) static_dir = "/run/gunicorn/nonexist" # gunicorn_upstream = "unix:/run/gunicorn/for-nginx.sock" # syntax "[ipv6]:port" matches between gunicorn and nginx gunicorn_upstream = gunicorn_bind - with TemporaryDirectory(suffix="_temp_py") as tempdir_name, Client( + with TemporaryDirectory(suffix="_temp_py") as tempdir_name, StdlibClient( nginx_bind ) as client: temp_path = Path(tempdir_name) - nginx_config = NGINX_CONFIG_TEMPLATE.format( - server_bind=nginx_bind, - pid_path="%s" % (temp_path / "nginx.pid"), - gunicorn_upstream=gunicorn_upstream, - server_name=HTTP_HOST, + nginx_config = NginxProcess.gen_config( + bind=nginx_bind, + temp_path=temp_path, + upstream=gunicorn_upstream, static_dir=static_dir, - proxy_method="https" if ssl else "http", + ssl=ssl, ) with GunicornProcess( @@ -437,18 +66,19 @@ def test_nginx_proxy(*, ssl, worker_class, dummy_ssl_cert, read_size=1024): read_size=read_size, ssl_files=dummy_ssl_cert if ssl else None, temp_path=temp_path, + log_level="debug", ) as server, NginxProcess( config=nginx_config, temp_path=temp_path, ) as proxy: proxy.read_stdio( - key=STDERR, + stderr=True, timeout_sec=8, wait_for_keyword="start worker processes", ) server.read_stdio( - key=STDERR, + stderr=True, wait_for_keyword="Arbiter booted", timeout_sec=8, expect={ @@ -465,11 +95,11 @@ def test_nginx_proxy(*, ssl, worker_class, dummy_ssl_cert, read_size=1024): # using 1.1 to not fail on tornado reporting for 1.0 # nginx sees our HTTP/1.1 request proxy.read_stdio( - key=STDOUT, timeout_sec=4, wait_for_keyword="GET %s HTTP/1.1" % path + timeout_sec=4, wait_for_keyword="GET %s HTTP/1.1" % path ) # gunicorn sees the HTTP/1.1 request from nginx server.read_stdio( - key=STDOUT, timeout_sec=4, wait_for_keyword="GET %s HTTP/1.1" % path + timeout_sec=4, wait_for_keyword="GET %s HTTP/1.1" % path ) server.graceful_quit( diff --git a/tests/test_wrk.py b/tests/test_wrk.py index 08ccf71d9..4d416e7f3 100644 --- a/tests/test_wrk.py +++ b/tests/test_wrk.py @@ -7,362 +7,23 @@ # --override-ini=addopts=--strict-markers --exitfirst \ # -- tests/test_nginx.py -import importlib -import os -import re -import secrets -import shutil -import signal -import subprocess -import sys -import time -from itertools import chain from pathlib import Path from tempfile import TemporaryDirectory -from typing import TYPE_CHECKING import pytest - -if TYPE_CHECKING: - import http.client - from typing import Any, NamedTuple, Self - -# path may be /usr/local/bin for packages ported from other OS -CMD_OPENSSL = shutil.which("openssl") -CMD_WRK = shutil.which("wrk") - -RATE = re.compile(r"^Requests/sec: *([0-9]+(?:\.[0-9]+)?)$", re.MULTILINE) - -pytestmark = pytest.mark.skipif( - CMD_OPENSSL is None or CMD_WRK is None, - reason="need openssl and wrk binaries", +from support_subprocess import ( + WORKER_ORDER, + WORKER_PYTEST_LIST, + GunicornProcess, + WrkClient, + dummy_ssl_cert, ) -STDOUT = 0 -STDERR = 1 - -TEST_SIMPLE = [ - pytest.param("sync"), - "eventlet", - "gevent", - "gevent_wsgi", - "gevent_pywsgi", - # "tornado", - "gthread", - # "aiohttp.GunicornWebWorker", # different app signature - # "aiohttp.GunicornUVLoopWebWorker", # " -] # type: list[str|NamedTuple] - -WORKER_DEPENDS = { - "sync": [], - "gthread": [], - "aiohttp.GunicornWebWorker": ["aiohttp"], - "aiohttp.GunicornUVLoopWebWorker": ["aiohttp", "uvloop"], - "uvicorn.workers.UvicornWorker": ["uvicorn"], # deprecated - "uvicorn.workers.UvicornH11Worker": ["uvicorn"], # deprecated - "uvicorn_worker.UvicornWorker": ["uvicorn_worker"], - "uvicorn_worker.UvicornH11Worker": ["uvicorn_worker"], - "eventlet": ["eventlet"], - "gevent": ["gevent"], - "gevent_wsgi": ["gevent"], - "gevent_pywsgi": ["gevent"], - "tornado": ["tornado"], -} - -DEP_WANTED = set(chain(*WORKER_DEPENDS.values())) # type: set[str] -DEP_INSTALLED = set() # type: set[str] -WORKER_ORDER = list(WORKER_DEPENDS.keys()) - -for dependency in DEP_WANTED: - try: - importlib.import_module(dependency) - DEP_INSTALLED.add(dependency) - except ImportError: - pass - -for worker_name, worker_needs in WORKER_DEPENDS.items(): - missing = list(pkg for pkg in worker_needs if pkg not in DEP_INSTALLED) - if missing: - for T in (TEST_SIMPLE,): - if worker_name not in T: - continue - T.remove(worker_name) - skipped_worker = pytest.param( - worker_name, marks=pytest.mark.skip("%s not installed" % (missing[0])) - ) - T.append(skipped_worker) - -WORKER_COUNT = 2 -GRACEFUL_TIMEOUT = 2 -APP_IMPORT_NAME = "testsyntax" -APP_FUNC_NAME = "myapp" -HTTP_HOST = "local.test" - -PY_APPLICATION = f""" -import time -def {APP_FUNC_NAME}(environ, start_response): - body = b"response body from app" - response_head = [ - ("Content-Type", "text/plain"), - ("Content-Length", "%d" % len(body)), - ] - start_response("200 OK", response_head) - time.sleep(0.02) - return iter([body]) -""" - - -class SubProcess: - GRACEFUL_SIGNAL = signal.SIGTERM - - def __enter__(self): - # type: () -> Self - self.run() - return self - - def __exit__(self, *exc): - # type: (*Any) -> None - if self.p is None: - return - self.p.send_signal(signal.SIGKILL) - stdout, stderr = self.p.communicate(timeout=2 + GRACEFUL_TIMEOUT) - ret = self.p.returncode - assert stdout[-512:] == b"", stdout - assert ret == 0, (ret, stdout, stderr) - - def read_stdio(self, *, key, timeout_sec, wait_for_keyword, expect=None): - # type: (int, int, str, set[str]|None) -> str - # try: - # stdout, stderr = self.p.communicate(timeout=timeout) - # except subprocess.TimeoutExpired: - buf = ["", ""] - seen_keyword = 0 - unseen_keywords = list(expect or []) - poll_per_second = 20 - assert key in {0, 1}, key - assert self.p is not None # this helps static type checkers - assert self.p.stdout is not None # this helps static type checkers - assert self.p.stderr is not None # this helps static type checkers - for _ in range(timeout_sec * poll_per_second): - keep_reading = False - # print(f"parsing {buf!r} waiting for {wait_for_keyword!r} + {unseen_keywords!r}") - for fd, file in enumerate([self.p.stdout, self.p.stderr]): - read = file.read(64 * 1024) - if read is not None: - buf[fd] += read.decode("utf-8", "surrogateescape") - keep_reading = True - if seen_keyword or wait_for_keyword in buf[key]: - seen_keyword += 1 - for additional_keyword in tuple(unseen_keywords): - for somewhere in buf: - if additional_keyword in somewhere: - unseen_keywords.remove(additional_keyword) - # gathered all the context we wanted - if seen_keyword and not unseen_keywords: - if not keep_reading: - break - # not seen expected output? wait for % of original timeout - # .. maybe we will still see better error context that way - if seen_keyword > (0.5 * timeout_sec * poll_per_second): - break - # retcode = self.p.poll() - # if retcode is not None: - # break - time.sleep(1.0 / poll_per_second) - # assert buf[abs(key - 1)] == "" - assert wait_for_keyword in buf[key], (wait_for_keyword, *buf) - assert not unseen_keywords, (unseen_keywords, *buf) - return buf[key] - - def run(self): - # type: () -> None - self.p = subprocess.Popen( - self._argv, - bufsize=0, # allow read to return short - cwd=self.temp_path, - shell=False, - close_fds=True, - stdin=subprocess.DEVNULL, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - # creationflags=subprocess.CREATE_NEW_PROCESS_GROUP, - ) - os.set_blocking(self.p.stdout.fileno(), False) - os.set_blocking(self.p.stderr.fileno(), False) - assert self.p.stdout is not None # this helps static type checkers - - def graceful_quit(self, expect=None, ignore=None): - # type: (set[str]|None) -> str - if self.p is None: - raise AssertionError("called graceful_quit() when not running") - self.p.send_signal(self.GRACEFUL_SIGNAL) - # self.p.kill() - stdout = self.p.stdout.read(64 * 1024) or b"" - stderr = self.p.stderr.read(64 * 1024) or b"" - try: - o, e = self.p.communicate(timeout=2 + GRACEFUL_TIMEOUT) - stdout += o - stderr += e - except subprocess.TimeoutExpired: - pass - out = stdout.decode("utf-8", "surrogateescape") - for line in out.split("\n"): - if any(i in line for i in (ignore or ())): - continue - assert line == "" - exitcode = self.p.poll() # will return None if running - self.p.stdout.close() - self.p.stderr.close() - assert exitcode == 0, (exitcode, stdout, stderr) - # print("output after signal: ", stdout, stderr, exitcode) - self.p = None - ret = stderr.decode("utf-8", "surrogateescape") - for keyword in expect or (): - assert keyword in ret, (keyword, ret) - return ret - - -def generate_dummy_ssl_cert(cert_path, key_path): - # dummy self-signed cert - subprocess.check_output( - [ - CMD_OPENSSL, - "req", - "-new", - "-newkey", - # "ed25519", - # OpenBSD 7.5 / LibreSSL 3.9.0 / Python 3.10.13 - # ssl.SSLError: [SSL: UNKNOWN_CERTIFICATE_TYPE] unknown certificate type (_ssl.c:3900) - # workaround: use RSA keys for testing - "rsa", - "-outform", - "PEM", - "-subj", - "/C=DE", - "-addext", - "subjectAltName=DNS:%s" % (HTTP_HOST), - "-days", - "1", - "-nodes", - "-x509", - "-keyout", - "%s" % (key_path), - "-out", - "%s" % (cert_path), - ], - shell=False, - ) - - -@pytest.fixture(scope="session") -def dummy_ssl_cert(tmp_path_factory): - base_tmp_dir = tmp_path_factory.getbasetemp().parent - crt = base_tmp_dir / "dummy.crt" - key = base_tmp_dir / "dummy.key" - print(crt, key) - # generate once, reuse for all tests - # with FileLock("%s.lock" % crt): - if not crt.is_file(): - generate_dummy_ssl_cert(crt, key) - return crt, key - - -class GunicornProcess(SubProcess): - def __init__( - self, - *, - temp_path, - server_bind, - read_size=1024, - ssl_files=None, - worker_class="sync", - ): - self.conf_path = Path(os.devnull) - self.p = None # type: subprocess.Popen[bytes] | None - assert isinstance(temp_path, Path) - self.temp_path = temp_path - self.py_path = (temp_path / ("%s.py" % APP_IMPORT_NAME)).absolute() - with open(self.py_path, "w+") as f: - f.write(PY_APPLICATION) - - ssl_opt = [] - if ssl_files is not None: - cert_path, key_path = ssl_files - ssl_opt = [ - "--do-handshake-on-connect", - "--certfile=%s" % cert_path, - "--keyfile=%s" % key_path, - ] - thread_opt = [] - if worker_class != "sync": - thread_opt = ["--threads=50"] - - self._argv = [ - sys.executable, - "-m", - "gunicorn", - "--config=%s" % self.conf_path, - "--log-level=info", - "--worker-class=%s" % (worker_class,), - "--workers=%d" % WORKER_COUNT, - # unsupported at the time this test was submitted - # "--buf-read-size=%d" % read_size, - "--enable-stdio-inheritance", - "--access-logfile=-", - "--disable-redirect-access-to-syslog", - "--graceful-timeout=%d" % (GRACEFUL_TIMEOUT,), - "--bind=%s" % server_bind, - "--reuse-port", - *thread_opt, - *ssl_opt, - "--", - f"{APP_IMPORT_NAME}:{APP_FUNC_NAME}", - ] - - -class Client: - def __init__(self, url_base): - # type: (str) -> None - self._url_base = url_base - self._env = os.environ.copy() - self._env["LC_ALL"] = "C" - - def __enter__(self): - # type: () -> Self - return self - - def __exit__(self, *exc): - pass - - def get(self, path): - # type: () -> http.client.HTTPResponse - assert path.startswith("/") - threads = 10 - connections = 100 - out = subprocess.check_output( - [ - CMD_WRK, - "-t", - "%d" % threads, - "-c", - "%d" % connections, - "-d5s", - "%s%s" - % ( - self._url_base, - path, - ), - ], - shell=False, - env=self._env, - ) - - return out.decode("utf-8", "replace") - # @pytest.mark.parametrize("read_size", [50+secrets.randbelow(2048)]) +@WrkClient.pytest_supported() @pytest.mark.parametrize("ssl", [False, True], ids=["plain", "ssl"]) -@pytest.mark.parametrize("worker_class", TEST_SIMPLE) +@pytest.mark.parametrize("worker_class", WORKER_PYTEST_LIST) def test_wrk(*, ssl, worker_class, dummy_ssl_cert, read_size=1024): if worker_class == "eventlet" and ssl: pytest.skip("eventlet worker does not catch errors in ssl.wrap_socket") @@ -376,9 +37,7 @@ def test_wrk(*, ssl, worker_class, dummy_ssl_cert, read_size=1024): proxy_method = "https" if ssl else "http" - with TemporaryDirectory(suffix="_temp_py") as tempdir_name, Client( - proxy_method + "://" + gunicorn_bind - ) as client: + with TemporaryDirectory(suffix="_temp_py") as tempdir_name: temp_path = Path(tempdir_name) with GunicornProcess( @@ -387,9 +46,10 @@ def test_wrk(*, ssl, worker_class, dummy_ssl_cert, read_size=1024): read_size=read_size, ssl_files=dummy_ssl_cert if ssl else None, temp_path=temp_path, + log_level="info", ) as server: server.read_stdio( - key=STDERR, + stderr=True, wait_for_keyword="[INFO] Starting gunicorn", timeout_sec=6, expect={ @@ -398,10 +58,11 @@ def test_wrk(*, ssl, worker_class, dummy_ssl_cert, read_size=1024): ) path = "/pytest/basic" - out = client.get(path) + with WrkClient(proxy_method + "://" + gunicorn_bind, path=path) as client: + out = client.get() print("##############\n" + out) - extract = RATE.search(out) + extract = WrkClient.RE_RATE.search(out) assert extract is not None, out rate = float(extract.groups()[0]) if worker_class == "sync": @@ -409,13 +70,11 @@ def test_wrk(*, ssl, worker_class, dummy_ssl_cert, read_size=1024): else: assert rate > 50 - server.read_stdio( - key=STDOUT, timeout_sec=2, wait_for_keyword="GET %s HTTP/1.1" % path - ) + server.read_stdio(timeout_sec=2, wait_for_keyword="GET %s HTTP/1.1" % path) if ssl: pass # server.read_stdio( - # key=STDERR, + # stderr=True, # wait_for_keyword="[DEBUG] ssl connection closed", # timeout_sec=4, # ) From dcc6d456b97d03ddfa52f5c15ccb8142b28f92a0 Mon Sep 17 00:00:00 2001 From: "Paul J. Dorn" Date: Thu, 28 Nov 2024 21:39:01 +0100 Subject: [PATCH 13/17] CI: leftover debug print --- tests/test_wrk.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_wrk.py b/tests/test_wrk.py index 4d416e7f3..791e40e1a 100644 --- a/tests/test_wrk.py +++ b/tests/test_wrk.py @@ -60,7 +60,7 @@ def test_wrk(*, ssl, worker_class, dummy_ssl_cert, read_size=1024): path = "/pytest/basic" with WrkClient(proxy_method + "://" + gunicorn_bind, path=path) as client: out = client.get() - print("##############\n" + out) + # print("##############\n" + out) extract = WrkClient.RE_RATE.search(out) assert extract is not None, out From 6f5ff0c6ebed52d78740d1afb87e7d9f9533771f Mon Sep 17 00:00:00 2001 From: "Paul J. Dorn" Date: Thu, 28 Nov 2024 21:45:14 +0100 Subject: [PATCH 14/17] CI: test reliability gevent worker on OmniOS py3.12/gcc14 fails with: AttributeError: module 'threading' has no attribute 'get_native_id' skip slow benchmark on PyPy --- .github/workflows/bsd.yml | 2 ++ .github/workflows/illumos.yml | 4 +++- .github/workflows/tox.yml | 4 +++- tests/support_subprocess.py | 38 ++++++++++++++++++++++++++--------- tests/test_nginx.py | 15 +++++++++++++- tests/test_wrk.py | 13 +++++++++--- 6 files changed, 61 insertions(+), 15 deletions(-) diff --git a/.github/workflows/bsd.yml b/.github/workflows/bsd.yml index 963c55dd4..fa4e56e64 100644 --- a/.github/workflows/bsd.yml +++ b/.github/workflows/bsd.yml @@ -39,6 +39,7 @@ jobs: run: | uname -a \ && python3.11 --version \ + && python3.11 -c 'import os,sys,platform;print(platform.system(),sys.platform,os.name)' \ && python3.11 -m tox --version \ && openssl version \ && pkg info nginx \ @@ -62,6 +63,7 @@ jobs: run: | uname -a \ && python3 --version \ + && python3 -c 'import os,sys,platform;print(platform.system(),sys.platform,os.name)' \ && python3 -m tox --version \ && openssl version \ && pkg_info nginx \ diff --git a/.github/workflows/illumos.yml b/.github/workflows/illumos.yml index 2bd853440..2f844d119 100644 --- a/.github/workflows/illumos.yml +++ b/.github/workflows/illumos.yml @@ -36,14 +36,16 @@ jobs: # /tmp/.nginx must exist because nginx will not create configured tmp # build-essential shall point to suitable gcc13/gcc14/.. # TODO: replace python-MN with version-agnostic alias + # release: "r151048" prepare: | - pkg install pip-312 python-312 sqlite-3 nginx build-essential + pkg install pip-312 python-312 sqlite-3 nginx gcc14 usesh: true copyback: false run: | cat /etc/release \ && uname -a \ && python3 --version \ + && python3 -c 'import os,sys,platform;print(platform.system(),sys.platform,os.name)' \ && openssl version \ && pkg info nginx \ && gcc -dM -E - Self import http.client - self.conn = http.client.HTTPConnection(self._host_port, timeout=2) + self.conn = http.client.HTTPConnection(self._host_port, timeout=5) return self def __exit__(self, *exc): self.conn.close() - def get(self, path): + def get(self, path="/", test=False): # type: () -> http.client.HTTPResponse - self.conn.request("GET", path, headers={"Host": HTTP_HOST}, body="GETBODY!") + body = b"GETBODY!" + self.conn.request( + "GET", + path, + headers={ + "Host": "invalid.invalid." if test else HTTP_HOST, + "Connection": "close", + "Content-Length": "%d" % (len(body),), + }, + body=body, + ) return self.conn.getresponse() diff --git a/tests/test_nginx.py b/tests/test_nginx.py index acb965bfc..ea047b914 100644 --- a/tests/test_nginx.py +++ b/tests/test_nginx.py @@ -86,9 +86,22 @@ def test_nginx_proxy(*, ssl, worker_class, dummy_ssl_cert, read_size=1024): }, ) + path = "/dummy" + try: + response = client.get(path, test=True) + except TimeoutError as exc: + raise AssertionError(f"failed to query proxy: {exc!r}") from exc + assert response.status == 400 + test_body = response.read() + assert b"nginx" in test_body + proxy.read_stdio(timeout_sec=4, wait_for_keyword="GET %s HTTP/1.1" % path) + for num_request in range(5): path = "/pytest/%d" % (num_request) - response = client.get(path) + try: + response = client.get(path) + except TimeoutError as exc: + raise AssertionError(f"failed to fetch {path!r}: {exc!r}") from exc assert response.status == 200 assert response.read() == b"response body from app" diff --git a/tests/test_wrk.py b/tests/test_wrk.py index 791e40e1a..40b0f9272 100644 --- a/tests/test_wrk.py +++ b/tests/test_wrk.py @@ -7,6 +7,7 @@ # --override-ini=addopts=--strict-markers --exitfirst \ # -- tests/test_nginx.py +import platform from pathlib import Path from tempfile import TemporaryDirectory @@ -22,6 +23,9 @@ # @pytest.mark.parametrize("read_size", [50+secrets.randbelow(2048)]) @WrkClient.pytest_supported() +@pytest.mark.skipif( + platform.python_implementation() == "PyPy", reason="slow on Github CI" +) @pytest.mark.parametrize("ssl", [False, True], ids=["plain", "ssl"]) @pytest.mark.parametrize("worker_class", WORKER_PYTEST_LIST) def test_wrk(*, ssl, worker_class, dummy_ssl_cert, read_size=1024): @@ -65,10 +69,13 @@ def test_wrk(*, ssl, worker_class, dummy_ssl_cert, read_size=1024): extract = WrkClient.RE_RATE.search(out) assert extract is not None, out rate = float(extract.groups()[0]) + expected = 50 if worker_class == "sync": - assert rate > 5 - else: - assert rate > 50 + expected = 5 + # test way too short to make slow GitHub runners fast on PyPy + if platform.python_implementation() == "PyPy": + expected //= 5 + assert rate > expected, (rate, expected) server.read_stdio(timeout_sec=2, wait_for_keyword="GET %s HTTP/1.1" % path) if ssl: From de679d81946a71833edcaea955250cadfd6eb832 Mon Sep 17 00:00:00 2001 From: "Paul J. Dorn" Date: Sat, 30 Nov 2024 18:21:42 +0100 Subject: [PATCH 15/17] temp: test gevent HEAD --- .github/workflows/illumos.yml | 4 ++-- requirements_test.txt | 2 +- tox.ini | 5 ++++- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/illumos.yml b/.github/workflows/illumos.yml index 2f844d119..75f870785 100644 --- a/.github/workflows/illumos.yml +++ b/.github/workflows/illumos.yml @@ -38,7 +38,7 @@ jobs: # TODO: replace python-MN with version-agnostic alias # release: "r151048" prepare: | - pkg install pip-312 python-312 sqlite-3 nginx gcc14 + pkg install pip-312 python-312 sqlite-3 nginx gcc14 git usesh: true copyback: false run: | @@ -49,7 +49,7 @@ jobs: && openssl version \ && pkg info nginx \ && gcc -dM -E - =7.2.0 diff --git a/tox.ini b/tox.ini index 9bf99e1be..7b1e0b0fc 100644 --- a/tox.ini +++ b/tox.ini @@ -10,8 +10,11 @@ envlist = [testenv] package = editable commands = pytest --cov=gunicorn {posargs} +setenv = + ac_cv_header_sys_inotify_h=no + ac_cv_func_inotify_init=no deps = - -rrequirements_test.txt + --pre -rrequirements_test.txt [testenv:run-entrypoint] package = wheel From c99864659083e8b77e848d1c7deb2ef32e61730b Mon Sep 17 00:00:00 2001 From: "Paul J. Dorn" Date: Sat, 30 Nov 2024 18:00:16 +0100 Subject: [PATCH 16/17] CI: praise the ominous solar OS on gevent 24.10.1 to 24.11.1 gevent worker on OmniOS py3.12/gcc14 fails with: AttributeError: module 'threading' has no attribute 'get_native_id' see https://github.com/gevent/gevent/issues/2053 --- .github/workflows/illumos.yml | 4 ++-- pyproject.toml | 1 + requirements_test.txt | 3 ++- tox.ini | 5 +---- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/illumos.yml b/.github/workflows/illumos.yml index 75f870785..994990ec1 100644 --- a/.github/workflows/illumos.yml +++ b/.github/workflows/illumos.yml @@ -38,7 +38,7 @@ jobs: # TODO: replace python-MN with version-agnostic alias # release: "r151048" prepare: | - pkg install pip-312 python-312 sqlite-3 nginx gcc14 git + pkg install pip-312 python-312 sqlite-3 nginx gcc14 usesh: true copyback: false run: | @@ -49,7 +49,7 @@ jobs: && openssl version \ && pkg info nginx \ && gcc -dM -E - =0.2"] gthread = [] setproctitle = ["setproctitle"] testing = [ + "gevent!=24.10.1,!=24.10.2,!=24.10.3,!=24.11.1;sys_platform=='sunos5'", "gevent", "eventlet", "coverage", diff --git a/requirements_test.txt b/requirements_test.txt index de9f36801..dc4f3d1b8 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -1,4 +1,5 @@ -git+https://github.com/gevent/gevent.git@676d86a3ae799118d79ddc225e9286c6a75755a4 +gevent!=24.10.1,!=24.10.2,!=24.10.3,!=24.11.1;sys_platform=='sunos5' +gevent eventlet coverage pytest>=7.2.0 diff --git a/tox.ini b/tox.ini index 7b1e0b0fc..9bf99e1be 100644 --- a/tox.ini +++ b/tox.ini @@ -10,11 +10,8 @@ envlist = [testenv] package = editable commands = pytest --cov=gunicorn {posargs} -setenv = - ac_cv_header_sys_inotify_h=no - ac_cv_func_inotify_init=no deps = - --pre -rrequirements_test.txt + -rrequirements_test.txt [testenv:run-entrypoint] package = wheel From ff48a1ff0e9fef2f025ddb30e47922cdce4e99d8 Mon Sep 17 00:00:00 2001 From: "Paul J. Dorn" Date: Sat, 30 Nov 2024 23:33:47 +0100 Subject: [PATCH 17/17] CI: avoid gcc14 --- .github/workflows/illumos.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/illumos.yml b/.github/workflows/illumos.yml index 994990ec1..5f6c0f487 100644 --- a/.github/workflows/illumos.yml +++ b/.github/workflows/illumos.yml @@ -36,9 +36,10 @@ jobs: # /tmp/.nginx must exist because nginx will not create configured tmp # build-essential shall point to suitable gcc13/gcc14/.. # TODO: replace python-MN with version-agnostic alias - # release: "r151048" + # pinning below r151050 to avoid gcc14 incompat + release: "r151050" prepare: | - pkg install pip-312 python-312 sqlite-3 nginx gcc14 + pkg install pip-312 python-312 sqlite-3 nginx gcc13 usesh: true copyback: false run: |