Fix #549 Add ability to subclass http_unit (#551)

- moved unbound_localhost_tcp_port to util from pkunit (deprecated)
- xlsx test fix due to .00 vs no ".00" version xlsxwriter 3.2.0 to 3.2.1

pykern/http.py

@@ -493,7 +493,7 @@ def _reply(call, obj):
                 r.call_id = call.call_id
                 self.handler.write_message(_pack_msg(r), binary=True)
             except Exception as e:
-                pkdlog("exception={} call={} stack={}", call, e, pkdexc())
+                pkdlog("exception={} call={} stack={}", e, call, pkdexc())
                 self.destroy()
 
         def _quest_kwargs():
@@ -507,7 +507,7 @@ def _quest_kwargs():
                 self._log("error", None, "msg unpack error={}", [e])
                 self.destroy()
                 return None
-            self._log("call", c)
+            self._log("call", c, "api={}", [c.api_name])
             if not (a := _api(c)):
                 return
             r = await _call(c, a, c.api_args)

pykern/http_unit.py

@@ -4,80 +4,173 @@
 :license: http://www.apache.org/licenses/LICENSE-2.0.html
 """
 
-# Defer imports for unit tests
+# Defer as many pykern imports as possible to defer pkconfig runing
+from pykern.pkcollections import PKDict
+import os
+import signal
+import time
 
 
 class Setup:
+    """Usage::
+
+        async with http_unit.Setup(api_classes=(_class())) as c:
+            from pykern.pkcollections import PKDict
+            from pykern import pkunit
+
+            e = PKDict(ping="pong")
+            pkunit.pkeq(e.pkupdate(counter=1), await c.call_api("echo", e))
+            pkunit.pkeq(e.pkupdate(counter=2), await c.call_api("echo", e))
+
+    May be subclassed to start multiple servers.
+    """
 
     AUTH_TOKEN = "http_unit_auth_secret"
 
-    def __init__(self, api_classes, attr_classes=(), coros=()):
-        import os, time
-        from pykern.pkcollections import PKDict
+    def __init__(self, **server_config):
+        # Must be first
+        self._global_config()
+        self.http_config = self._http_config()
+        self.server_config = self._server_config(server_config)
+        self.server_pid = self._server_process()
+        time.sleep(2)
+        self.client = self._client()
+
+    def destroy(self):
+        """Destroy client and kill attributes ``*_pid`` attrs"""
+
+        self.client.destroy()
+        for p in filter(lambda x: x.endswith("_pid"), dir(self)):
+            try:
+                os.kill(getattr(self, p), signal.SIGKILL)
+            except Exception:
+                pass
 
-        def _global_config():
-            c = PKDict(
-                PYKERN_PKDEBUG_WANT_PID_TIME="1",
-            )
-            os.environ.update(**c)
-            from pykern import pkconfig
+    def _client(self):
+        """Creates a client to be used for requests.
 
-            pkconfig.reset_state_for_testing(c)
+        Called in `__init__`.
 
-        def _http_config():
-            from pykern import pkconst, pkunit
+        Returns:
+            object: http client, set to ``self.client``
+        """
+        from pykern import http
 
-            return PKDict(
-                # any uri is fine
-                api_uri="/http_unit",
-                # just needs to be >= 16 word (required by http) chars; apps should generate this randomly
-                tcp_ip=pkconst.LOCALHOST_IP,
-                tcp_port=pkunit.unbound_localhost_tcp_port(),
-            )
+        return http.HTTPClient(self.http_config.copy())
 
-        def _server():
-            from pykern import pkdebug, http
+    def _client_awaitable(self):
+        """How to connect to client
 
-            def _api_classes():
-                class AuthAPI(http.AuthAPI):
-                    PYKERN_HTTP_TOKEN = self.AUTH_TOKEN
+        Awaited in `__aenter__`.
 
-                return list(api_classes) + [AuthAPI]
+        Returns:
+            Awaitable: coroutine to connect to client
+        """
 
-            if rv := os.fork():
-                return rv
-            try:
-                pkdebug.pkdlog("start server")
-                http.server_start(
-                    attr_classes=attr_classes,
-                    api_classes=_api_classes(),
-                    http_config=self.http_config.copy(),
-                    coros=coros,
-                )
-            except Exception as e:
-                pkdebug.pkdlog("server exception={} stack={}", e, pkdebug.pkdexc())
-            finally:
-                os._exit(0)
-
-        _global_config()
-        self.http_config = _http_config()
-        self.server_pid = _server()
-        time.sleep(1)
         from pykern import http
 
-        self.client = http.HTTPClient(self.http_config.copy())
+        return self.client.connect(http.AuthArgs(token=self.AUTH_TOKEN))
 
-    def destroy(self):
-        import os, signal
+    def _global_config(self, **kwargs):
+        """Initializes os.environ and pkconfig
 
-        os.kill(self.server_pid, signal.SIGKILL)
+        Called first.
 
-    async def __aenter__(self):
+        Args:
+            kwargs (dict): merged into environ and config (from subclasses)
+        """
+
+        c = PKDict(
+            PYKERN_PKDEBUG_WANT_PID_TIME="1",
+            **kwargs,
+        )
+        os.environ.update(**c)
+        from pykern import pkconfig
+
+        pkconfig.reset_state_for_testing(c)
+
+    def _http_config(self):
+        """Initializes ``self.http_config``
+
+        Returns:
+            PKDict: configuration to be shared with client and server
+        """
+        from pykern import pkconst, util
+
+        return PKDict(
+            # any uri is fine
+            api_uri="/http_unit",
+            # just needs to be >= 16 word (required by http) chars; apps should generate this randomly
+            tcp_ip=pkconst.LOCALHOST_IP,
+            tcp_port=util.unbound_localhost_tcp_port(),
+        )
+
+    def _server_config(self, init_config):
+        """Config to be passed to `pykern.http.server_start` in `server_start`
+
+        A simple `pykern.http.AuthAPI` implementation is defaulted if not in ``init_config.api_classes``.
+
+        Args:
+            init_config (dict): what was passed to `__init__`
+        Returns:
+            PKDict: configuration for `server_start`
+        """
+
+        def _api_classes(init_classes):
+            from pykern import http
+
+            class AuthAPI(http.AuthAPI):
+                PYKERN_HTTP_TOKEN = self.AUTH_TOKEN
+
+            rv = init_classes
+            if any(filter(lambda c: issubclass(c, http.AuthAPI), rv)):
+                return rv
+            return rv + [AuthAPI]
+
+        rv = PKDict(init_config) if init_config else PKDict()
+        rv.pksetdefault(
+            api_classes=(),
+            attr_classes=(),
+            coros=(),
+            http_config=PKDict,
+        )
+        rv.http_config.pksetdefault(**self.http_config)
+        rv.api_classes = _api_classes(list(rv.api_classes))
+        return rv
+
+    def _server_process(self):
+        """Call `server_start` in separate process
+
+        Override this method to start multiple servers, saving pids in
+        attributes that end in ``_pid`` so that `destroy` will kill
+        them.
+
+        Returns:
+            int: pid of process
+
+        """
+        from pykern import pkdebug
+
+        if rv := os.fork():
+            return rv
+        try:
+            pkdebug.pkdlog("start server")
+            self._server_start()
+        except Exception as e:
+            pkdebug.pkdlog("exception={} stack={}", e, pkdebug.pkdexc())
+        finally:
+            os._exit(0)
+
+    def _server_start(self):
+        """Calls http.server_start with ``self.server_config``"""
         from pykern import http
 
-        await self.client.connect(http.AuthArgs(token=self.AUTH_TOKEN))
+        http.server_start(**self.server_config)
+
+    async def __aenter__(self):
+        await self._client_awaitable()
         return self.client
 
     async def __aexit__(self, *args, **kwargs):
-        self.client.destroy()
+        self.destroy()
         return False

pykern/pkinspect.py

@@ -1,10 +1,8 @@
-# -*- coding: utf-8 -*-
 """Helper functions for to :mod:`inspect`.
 
 :copyright: Copyright (c) 2015 RadiaSoft, Inc.  All Rights Reserved.
 :license: http://www.apache.org/licenses/LICENSE-2.0.html
 """
-from __future__ import absolute_import, division, print_function
 
 # Avoid pykern imports so avoid dependency issues for pkconfig
 from pykern.pkcollections import PKDict

pykern/pkunit.py

@@ -4,23 +4,21 @@
 :license: http://www.apache.org/licenses/LICENSE-2.0.html
 """
 
+# defer importing pkconfig
 from pykern import pkcompat
 from pykern import pkconst
 from pykern import pkinspect
 from pykern import pkio
-
-# defer importing pkconfig
-import pykern.pkconst
 import contextlib
 import importlib
 import inspect
 import json
 import os
 import py
+import pykern.pkconst
+import pykern.util
 import pytest
-import random
 import re
-import socket
 import subprocess
 import sys
 import traceback
@@ -279,31 +277,6 @@ def file_eq(expect_path, *args, **kwargs):
     _FileEq(expect_path, *args, **kwargs)
 
 
-def unbound_localhost_tcp_port(start=10000, stop=20000):
-    """Looks for AF_INET SOCK_STREAM port for which bind succeeds
-
-    Args:
-        start (int): first port [10000]
-        stop (int): one greater than last port (passed to range) [20000]
-    Returns:
-        int: port is available or raises ValueError
-    """
-
-    def _check_port(port):
-        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
-            s.bind((LOCALHOST_IP, int(port)))
-        return port
-
-    for p in random.sample(range(start, stop), 100):
-        try:
-            return _check_port(p)
-        except Exception:
-            pass
-    raise ValueError(
-        f"unable find port random sample range={start}-{stop} tries=100 ip={LOCALHOST_IP}"
-    )
-
-
 def is_test_run():
     """Running in a test?
 
@@ -505,6 +478,10 @@ def save_chdir_work(is_pkunit_prefix=False, want_empty=True):
     )
 
 
+#: DEPRECATED
+unbound_localhost_tcp_port = pykern.util.unbound_localhost_tcp_port
+
+
 def work_dir():
     """Returns ephemeral work directory, created if necessary.
 

pykern/util.py

@@ -4,11 +4,10 @@
 :license: http://www.apache.org/licenses/LICENSE-2.0.html
 """
 
-# Root module: Limit imports to avoid dependency issues
+# Root module: avoid global pykern imports
 import os.path
 import sys
 
-
 _ACCEPTABLE_CONTROL_CODE_RATIO = 0.33
 _DEFAULT_ROOT = "run"
 _DEV_ONLY_FILES = ("setup.py", "pyproject.toml")
@@ -155,3 +154,30 @@ def random_base62(length=16):
 
     r = random.SystemRandom()
     return "".join(r.choice(pkconst.BASE62_CHARS) for x in range(length))
+
+
+def unbound_localhost_tcp_port(start=10000, stop=20000):
+    """Looks for AF_INET SOCK_STREAM port for which bind succeeds
+
+    Args:
+        start (int): first port [10000]
+        stop (int): one greater than last port (passed to range) [20000]
+    Returns:
+        int: port is available or raises ValueError
+    """
+    import random, socket
+    from pykern import pkasyncio, pkconst
+
+    def _check_port(port):
+        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+            s.bind((pkconst.LOCALHOST_IP, int(port)))
+        return port
+
+    for p in random.sample(range(start, stop), 100):
+        try:
+            return _check_port(p)
+        except Exception:
+            pass
+    raise ValueError(
+        f"unable find port random sample range={start}-{stop} tries=100 ip={pkconst.LOCALHOST_IP}"
+    )

tests/xlsx_data/1.out/xl/worksheets/sheet1.xml

@@ -1,2 +1,2 @@
 <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
-<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships"><dimension ref="A1:F7"/><sheetViews><sheetView tabSelected="1" workbookViewId="0"/></sheetViews><sheetFormatPr defaultRowHeight="15"/><cols><col min="1" max="1" width="9.7109375" customWidth="1"/><col min="2" max="2" width="20.7109375" customWidth="1"/><col min="3" max="4" width="8.7109375" customWidth="1"/><col min="5" max="5" width="0" customWidth="1"/><col min="6" max="6" width="4.7109375" customWidth="1"/></cols><sheetData><row r="1" spans="1:6"><c r="A1" s="1"><v>375</v></c><c r="B1" s="2" t="s"><v>0</v></c><c r="C1" s="1"><v>7.77</v></c><c r="D1" s="2" t="s"><v>1</v></c><c r="E1" s="2"/><c r="F1" s="1"><v>3.14</v></c></row><row r="2" spans="1:6"><c r="A2" s="3" t="s"><v>2</v></c><c r="B2" s="3" t="s"><v>3</v></c><c r="C2" s="3" t="s"><v>4</v></c><c r="D2" s="3" t="s"><v>5</v></c></row><row r="3" spans="1:6"><c r="A3" s="4"><f>ROUND(PRODUCT(100--B3,1),2)</f><v>135.34</v></c><c r="B3" s="5"><v>35.34</v></c><c r="C3" s="1"><f>ROUND(MOD(B3,1),2)</f><v>0.34</v></c><c r="D3" s="5" t="str"><f>IF(B3&lt;=2,"red","green")</f><v>green</v></c></row><row r="4" spans="1:6"><c r="A4" s="6"><f>ROUND(MAX(999),2)</f><v>999.00</v></c><c r="B4" s="6"><f>ROUND(MAX(111,222),2)</f><v>222.00</v></c><c r="C4" s="6"><f>ROUND(MIN(333,444),2)</f><v>333.00</v></c><c r="D4" s="6"><f>ROUND(IF(0,1/0,99),2)</f><v>99.00</v></c></row><row r="5" spans="1:6"><c r="A5" s="2" t="str"><f>AND(TRUE,1&lt;=2)</f><v>TRUE</v></c><c r="B5" s="2" t="str"><f>OR(FALSE,3&gt;4)</f><v>FALSE</v></c><c r="C5" s="2" t="str"><f>NOT(6&lt;=7)</f><v>FALSE</v></c><c r="D5" s="2" t="s"><v>6</v></c></row><row r="6" spans="1:6"><c r="A6" s="7" t="s"><v>7</v></c><c r="B6" s="7"/><c r="C6" s="8" t="s"><v>8</v></c><c r="D6" s="7"/></row><row r="7" spans="1:6"><c r="A7" s="2" t="s"><v>9</v></c><c r="B7" s="2"/><c r="C7" s="2"/><c r="D7" s="2"/></row></sheetData><pageMargins left="0.7" right="0.7" top="0.75" bottom="0.75" header="0.3" footer="0.3"/><ignoredErrors><ignoredError sqref="A1 C1 F1 A3 B3 C3 A4 B4 C4 D4" numberStoredAsText="1"/></ignoredErrors></worksheet>
+<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships"><dimension ref="A1:F7"/><sheetViews><sheetView tabSelected="1" workbookViewId="0"/></sheetViews><sheetFormatPr defaultRowHeight="15"/><cols><col min="1" max="1" width="9.7109375" customWidth="1"/><col min="2" max="2" width="20.7109375" customWidth="1"/><col min="3" max="4" width="8.7109375" customWidth="1"/><col min="5" max="5" width="0" customWidth="1"/><col min="6" max="6" width="4.7109375" customWidth="1"/></cols><sheetData><row r="1" spans="1:6"><c r="A1" s="1"><v>375.00</v></c><c r="B1" s="2" t="s"><v>0</v></c><c r="C1" s="1"><v>7.77</v></c><c r="D1" s="2" t="s"><v>1</v></c><c r="E1" s="2"/><c r="F1" s="1"><v>3.14</v></c></row><row r="2" spans="1:6"><c r="A2" s="3" t="s"><v>2</v></c><c r="B2" s="3" t="s"><v>3</v></c><c r="C2" s="3" t="s"><v>4</v></c><c r="D2" s="3" t="s"><v>5</v></c></row><row r="3" spans="1:6"><c r="A3" s="4"><f>ROUND(PRODUCT(100--B3,1),2)</f><v>135.34</v></c><c r="B3" s="5"><v>35.34</v></c><c r="C3" s="1"><f>ROUND(MOD(B3,1),2)</f><v>0.34</v></c><c r="D3" s="5" t="str"><f>IF(B3&lt;=2,"red","green")</f><v>green</v></c></row><row r="4" spans="1:6"><c r="A4" s="6"><f>ROUND(MAX(999),2)</f><v>999.00</v></c><c r="B4" s="6"><f>ROUND(MAX(111,222),2)</f><v>222.00</v></c><c r="C4" s="6"><f>ROUND(MIN(333,444),2)</f><v>333.00</v></c><c r="D4" s="6"><f>ROUND(IF(0,1/0,99),2)</f><v>99.00</v></c></row><row r="5" spans="1:6"><c r="A5" s="2" t="str"><f>AND(TRUE,1&lt;=2)</f><v>TRUE</v></c><c r="B5" s="2" t="str"><f>OR(FALSE,3&gt;4)</f><v>FALSE</v></c><c r="C5" s="2" t="str"><f>NOT(6&lt;=7)</f><v>FALSE</v></c><c r="D5" s="2" t="s"><v>6</v></c></row><row r="6" spans="1:6"><c r="A6" s="7" t="s"><v>7</v></c><c r="B6" s="7"/><c r="C6" s="8" t="s"><v>8</v></c><c r="D6" s="7"/></row><row r="7" spans="1:6"><c r="A7" s="2" t="s"><v>9</v></c><c r="B7" s="2"/><c r="C7" s="2"/><c r="D7" s="2"/></row></sheetData><pageMargins left="0.7" right="0.7" top="0.75" bottom="0.75" header="0.3" footer="0.3"/><ignoredErrors><ignoredError sqref="A1 C1 F1 A3 B3 C3 A4 B4 C4 D4" numberStoredAsText="1"/></ignoredErrors></worksheet>