clean up

.github/workflows/ci.yml

@@ -21,6 +21,7 @@ jobs:
        test-script: 'tests/test.sh'
        python-version: '3.9'
        use-cache: true
+       run-coveralls: true
 
   # # this creates linter settings and uploads to an artifact so the configs can be pulled and used across jobs
   # LintConfig:

gen3workflow/app.py

@@ -1,8 +1,8 @@
 from fastapi import FastAPI
 import httpx
 from importlib.metadata import version
-# import os
-# from gen3authz.client.arborist.async_client import ArboristClient
+
+from cdislogging import get_logger
 
 from gen3workflow import logger
 from gen3workflow.config import config
@@ -26,20 +26,7 @@ def get_app(httpx_client=None) -> FastAPI:
     app.include_router(ga4gh_tes_router, tags=["GA4GH TES"])
 
     # Following will update logger level, propagate, and handlers
-    # TODO test that
-    # get_logger("gen3workflow", log_level="debug" if debug == True else "info")
-
-    logger.info("Initializing Arborist client")
-    # custom_arborist_url = os.environ.get("ARBORIST_URL", config["ARBORIST_URL"])
-    # if custom_arborist_url:
-    #     app.arborist_client = ArboristClient(
-    #         arborist_base_url=custom_arborist_url,
-    #         logger=get_logger("gen3workflow.gen3authz", log_level="debug"),
-    #     )
-    # else:
-    #     app.arborist_client = ArboristClient(
-    #         logger=get_logger("gen3workflow.gen3authz", log_level="debug"),
-    #     )
+    get_logger("gen3workflow", log_level="debug" if debug == True else "info")
 
     return app
 

gen3workflow/config-default.yaml

@@ -3,19 +3,8 @@
 ####################
 
 DEBUG: true
-# TEST_KEEP_DB: false
 DOCS_URL_PREFIX: /gen3workflow
 
-# # ignored if already set as an environment variable
-# ARBORIST_URL:
-
-# DB_DRIVER: postgresql
-# DB_HOST: localhost
-# DB_PORT: 5432
-# DB_USER: postgres
-# DB_PASSWORD: postgres
-# DB_DATABASE: gen3workflow
-
 ####################
 # GA4GH TES        #
 ####################

gen3workflow/config.py

@@ -1,7 +1,4 @@
-from itertools import chain
-# from jsonschema import validate
 import os
-from sqlalchemy.engine.url import make_url, URL
 
 from gen3config import Config
 
@@ -18,54 +15,12 @@ class Gen3WorkflowConfig(Config):
     def __init__(self, *args, **kwargs):
         super(Gen3WorkflowConfig, self).__init__(*args, **kwargs)
 
-    # def post_process(self) -> None:
-    #     # generate DB_URL from DB configs or env vars
-    #     self["DB_URL"] = make_url(
-    #         URL(
-    #             drivername=os.environ.get("DB_DRIVER", self["DB_DRIVER"]),
-    #             host=os.environ.get("DB_HOST", self["DB_HOST"]),
-    #             port=os.environ.get("DB_PORT", self["DB_PORT"]),
-    #             username=os.environ.get("DB_USER", self["DB_USER"]),
-    #             password=os.environ.get("DB_PASSWORD", self["DB_PASSWORD"]),
-    #             database=os.environ.get("DB_DATABASE", self["DB_DATABASE"]),
-    #         ),
-    #     )
-
     def validate(self) -> None:
         """
         Perform a series of sanity checks on a loaded config.
         """
         logger.info("Validating configuration")
-        # self.validate_redirect_configs()
-
-    # def validate_redirect_configs(self):
-    #     """
-    #     Example:
-    #         REDIRECT_CONFIGS:
-    #             my_redirect:
-    #                 redirect_url: http://url.com
-    #                 params:
-    #                     - request_id
-    #     """
-    #     schema = {
-    #         "type": "object",
-    #         "patternProperties": {
-    #             ".*": {  # unique ID
-    #                 "type": "object",
-    #                 "additionalProperties": False,
-    #                 "required": ["redirect_url"],
-    #                 "properties": {
-    #                     "redirect_url": NON_EMPTY_STRING_SCHEMA,
-    #                     "params": {
-    #                         "type": "array",
-    #                         "items": {"enum": self.allowed_params_from_db},
-    #                     },
-    #                 },
-    #             }
-    #         },
-    #     }
-    #     validate(instance=self["REDIRECT_CONFIGS"], schema=schema)
-
+        # will do more here when there is more config
 
 config = Gen3WorkflowConfig(DEFAULT_CFG_PATH)
 try:

gen3workflow/routes/system.py

@@ -22,5 +22,5 @@ async def get_status(request: Request) -> dict:
         logger.error(
             f"Expected status code {HTTP_200_OK} from '{tes_status_url}' and got {res.status_code}: {res.text}"
         )
-        raise HTTPException(HTTP_500_INTERNAL_SERVER_ERROR, "TODO")
+        raise HTTPException(HTTP_500_INTERNAL_SERVER_ERROR, "Unable to reach TES API")
     return dict(status="OK")

gunicorn.conf.py

@@ -1,23 +1,12 @@
 import logging
-import multiprocessing
 
 import cdislogging
 import gunicorn.glogging
-# from prometheus_client import multiprocess
 
 # do not overwrite gunicorn's `config`
 from gen3workflow.config import config as app_config
 
 
-# TODO add metrics support
-# def child_exit(server, worker):
-#     """
-#     Required for Prometheus multiprocess setup
-#     See: https://prometheus.github.io/client_python/multiprocess/
-#     """
-#     multiprocess.mark_process_dead(worker.pid)
-
-
 class CustomLogger(gunicorn.glogging.Logger):
     """
     Initialize root and gunicorn loggers with cdislogging configuration.
@@ -55,8 +44,7 @@ def __init__(self, cfg):
 wsgi_app = "gen3workflow.app:app"
 bind = "0.0.0.0:8000"
 
-# NOTE: This is always more than 2
-workers = 2  # multiprocessing.cpu_count() * 2 + 1
+workers = 2
 
 # default was `30` for the 2 below
 timeout = 90

pyproject.toml

@@ -19,7 +19,6 @@ fastapi = "<1"
 gen3authz = "<3"
 gen3config = "<2"
 httpx = "<1"
-# requests = "<3"
 uvicorn = "<1"
 gunicorn = "<24"
 
@@ -28,10 +27,6 @@ pytest = "<9"
 pytest-asyncio = "<1"
 pytest-cov = "<6"
 
-#[tool.poetry.plugins."gen3workflow.modules"]
-#"system" = "gen3workflow.routes.system"
-#"ga4gh_tes" = "gen3workflow.routes.ga4gh_tes"
-
 [build-system]
 requires = ["poetry<2"]
 build-backend = "poetry.masonry.api"

tests/conftest.py

@@ -1,8 +1,7 @@
 import os
 from unittest.mock import MagicMock
 from urllib.parse import urlparse
-# from alembic.config import main as alembic_main
-# import copy
+
 from fastapi import Request
 import httpx
 import pytest_asyncio
@@ -18,24 +17,6 @@
 from gen3workflow.config import config
 
 
-# @pytest.fixture(autouse=True, scope="session")
-# def setup_test_database():
-#     """
-#     At teardown, restore original config and reset test DB.
-#     """
-#     saved_config = copy.deepcopy(config._configs)
-
-#     alembic_main(["--raiseerr", "upgrade", "head"])
-
-#     yield
-
-#     # restore old configs
-#     config.update(saved_config)
-
-#     if not config["TEST_KEEP_DB"]:
-#         alembic_main(["--raiseerr", "downgrade", "base"])
-
-
 def mock_tes_server_request(
     method: str, path: str, query_params: str, body: str, status_code: int
 ):
@@ -92,12 +73,13 @@ async def client(request):
         status_code = request.param.get("status_code", 200)
 
     async def handle_request(request: Request):
-        print(
-            f"Mocking request '{request.method} {request.url}' to return code {status_code}"
-        )
         url = str(request.url)
-        parsed_url = urlparse(url)
+
         if url.startswith(config["TES_SERVER_URL"]):
+            print(
+                f"Mocking request '{request.method} {url}' to return code {status_code}"
+            )
+            parsed_url = urlparse(url)
             path = url[len(config["TES_SERVER_URL"]) :].split("?")[0]
             return mock_tes_server_request(
                 method=request.method,
@@ -106,90 +88,13 @@ async def handle_request(request: Request):
                 body=request.content.decode(),
                 status_code=status_code,
             )
-        return await real_httpx_client.get("/_status")
+        else:
+            print(f"Not mocking request '{request.method} {url}'")
+            httpx_client_function = getattr(httpx.AsyncClient(), request.method.lower())
+            return await httpx_client_function(url)
 
     mock_httpx_client = httpx.AsyncClient(transport=httpx.MockTransport(handle_request))
     app = get_app(httpx_client=mock_httpx_client)
-    async with httpx.AsyncClient(app=app, base_url="http://test") as real_httpx_client:
-        real_httpx_client.status_code = status_code
+    async with httpx.AsyncClient(app=app, base_url="http://test-gen3-wf") as real_httpx_client:
+        real_httpx_client.status_code = status_code  # for easier access to the param in the tests
         yield real_httpx_client
-
-
-# @pytest.fixture(autouse=True)
-# def clean_db():
-#     """
-#     Before each test, delete all existing requests from the DB
-#     """
-#     # The code below doesn't work because of this issue
-#     # https://github.com/encode/starlette/issues/440, so for now reset
-#     # using alembic.
-#     # pytest-asyncio = "^0.14.0"
-#     # from gen3workflow.models import Request as RequestModel
-#     # @pytest.mark.asyncio
-#     # async def clean_db():
-#     #     await RequestModel.delete.gino.all()
-#     #     yield
-
-#     alembic_main(["--raiseerr", "downgrade", "base"])
-#     alembic_main(["--raiseerr", "upgrade", "head"])
-
-#     yield
-
-
-# @pytest.fixture(scope="function")
-# def mock_arborist_requests(request):
-#     """
-#     This fixture returns a function which you call to mock the call to
-#     arborist client's auth_request method.
-#     By default, it returns a 200 response. If parameter "authorized" is set
-#     to False, it raises a 401 error.
-#     """
-
-#     def do_patch(authorized=True):
-#         # URLs to reponses: { URL: { METHOD: ( content, code ) } }
-#         urls_to_responses = {
-#             "http://arborist-service/auth/request": {
-#                 "POST": ({"auth": authorized}, 200)
-#             },
-#         }
-
-#         def make_mock_response(method, url, *args, **kwargs):
-#             method = method.upper()
-#             mocked_response = MagicMock(requests.Response)
-
-#             if url not in urls_to_responses:
-#                 mocked_response.status_code = 404
-#                 mocked_response.text = "NOT FOUND"
-#             elif method not in urls_to_responses[url]:
-#                 mocked_response.status_code = 405
-#                 mocked_response.text = "METHOD NOT ALLOWED"
-#             else:
-#                 content, code = urls_to_responses[url][method]
-#                 mocked_response.status_code = code
-#                 if isinstance(content, dict):
-#                     mocked_response.json.return_value = content
-#                 else:
-#                     mocked_response.text = content
-
-#             return mocked_response
-
-#         mocked_method = AsyncMock(side_effect=make_mock_response)
-#         patch_method = patch(
-#             "gen3authz.client.arborist.async_client.httpx.AsyncClient.request",
-#             mocked_method,
-#         )
-
-#         patch_method.start()
-#         request.addfinalizer(patch_method.stop)
-
-#     return do_patch
-
-
-# @pytest.fixture(autouse=True)
-# def arborist_authorized(mock_arborist_requests):
-#     """
-#     By default, mocked arborist calls return Authorized.
-#     To mock an unauthorized response, use fixture
-#     "mock_arborist_requests(authorized=False)" in the test itself
-#     """
-#     mock_arborist_requests()

tests/test-gen3workflow-config.yaml

@@ -1 +1 @@
-TES_SERVER_URL: http://test-server/tes
+TES_SERVER_URL: http://external-tes-server/tes