Merge pull request #188 from SciCatProject/fix-backend-ci-tests

Attempt to fix backend ci tests

docs/release-notes.rst

@@ -51,6 +51,8 @@ Breaking changes
 Bugfixes
 ~~~~~~~~
 
+* Fixed testing setup to explicitly define user accounts in the test backend.
+
 Documentation
 ~~~~~~~~~~~~~
 

src/scitacean/client.py

@@ -1108,7 +1108,7 @@ def _log_in_via_users_login(
     )
     if not response.ok:
         get_logger().info(
-            "Failed to log in via endpoint Users/login: %s", response.json()["error"]
+            "Failed to log in via endpoint Users/login: %s", response.text
         )
     return response
 
@@ -1129,9 +1129,7 @@ def _log_in_via_auth_msad(
         timeout=timeout.seconds,
     )
     if not response.ok:
-        get_logger().error(
-            "Failed to log in via auth/msad: %s", response.json()["error"]
-        )
+        get_logger().error("Failed to log in via auth/msad: %s", response.text)
     return response
 
 
@@ -1159,7 +1157,7 @@ def _get_token(
     if response.ok:
         return str(response.json()["access_token"])
 
-    get_logger().error("Failed log in:  %s", response.json()["error"])
+    get_logger().error("Failed log in:  %s", response.text)
     raise ScicatLoginError(response.content)
 
 

src/scitacean/testing/backend/__init__.py

@@ -63,7 +63,6 @@ def test_manual_client(require_scicat_backend, scicat_access):
 
    add_pytest_option
    backend_enabled
-   can_connect
    configure
    skip_if_not_backend
    start_backend
@@ -73,7 +72,6 @@ def test_manual_client(require_scicat_backend, scicat_access):
 
 from . import config, seed
 from ._backend import (
-    can_connect,
     configure,
     start_backend,
     stop_backend,
@@ -84,7 +82,6 @@ def test_manual_client(require_scicat_backend, scicat_access):
 __all__ = [
     "add_pytest_option",
     "backend_enabled",
-    "can_connect",
     "config",
     "configure",
     "seed",

src/scitacean/testing/backend/_backend.py

@@ -4,6 +4,7 @@
 import os
 import time
 from copy import deepcopy
+from pathlib import Path
 from typing import Any, Dict, Union
 from urllib.parse import urljoin
 
@@ -35,7 +36,9 @@ def _docker_compose_template() -> Dict[str, Any]:
     return template  # type: ignore[no-any-return]
 
 
-def _apply_config(template: Dict[str, Any]) -> Dict[str, Any]:
+def _apply_config(
+    template: Dict[str, Any], account_config_path: Path
+) -> Dict[str, Any]:
     res = deepcopy(template)
     scicat = res["services"]["scicat"]
     ports = scicat["ports"][0].split(":")
@@ -46,6 +49,10 @@ def _apply_config(template: Dict[str, Any]) -> Dict[str, Any]:
     env["PID_PREFIX"] = config.PID_PREFIX
     env["SITE"] = config.SITE
 
+    scicat["volumes"] = [
+        f"{account_config_path}:/home/node/app/functionalAccounts.json",
+    ]
+
     return res
 
 
@@ -57,7 +64,9 @@ def configure(target_path: _PathLike) -> None:
     target_path:
         Generate a docker-compose file at this path.
     """
-    c = yaml.dump(_apply_config(_docker_compose_template()))
+    account_config_path = Path(target_path).parent / "functionalAccounts.json"
+    config.dump_account_config(account_config_path)
+    c = yaml.dump(_apply_config(_docker_compose_template(), account_config_path))
     if "PLACEHOLDER" in c:
         raise RuntimeError("Incorrect config")
 
@@ -75,13 +84,14 @@ def stop_backend(docker_compose_file: _PathLike) -> None:
     docker_compose_down(docker_compose_file)
 
 
-def can_connect() -> bool:
+def _can_connect() -> tuple[bool, str]:
     """Test the connection to the testing SciCat backend.
 
     Returns
     -------
     :
-        True if the backend is reachable, False otherwise.
+        The first element indicates whether the connection was successful.
+        The second element is an error message.
     """
     scicat_access = config.local_access("user1")
     try:
@@ -90,9 +100,11 @@ def can_connect() -> bool:
             json=scicat_access.user.credentials,
             timeout=0.5,
         )
-    except requests.ConnectionError:
-        return False
-    return response.ok
+    except requests.ConnectionError as err:
+        return False, str(err)
+    if response.ok:
+        return True, ""
+    return False, str(f"{response}: {response.text}")
 
 
 def wait_until_backend_is_live(max_time: float, n_tries: int) -> None:
@@ -116,8 +128,9 @@ def wait_until_backend_is_live(max_time: float, n_tries: int) -> None:
         If no connection can be made within the time limit.
     """
     for _ in range(n_tries):
-        if can_connect():
+        if _can_connect()[0]:
             return
         time.sleep(max_time / n_tries)
-    if not can_connect():
-        raise RuntimeError("Cannot connect to backend")
+    ok, err = _can_connect()
+    if not ok:
+        raise RuntimeError(f"Cannot connect to backend: {err}")

src/scitacean/testing/backend/config.py

@@ -2,8 +2,10 @@
 # Copyright (c) 2024 SciCat Project (https://github.com/SciCatProject/scitacean)
 """Backend configuration."""
 
+import json
 from dataclasses import dataclass
-from typing import Dict
+from pathlib import Path
+from typing import Union
 
 
 @dataclass
@@ -23,7 +25,7 @@ class SciCatUser:
     group: str
 
     @property
-    def credentials(self) -> Dict[str, str]:
+    def credentials(self) -> dict[str, str]:
         """Return login credentials for this user.
 
         User as
@@ -37,6 +39,16 @@ def credentials(self) -> Dict[str, str]:
             "password": self.password,
         }
 
+    def dump(self) -> dict[str, Union[str, bool]]:
+        """Return a dict that can be serialized to functionalAccounts.json."""
+        return {
+            "username": self.username,
+            "password": self.password,
+            "email": self.email,
+            "role": self.group,
+            "global": False,
+        }
+
 
 # see https://github.com/SciCatProject/scicat-backend-next/blob/master/src/config/configuration.ts
 USERS = {
@@ -116,3 +128,9 @@ def local_access(user: str) -> SciCatAccess:
         Parameters for the local SciCat backend.
     """
     return SciCatAccess(url=f"http://localhost:{SCICAT_PORT}/api/v3/", user=USERS[user])
+
+
+def dump_account_config(path: Path) -> None:
+    """Write a functional account config for the backend."""
+    with path.open("w") as f:
+        json.dump([user.dump() for user in USERS.values()], f)

src/scitacean/testing/backend/docker-compose-backend-template.yaml

@@ -11,9 +11,7 @@
 
 services:
   mongodb:
-    image: bitnami/mongodb:4.2
-    volumes:
-      - mongodb_data:/bitnami
+    image: mongo:latest
     ports:
       - "27017:27017"
 
@@ -26,6 +24,7 @@ services:
       - 3000:PLACEHOLDER
     environment:
       DOI_PREFIX: DOI.SAMPLE.PREFIX
+      ELASTICSEARCH_ENABLED: no
       HTTP_MAX_REDIRECTS: 1  # Scitacean should not require redirects
       HTTP_TIMEOUT: 5000  # in ms
       JWT_EXPIRES_IN: 3600  # in s (expiry of token)
@@ -37,7 +36,3 @@ services:
       PID_PREFIX: PLACEHOLDER
       SITE: PLACEHOLDER
       PORT: PLACEHOLDER
-
-volumes:
-  mongodb_data:
-    driver: local

src/scitacean/testing/sftp/__init__.py

@@ -102,15 +102,13 @@ def test_sftp_upload(
     IgnorePolicy,
     SFTPAccess,
     SFTPUser,
-    can_connect,
     configure,
     local_access,
     wait_until_sftp_server_is_live,
 )
 
 __all__ = [
     "add_pytest_option",
-    "can_connect",
     "configure",
     "local_access",
     "sftp_enabled",

src/scitacean/testing/sftp/_sftp.py

@@ -108,7 +108,7 @@ def configure(target_dir: Union[Path, str]) -> Path:
     return config_target
 
 
-def can_connect(sftp_access: SFTPAccess) -> bool:
+def _can_connect(sftp_access: SFTPAccess) -> bool:
     try:
         _make_client(sftp_access)
     except paramiko.SSHException:
@@ -121,10 +121,10 @@ def wait_until_sftp_server_is_live(
 ) -> None:
     # The container takes a while to be fully live.
     for _ in range(n_tries):
-        if can_connect(sftp_access):
+        if _can_connect(sftp_access):
             return
         time.sleep(max_time / n_tries)
-    if not can_connect(sftp_access):
+    if not _can_connect(sftp_access):
         raise RuntimeError("Cannot connect to SFTP server")
 
 

src/scitacean/testing/ssh/__init__.py

@@ -102,15 +102,13 @@ def test_ssh_upload(
     IgnorePolicy,
     SSHAccess,
     SSHUser,
-    can_connect,
     configure,
     local_access,
     wait_until_ssh_server_is_live,
 )
 
 __all__ = [
     "add_pytest_option",
-    "can_connect",
     "configure",
     "local_access",
     "ssh_enabled",

src/scitacean/testing/ssh/_ssh.py

@@ -98,7 +98,7 @@ def configure(target_dir: Union[Path, str]) -> Path:
     return config_target
 
 
-def can_connect(ssh_access: SSHAccess) -> bool:
+def _can_connect(ssh_access: SSHAccess) -> bool:
     try:
         _make_client(ssh_access)
     except paramiko.SSHException:
@@ -111,10 +111,10 @@ def wait_until_ssh_server_is_live(
 ) -> None:
     # The container takes a while to be fully live.
     for _ in range(n_tries):
-        if can_connect(ssh_access):
+        if _can_connect(ssh_access):
             return
         time.sleep(max_time / n_tries)
-    if not can_connect(ssh_access):
+    if not _can_connect(ssh_access):
         raise RuntimeError("Cannot connect to SSH server")