From 546cd7b087196b4a61b1597fcd49608802b711c7 Mon Sep 17 00:00:00 2001 From: Jan-Lukas Wynen Date: Fri, 19 Jan 2024 10:48:58 +0100 Subject: [PATCH 01/10] Return error message in can_connect --- src/scitacean/testing/backend/__init__.py | 3 --- src/scitacean/testing/backend/_backend.py | 20 ++++++++++++-------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/src/scitacean/testing/backend/__init__.py b/src/scitacean/testing/backend/__init__.py index 619f6623..612a162e 100644 --- a/src/scitacean/testing/backend/__init__.py +++ b/src/scitacean/testing/backend/__init__.py @@ -63,7 +63,6 @@ def test_manual_client(require_scicat_backend, scicat_access): add_pytest_option backend_enabled - can_connect configure skip_if_not_backend start_backend @@ -73,7 +72,6 @@ def test_manual_client(require_scicat_backend, scicat_access): from . import config, seed from ._backend import ( - can_connect, configure, start_backend, stop_backend, @@ -84,7 +82,6 @@ def test_manual_client(require_scicat_backend, scicat_access): __all__ = [ "add_pytest_option", "backend_enabled", - "can_connect", "config", "configure", "seed", diff --git a/src/scitacean/testing/backend/_backend.py b/src/scitacean/testing/backend/_backend.py index eb0a00ea..866666fa 100644 --- a/src/scitacean/testing/backend/_backend.py +++ b/src/scitacean/testing/backend/_backend.py @@ -75,13 +75,14 @@ def stop_backend(docker_compose_file: _PathLike) -> None: docker_compose_down(docker_compose_file) -def can_connect() -> bool: +def _can_connect() -> tuple[bool, str]: """Test the connection to the testing SciCat backend. Returns ------- : - True if the backend is reachable, False otherwise. + The first element indicates whether the connection was successful. + The second element is an error message. """ scicat_access = config.local_access("user1") try: @@ -90,9 +91,11 @@ def can_connect() -> bool: json=scicat_access.user.credentials, timeout=0.5, ) - except requests.ConnectionError: - return False - return response.ok + except requests.ConnectionError as err: + return False, str(err) + if response.ok: + return True, "" + return False, str(f"{response}: {response.text}") def wait_until_backend_is_live(max_time: float, n_tries: int) -> None: @@ -116,8 +119,9 @@ def wait_until_backend_is_live(max_time: float, n_tries: int) -> None: If no connection can be made within the time limit. """ for _ in range(n_tries): - if can_connect(): + if _can_connect()[0]: return time.sleep(max_time / n_tries) - if not can_connect(): - raise RuntimeError("Cannot connect to backend") + ok, err = _can_connect() + if not ok: + raise RuntimeError(f"Cannot connect to backend: {err}") From dc14ccc0d5ff1dcf21933d769ff8c8c1a0bc32f4 Mon Sep 17 00:00:00 2001 From: Jan-Lukas Wynen Date: Fri, 19 Jan 2024 10:50:33 +0100 Subject: [PATCH 02/10] Make can_connect private --- src/scitacean/testing/sftp/__init__.py | 2 -- src/scitacean/testing/sftp/_sftp.py | 6 +++--- src/scitacean/testing/ssh/__init__.py | 2 -- src/scitacean/testing/ssh/_ssh.py | 6 +++--- 4 files changed, 6 insertions(+), 10 deletions(-) diff --git a/src/scitacean/testing/sftp/__init__.py b/src/scitacean/testing/sftp/__init__.py index 2ff3043a..286f3380 100644 --- a/src/scitacean/testing/sftp/__init__.py +++ b/src/scitacean/testing/sftp/__init__.py @@ -102,7 +102,6 @@ def test_sftp_upload( IgnorePolicy, SFTPAccess, SFTPUser, - can_connect, configure, local_access, wait_until_sftp_server_is_live, @@ -110,7 +109,6 @@ def test_sftp_upload( __all__ = [ "add_pytest_option", - "can_connect", "configure", "local_access", "sftp_enabled", diff --git a/src/scitacean/testing/sftp/_sftp.py b/src/scitacean/testing/sftp/_sftp.py index 1c9dcfaf..71464286 100644 --- a/src/scitacean/testing/sftp/_sftp.py +++ b/src/scitacean/testing/sftp/_sftp.py @@ -108,7 +108,7 @@ def configure(target_dir: Union[Path, str]) -> Path: return config_target -def can_connect(sftp_access: SFTPAccess) -> bool: +def _can_connect(sftp_access: SFTPAccess) -> bool: try: _make_client(sftp_access) except paramiko.SSHException: @@ -121,10 +121,10 @@ def wait_until_sftp_server_is_live( ) -> None: # The container takes a while to be fully live. for _ in range(n_tries): - if can_connect(sftp_access): + if _can_connect(sftp_access): return time.sleep(max_time / n_tries) - if not can_connect(sftp_access): + if not _can_connect(sftp_access): raise RuntimeError("Cannot connect to SFTP server") diff --git a/src/scitacean/testing/ssh/__init__.py b/src/scitacean/testing/ssh/__init__.py index 56362547..7624220d 100644 --- a/src/scitacean/testing/ssh/__init__.py +++ b/src/scitacean/testing/ssh/__init__.py @@ -102,7 +102,6 @@ def test_ssh_upload( IgnorePolicy, SSHAccess, SSHUser, - can_connect, configure, local_access, wait_until_ssh_server_is_live, @@ -110,7 +109,6 @@ def test_ssh_upload( __all__ = [ "add_pytest_option", - "can_connect", "configure", "local_access", "ssh_enabled", diff --git a/src/scitacean/testing/ssh/_ssh.py b/src/scitacean/testing/ssh/_ssh.py index f16eca57..3e6aa580 100644 --- a/src/scitacean/testing/ssh/_ssh.py +++ b/src/scitacean/testing/ssh/_ssh.py @@ -98,7 +98,7 @@ def configure(target_dir: Union[Path, str]) -> Path: return config_target -def can_connect(ssh_access: SSHAccess) -> bool: +def _can_connect(ssh_access: SSHAccess) -> bool: try: _make_client(ssh_access) except paramiko.SSHException: @@ -111,10 +111,10 @@ def wait_until_ssh_server_is_live( ) -> None: # The container takes a while to be fully live. for _ in range(n_tries): - if can_connect(ssh_access): + if _can_connect(ssh_access): return time.sleep(max_time / n_tries) - if not can_connect(ssh_access): + if not _can_connect(ssh_access): raise RuntimeError("Cannot connect to SSH server") From 3a9695ed9b2f5c0a6f2cdb33805ad304c071e7f6 Mon Sep 17 00:00:00 2001 From: Jan-Lukas Wynen Date: Fri, 19 Jan 2024 12:05:57 +0100 Subject: [PATCH 03/10] Disable elasticsearch --- .../testing/backend/docker-compose-backend-template.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/src/scitacean/testing/backend/docker-compose-backend-template.yaml b/src/scitacean/testing/backend/docker-compose-backend-template.yaml index 92293199..81c57b9d 100644 --- a/src/scitacean/testing/backend/docker-compose-backend-template.yaml +++ b/src/scitacean/testing/backend/docker-compose-backend-template.yaml @@ -26,6 +26,7 @@ services: - 3000:PLACEHOLDER environment: DOI_PREFIX: DOI.SAMPLE.PREFIX + ELASTICSEARCH_ENABLED: no HTTP_MAX_REDIRECTS: 1 # Scitacean should not require redirects HTTP_TIMEOUT: 5000 # in ms JWT_EXPIRES_IN: 3600 # in s (expiry of token) From 93e400281b988673c2986e0d047beec22f1f7dac Mon Sep 17 00:00:00 2001 From: Jan-Lukas Wynen Date: Fri, 19 Jan 2024 13:10:57 +0100 Subject: [PATCH 04/10] Use latest mongo version --- .../testing/backend/docker-compose-backend-template.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scitacean/testing/backend/docker-compose-backend-template.yaml b/src/scitacean/testing/backend/docker-compose-backend-template.yaml index 81c57b9d..a6169bf1 100644 --- a/src/scitacean/testing/backend/docker-compose-backend-template.yaml +++ b/src/scitacean/testing/backend/docker-compose-backend-template.yaml @@ -11,7 +11,7 @@ services: mongodb: - image: bitnami/mongodb:4.2 + image: mongo:latest volumes: - mongodb_data:/bitnami ports: From db3b831990364a2d60761e36215100385fe9ae49 Mon Sep 17 00:00:00 2001 From: Jan-Lukas Wynen Date: Fri, 19 Jan 2024 16:09:40 +0100 Subject: [PATCH 05/10] Log text instead of json field --- src/scitacean/client.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/scitacean/client.py b/src/scitacean/client.py index a483fef4..3206ae77 100644 --- a/src/scitacean/client.py +++ b/src/scitacean/client.py @@ -1108,7 +1108,7 @@ def _log_in_via_users_login( ) if not response.ok: get_logger().info( - "Failed to log in via endpoint Users/login: %s", response.json()["error"] + "Failed to log in via endpoint Users/login: %s", response.text ) return response @@ -1129,9 +1129,7 @@ def _log_in_via_auth_msad( timeout=timeout.seconds, ) if not response.ok: - get_logger().error( - "Failed to log in via auth/msad: %s", response.json()["error"] - ) + get_logger().error("Failed to log in via auth/msad: %s", response.text) return response @@ -1159,7 +1157,7 @@ def _get_token( if response.ok: return str(response.json()["access_token"]) - get_logger().error("Failed log in: %s", response.json()["error"]) + get_logger().error("Failed log in: %s", response.text) raise ScicatLoginError(response.content) From 5215b18052d82f9522e9e1a70c5f25fd576f0afc Mon Sep 17 00:00:00 2001 From: Jan-Lukas Wynen Date: Fri, 19 Jan 2024 16:29:16 +0100 Subject: [PATCH 06/10] Do not persist mongo db --- .../testing/backend/docker-compose-backend-template.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/scitacean/testing/backend/docker-compose-backend-template.yaml b/src/scitacean/testing/backend/docker-compose-backend-template.yaml index a6169bf1..ac693617 100644 --- a/src/scitacean/testing/backend/docker-compose-backend-template.yaml +++ b/src/scitacean/testing/backend/docker-compose-backend-template.yaml @@ -12,8 +12,6 @@ services: mongodb: image: mongo:latest - volumes: - - mongodb_data:/bitnami ports: - "27017:27017" @@ -38,7 +36,3 @@ services: PID_PREFIX: PLACEHOLDER SITE: PLACEHOLDER PORT: PLACEHOLDER - -volumes: - mongodb_data: - driver: local From 399fab8cd11ebb2ab2c11bb8f05ace68bd3e4a22 Mon Sep 17 00:00:00 2001 From: Jan-Lukas Wynen Date: Fri, 19 Jan 2024 16:58:37 +0100 Subject: [PATCH 07/10] Use functionalAccounts.json --- src/scitacean/testing/backend/_backend.py | 13 +++++++++++-- src/scitacean/testing/backend/config.py | 21 +++++++++++++++++++-- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/src/scitacean/testing/backend/_backend.py b/src/scitacean/testing/backend/_backend.py index 866666fa..ecf3309e 100644 --- a/src/scitacean/testing/backend/_backend.py +++ b/src/scitacean/testing/backend/_backend.py @@ -4,6 +4,7 @@ import os import time from copy import deepcopy +from pathlib import Path from typing import Any, Dict, Union from urllib.parse import urljoin @@ -35,7 +36,9 @@ def _docker_compose_template() -> Dict[str, Any]: return template # type: ignore[no-any-return] -def _apply_config(template: Dict[str, Any]) -> Dict[str, Any]: +def _apply_config( + template: Dict[str, Any], account_config_path: Path +) -> Dict[str, Any]: res = deepcopy(template) scicat = res["services"]["scicat"] ports = scicat["ports"][0].split(":") @@ -46,6 +49,10 @@ def _apply_config(template: Dict[str, Any]) -> Dict[str, Any]: env["PID_PREFIX"] = config.PID_PREFIX env["SITE"] = config.SITE + scicat["volumes"] = [ + f"/home/node/app/functionalAccounts.json:{account_config_path}", + ] + return res @@ -57,7 +64,9 @@ def configure(target_path: _PathLike) -> None: target_path: Generate a docker-compose file at this path. """ - c = yaml.dump(_apply_config(_docker_compose_template())) + account_config_path = Path(target_path).parent / "functionalAccounts.json" + config.dump_account_config(account_config_path) + c = yaml.dump(_apply_config(_docker_compose_template(), account_config_path)) if "PLACEHOLDER" in c: raise RuntimeError("Incorrect config") diff --git a/src/scitacean/testing/backend/config.py b/src/scitacean/testing/backend/config.py index 387269c3..e800f8ba 100644 --- a/src/scitacean/testing/backend/config.py +++ b/src/scitacean/testing/backend/config.py @@ -2,8 +2,9 @@ # Copyright (c) 2024 SciCat Project (https://github.com/SciCatProject/scitacean) """Backend configuration.""" +import json from dataclasses import dataclass -from typing import Dict +from pathlib import Path @dataclass @@ -23,7 +24,7 @@ class SciCatUser: group: str @property - def credentials(self) -> Dict[str, str]: + def credentials(self) -> dict[str, str]: """Return login credentials for this user. User as @@ -37,6 +38,16 @@ def credentials(self) -> Dict[str, str]: "password": self.password, } + def dump(self) -> dict[str, str]: + """Return a dict that can be serialized to functionalAccounts.json.""" + return { + "username": self.username, + "password": self.password, + "email": self.email, + "role": self.group, + "global": False, + } + # see https://github.com/SciCatProject/scicat-backend-next/blob/master/src/config/configuration.ts USERS = { @@ -116,3 +127,9 @@ def local_access(user: str) -> SciCatAccess: Parameters for the local SciCat backend. """ return SciCatAccess(url=f"http://localhost:{SCICAT_PORT}/api/v3/", user=USERS[user]) + + +def dump_account_config(path: Path) -> None: + """Write a functional account config for the backend.""" + with path.open("w") as f: + json.dump([user.dump() for user in USERS.values()], f) From 7001bd40a029238192903f5c64243cf89383268d Mon Sep 17 00:00:00 2001 From: Jan-Lukas Wynen Date: Mon, 22 Jan 2024 14:38:56 +0100 Subject: [PATCH 08/10] Fix volume declaration --- src/scitacean/testing/backend/_backend.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scitacean/testing/backend/_backend.py b/src/scitacean/testing/backend/_backend.py index ecf3309e..d565ac96 100644 --- a/src/scitacean/testing/backend/_backend.py +++ b/src/scitacean/testing/backend/_backend.py @@ -50,7 +50,7 @@ def _apply_config( env["SITE"] = config.SITE scicat["volumes"] = [ - f"/home/node/app/functionalAccounts.json:{account_config_path}", + f"{account_config_path}:/home/node/app/functionalAccounts.json", ] return res From 33d3d95cf4a866ff44b4d9704f543ad5dbcc6ffe Mon Sep 17 00:00:00 2001 From: Jan-Lukas Wynen Date: Mon, 22 Jan 2024 14:42:36 +0100 Subject: [PATCH 09/10] Fix type error --- src/scitacean/testing/backend/config.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/scitacean/testing/backend/config.py b/src/scitacean/testing/backend/config.py index e800f8ba..f7216d1d 100644 --- a/src/scitacean/testing/backend/config.py +++ b/src/scitacean/testing/backend/config.py @@ -5,6 +5,7 @@ import json from dataclasses import dataclass from pathlib import Path +from typing import Union @dataclass @@ -38,7 +39,7 @@ def credentials(self) -> dict[str, str]: "password": self.password, } - def dump(self) -> dict[str, str]: + def dump(self) -> dict[str, Union[str, bool]]: """Return a dict that can be serialized to functionalAccounts.json.""" return { "username": self.username, From dfb8a812fa2bdcf835ee4b6cd77e7009e9e84101 Mon Sep 17 00:00:00 2001 From: Jan-Lukas Wynen Date: Mon, 22 Jan 2024 14:47:55 +0100 Subject: [PATCH 10/10] Add release note --- docs/release-notes.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/release-notes.rst b/docs/release-notes.rst index 96855948..0cfa5cd7 100644 --- a/docs/release-notes.rst +++ b/docs/release-notes.rst @@ -51,6 +51,8 @@ Breaking changes Bugfixes ~~~~~~~~ +* Fixed testing setup to explicitly define user accounts in the test backend. + Documentation ~~~~~~~~~~~~~