WIP: Keycloak logic, bump deps, filter: count feature + expand parame…

…ters and apispec

compose.test.yml

@@ -27,8 +27,6 @@ services:
       - KC_HOST=http://keycloak:8080/
       - KC_REALM=3TR
       - KC_PUBLIC_KEY=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA0juOxC3+S97HFnlmRgWqUaSpTlscaH6IQaoLuqXFYakDJCV6WU0andDRQFJH8CeOaiVx84J1g7m/cNzxX6Ilz+0MZ6mnBFShaGY0+Qk6zIipFU2ehWQtAm0IWGwQipXC2enlXLIglRXJJepH7jOxC+fyY+f++09+68KuNAAUL8IjvZRMCu/AV3qlm6zdeCztTxy8eiBH9shg+wNLRpWczfMBAHetqqpzy9kVhVizHFdSxd21yESRce7iUQn+KzwsGzBve0Ds68GzhgyUXYjXV/sQ3jaNqDAy+qiCkv0nXKPBxVFUstPQQJvhlQ4gZW7SUdIV3IynBXckpGQhE24tcQIDAQAB
-      - KC_ADMIN=admin
-      - KC_ADMIN_PASSWORD=1234
       - KC_CLIENT_ID=submission_client
       - KC_CLIENT_SECRET=38wBvfSVS7fa3LprqSL5YCDPaMUY1bTl
     ports:

src/biodm/components/services/dbservice.py

@@ -4,11 +4,11 @@
 from typing import Callable, List, Sequence, Any, Dict, overload, Literal, Type, Set
 
 from sqlalchemy import select, delete, or_, func
-from sqlalchemy.exc import IntegrityError
+from sqlalchemy.exc import SQLAlchemyError
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.ext.hybrid import hybrid_property
 from sqlalchemy.orm import (
-    load_only, selectinload, joinedload, ONETOMANY, MANYTOONE, make_transient, Relationship
+    load_only, selectinload, joinedload, ONETOMANY, MANYTOONE, Relationship
 )
 from sqlalchemy.sql import Delete, Select
 from sqlalchemy.sql.selectable import Alias
@@ -17,7 +17,7 @@
 from biodm.component import ApiService
 from biodm.components import Base
 from biodm.exceptions import (
-    DataError, EndpointError, FailedCreate, FailedRead, FailedDelete, ReleaseVersionError, UpdateVersionedError, UnauthorizedError
+    DataError, EndpointError, FailedCreate, FailedRead, FailedDelete, ImplementionError, ReleaseVersionError, UpdateVersionedError, UnauthorizedError
 )
 from biodm.managers import DatabaseManager
 from biodm.tables import ListGroup, Group
@@ -57,13 +57,9 @@ async def _insert(
 
             missing = self.table.required - stmt.keys()
             raise DataError(f"{self.table.__name__} missing the following: {missing}.")
-        # May occur in some cases for versioned resources.
-        except IntegrityError as ie:
-            if 'unique' in ie.args[0].lower() and 'version' in ie.args[0]:
-                raise UpdateVersionedError(
-                    "Attempt at updating versioned resources."
-                )
-            raise FailedCreate(str(ie))
+
+        except SQLAlchemyError as se:
+            raise FailedCreate(str(se))
 
     @DatabaseManager.in_session
     async def _insert_list(
@@ -445,18 +441,18 @@ def gen_upsert_holder(
         missing_data = self.table.required - pending_keys
 
         if missing_data:
-            if all(k in pending_keys for k in self.table.pk): # pk present: UPDATE.
-                if (data.keys() - self.table.pk) and self.table.is_versioned:
-                    raise UpdateVersionedError(
-                        "Attempt at updating versioned resources detected"
-                    )
-
             # submitter_username special col
-            elif missing_data == {'submitter_username'} and self.table.has_submitter_username:
+            if missing_data == {'submitter_username'} and self.table.has_submitter_username:
                 if not user_info or not user_info.is_authenticated:
                     raise UnauthorizedError()
                 data['submitter_username'] = user_info.display_name
 
+            elif all(k in pending_keys for k in self.table.pk): # pk present: UPDATE.
+                if (data.keys() - self.table.pk) and self.table.is_versioned:
+                    raise UpdateVersionedError(
+                        "Attempt at updating versioned resources detected"
+                    )
+
             else:
                 raise DataError(f"{self.table.__name__} missing the following: {missing_data}.")
 
@@ -741,6 +737,7 @@ async def filter(
         self,
         fields: List[str],
         params: Dict[str, str],
+        count: bool = False,
         stmt_only: bool = False,
         user_info: UserInfo | None = None,
         **kwargs
@@ -752,7 +749,7 @@ async def filter(
         reverse = params.pop('reverse', None) # TODO: ?
 
         # start building statement.
-        stmt = select(self.table)
+        stmt = select(self.table).distinct()
 
         # For lower level(s) propagation.
         propagate = {"start": offset, "end": limit, "reverse": reverse}
@@ -796,7 +793,16 @@ async def filter(
 
         # if exclude:
         #     stmt = select(self.table.not_in(stmt))
+        if count: # Count can only be passed from a controller.
+            if stmt_only:
+                raise ImplementionError(
+                    "filter arguments: count cannot be used in conjunction with stmt_only !"
+                )
+            stmt = select(func.count()).select_from(stmt)
+            return await self._select(stmt)
+
         stmt = stmt.offset(offset).limit(limit)
+        # stmt = stmt.slice(offset-1, limit-1) # TODO [prio-low] investigate
         return stmt if stmt_only else await self._select_many(stmt, **kwargs)
 
     @DatabaseManager.in_session

src/biodm/components/services/kcservice.py

@@ -2,10 +2,14 @@
 from typing import Any, Dict, List
 from pathlib import Path
 
-from biodm.exceptions import DataError, UnauthorizedError
-from biodm.managers import KeycloakManager
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from biodm.components import Base
+from biodm.exceptions import DataError
+from biodm.managers import KeycloakManager, DatabaseManager
 from biodm.tables import Group, User
 from biodm.utils.security import UserInfo
+from biodm.utils.sqla import UpsertStmtValuesHolder
 from biodm.utils.utils import to_it, classproperty
 from .dbservice import CompositeEntityService
 
@@ -17,11 +21,35 @@ def kc(cls) -> KeycloakManager:
         """Return KCManager instance."""
         return cls.app.kc
 
+    # @DatabaseManager.in_session
+    # async def _insert(
+    #     self,
+    #     stmt: UpsertStmtValuesHolder,
+    #     user_info: UserInfo | None,
+    #     session: AsyncSession
+    # ) -> Base:
+    #     """INSERT one object into the DB, check token write permissions before commit."""
+    #     await self._check_permissions("write", user_info, stmt)
+    #     try:
+    #         item = await session.scalar(stmt.to_stmt(self))
+    #         if item:
+    #             return item
+
+    #         missing = self.table.required - stmt.keys()
+    #         raise DataError(f"{self.table.__name__} missing the following: {missing}.")
+    #     except:
+    # TODO: [prio-high] catch missing = 'id' case, that indicates resource couldn't be created on keycloak -> unsuficiently priviledge token.
+
     @abstractmethod
-    async def _update(self, remote_id: str, data: Dict[str, Any]):
+    async def _update(self, remote_id: str, data: Dict[str, Any], user_info: UserInfo):
         """Keycloak entity update method."""
         raise NotImplementedError
 
+    @abstractmethod
+    async def import_all(self) -> None:
+        """Import all entities of that type from keycloak."""
+        raise NotImplementedError
+
     async def sync(
         self,
         remote: Dict[str, Any],
@@ -38,11 +66,7 @@ async def sync(
             if data.get(key, None) and data.get(key, None) != remote.get(key, None)
         }
         if update:
-            if not user_info.is_admin:
-                raise UnauthorizedError(
-                    f"only administrators are allowed to update keycloak entities."
-                )
-            await self._update(remote['id'], update)
+            await self._update(remote['id'], update, user_info=user_info)
         data.update(fill)
 
     @abstractmethod
@@ -63,8 +87,11 @@ def kcpath(path) -> Path:
         """Compute keycloak path from api path."""
         return Path("/" + path.replace("__", "/"))
 
-    async def _update(self, remote_id: str, data: Dict[str, Any]):
-        return await self.kc.update_group(group_id=remote_id, data=data)
+    async def import_all(self, user_info: UserInfo) -> None:
+        raise NotImplementedError
+
+    async def _update(self, remote_id: str, data: Dict[str, Any], user_info: UserInfo):
+        return await self.kc.update_group(group_id=remote_id, data=data, user_info=user_info)
 
     async def read_or_create(
         self,
@@ -79,25 +106,22 @@ async def read_or_create(
         :type user_info: UserInfo
         """
         path = self.kcpath(data['path'])
-        group = await self.kc.get_group_by_path(str(path))
+        group = await self.kc.get_group_by_path(str(path), user_info=user_info)
 
         if group:
             await self.sync(group, data, user_info=user_info)
             return
 
-        if not user_info.is_admin:
-            raise UnauthorizedError(
-                f"group {path} does not exists, only administrators are allowed to create new ones."
-            )
-
         parent_id = None
         if not path.parent.parts == ('/',):
-            parent = await self.kc.get_group_by_path(str(path.parent))
+            parent = await self.kc.get_group_by_path(str(path.parent), user_info=user_info)
             if not parent:
                 raise DataError("Input path does not match any parent group.")
             parent_id = parent['id']
 
-        data['id'] = await self.kc.create_group(path.name, parent_id)
+        cr_id = await self.kc.create_group(path.name, parent_id, user_info=user_info)
+        if cr_id:
+            data['id'] = cr_id
 
     async def write(
         self,
@@ -108,31 +132,35 @@ async def write(
     ):
         """Create entities on Keycloak Side before passing to parent class for DB."""
         # Create on keycloak side
-        for group in to_it(data):
-            # Group first.
-            await self.read_or_create(group, user_info=user_info)
-            # Then Users.
-            for user in group.get("users", []):
-                await User.svc.read_or_create(
-                    user,
-                    user_info=user_info,
-                    groups=[group["path"]],
-                    group_ids=[group["id"]]
-                )
-
-        # Send to DB without user_info.
-        return await super().write(data, stmt_only=stmt_only, **kwargs)
+        if user_info and user_info.keycloak_admin:
+            for group in to_it(data):
+                # Group first.
+                await self.read_or_create(group, user_info=user_info)
+                # Then Users.
+                for user in group.get("users", []):
+                    await User.svc.read_or_create(
+                        user,
+                        user_info=user_info,
+                        groups=[group["path"]],
+                        group_ids=[group["id"]]
+                    )
+
+        return await super().write(data, stmt_only=stmt_only, user_info=user_info, **kwargs)
 
     async def delete(self, pk_val: List[Any], user_info: UserInfo | None = None, **_) -> None:
         """DELETE Group from DB then from Keycloak."""
         group_id = (await self.read(pk_val, fields=['id'])).id
         await super().delete(pk_val, user_info=user_info)
-        await self.kc.delete_group(group_id)
+        await self.kc.delete_group(group_id, user_info=user_info)
 
 
 class KCUserService(KCService):
-    async def _update(self, remote_id: str, data: Dict[str, Any]):
-        return await self.kc.update_user(user_id=remote_id, data=data)
+    async def import_all(self) -> None:
+        """Import all entities of that type from keycloak."""
+        raise NotImplementedError
+
+    async def _update(self, remote_id: str, data: Dict[str, Any], user_info: UserInfo):
+        return await self.kc.update_user(user_id=remote_id, data=data, user_info=user_info)
 
     async def read_or_create(
         self,
@@ -145,35 +173,26 @@ async def read_or_create(
 
         :param data: Entry object representation
         :type data: Dict[str, Any]
-        :param user_info: requesting user info
-        :type user_info: UserInfo
         :param groups: User groups names, defaults to None
         :type groups: List[str], optional
         :param group_ids: User groups ids, defaults to None
         :type group_ids: List[str], optional
         :return: User id
         :rtype: str
         """
-        user = await self.kc.get_user_by_username(data["username"])
+        user = await self.kc.get_user_by_username(data["username"], user_info=user_info)
         groups = [str(KCGroupService.kcpath(group)) for group in groups]
         if user:
             # TODO: manage groups ? Maybe useless.
             group_ids = group_ids or []
             for gid in group_ids:
-                await self.kc.group_user_add(user['id'], gid)
+                await self.kc.group_user_add(user['id'], gid, user_info=user_info)
             await self.sync(user, data, user_info=user_info)
 
-        elif not user_info.is_admin:
-            raise UnauthorizedError(
-                f"user {data['username']} does not exists, "
-                "only administrators are allowed to create new ones."
-            )
-
-        elif not data.get('password', None):
-            raise DataError("Missing password in order to create User.")
-
         else:
-            data['id'] = await self.kc.create_user(data, groups)
+            cr_id = await self.kc.create_user(data, groups, user_info=user_info)
+            if cr_id:
+                data['id'] = cr_id
 
         # Important to remove password as it is not stored locally, SQLA would throw error.
         data.pop('password', None)
@@ -186,28 +205,29 @@ async def write(
         **kwargs
     ):
         """CREATE entities on Keycloak, before inserting in DB."""
-        for user in to_it(data):
-            # Groups first.
-            group_paths, group_ids = [], []
-            for group in user.get("groups", []):
-                await Group.svc.read_or_create(
-                    group,
+        if user_info and user_info.keycloak_admin:
+            for user in to_it(data):
+                # Groups first.
+                group_paths, group_ids = [], []
+                for group in user.get("groups", []):
+                    await Group.svc.read_or_create(
+                        group,
+                        user_info=user_info,
+                    )
+                    group_paths.append(group['path'])
+                    group_ids.append(group['id'])
+                # Then User.
+                await self.read_or_create(
+                    user,
                     user_info=user_info,
+                    groups=group_paths,
+                    group_ids=group_ids
                 )
-                group_paths.append(group['path'])
-                group_ids.append(group['id'])
-            # Then User.
-            await self.read_or_create(
-                user,
-                user_info=user_info,
-                groups=group_paths,
-                group_ids=group_ids
-            )
-
-        return await super().write(data, stmt_only=stmt_only, **kwargs)
+
+        return await super().write(data, stmt_only=stmt_only, user_info=user_info, **kwargs)
 
     async def delete(self, pk_val: List[Any], user_info: UserInfo | None = None, **_) -> None:
         """DELETE User from DB then from keycloak."""
         user_id = (await self.read(pk_val, fields=['id'])).id
         await super().delete(pk_val, user_info=user_info)
-        await self.kc.delete_user(user_id)
+        await self.kc.delete_user(user_id, user_info=user_info)

src/biodm/config.py

@@ -7,14 +7,11 @@
 except FileNotFoundError:
     config = Config()
 
-# TODO: [prio medium - before release]
-# Change credentials to Secret type
-# Avoids leaking them in stacktraces
-
 # Server.
 API_NAME        = config("API_NAME",        cast=str,  default="biodm_instance")
 API_VERSION     = config("API_VERSION",     cast=str,  default="0.1.0")
 API_DESCRIPTION = config("API_DESCRIPTION", cast=str,  default="")
+
 SERVER_SCHEME   = config("SERVER_SCHEME",   cast=str,  default="http://")
 SERVER_HOST     = config("SERVER_HOST",     cast=str,  default="0.0.0.0")
 SERVER_PORT     = config("SERVER_PORT",     cast=int,  default=8000)
@@ -44,12 +41,8 @@
 KC_HOST            = config("KC_HOST",            cast=str,    default=None)
 KC_REALM           = config("KC_REALM",           cast=str,    default=None)
 KC_PUBLIC_KEY      = config("KC_PUBLIC_KEY",      cast=str,    default=None)
-KC_ADMIN           = config("KC_ADMIN",           cast=str,    default=None)
-KC_ADMIN_PASSWORD  = config("KC_ADMIN_PASSWORD",  cast=Secret, default=None)
 KC_CLIENT_ID       = config("KC_CLIENT_ID",       cast=str,    default=None)
 KC_CLIENT_SECRET   = config("KC_CLIENT_SECRET",   cast=Secret, default=None)
-KC_JWT_OPTIONS     = config("KC_JWT_OPTIONS",     cast=dict,   default={'verify_exp': False,
-                                                                       'verify_aud': False})
 
 # Kubernetes.
 K8_IP         = config("K8_IP",         cast=str,     default=None)