Refactor, add & doc submitter_username feature, file size checking + …

…tests, support permissions schema discovery, flip FK naming convention

README.md

@@ -28,6 +28,11 @@ BioDM is a fast, modular, stateless and asynchronous REST API framework with the
   - Login and token retrieval system
   - OpenAPI schema generation through [apispec](https://github.com/marshmallow-code/apispec)
 
+
+It sits on the **F**indability and **A**ccessibility part of the **F.A.I.R** principles,
+while remaining flexible for the remainder to be implemented.
+
+
 ## Quickstart
 ### Install
 ```bash

docker/Dockerfile.biodm-test-api

@@ -20,7 +20,7 @@ COPY ./src/biodm /biodm/src/biodm
 COPY ./src/example /biodm/src/example
 
 # conditionally remove .env to replace it with environment variables in compose file.
-RUN if [[ -z "$KEEPENV" ]] ; then find /biodm/src/example -name '.env' | xargs rm -rf ; else : ; fi
+RUN if [ -z "$KEEPENV" ] ; then find /biodm/src/example -name '.env' | xargs rm -rf ; else : ; fi
 
 RUN pip3 install .
 

docs/developer_manual/advanced_use.rst

@@ -317,7 +317,7 @@ A lot of that code has to do with retrieving async SQLAlchemy objects attributes
             key = await self.gen_key(file, session=session)
             parts.append(
                 UploadPart(
-                    id_upload=file.upload.id,
+                    upload_id=file.upload.id,
                     form=str(
                         self.s3.create_presigned_post(
                             object_name=key,

docs/developer_manual/index.rst

@@ -22,7 +22,7 @@ metadata and relationships, to setup standard RESTful endpoints.
 
 Furthermore, it is providing a structure and toolkit in order to manage common Data Management problems
 such as: s3 protocol remote file storage, group based permissions access (on both resources and
-endpoints), resource versioning (coming up), cluster jobs and so on.
+endpoints), resource versioning, cluster jobs and so on.
 
 Moreover, the modular and flexible architecture allows you to easily extend base features for
 instance specific use cases.

docs/developer_manual/table_schema.rst

@@ -30,6 +30,27 @@ In particular, if a relationship is one-armed (pointing in one direction only),
 be possible to create a nested resource in the other direction.
 
 
+Special columns
+~~~~~~~~~~~~~~~
+
+Some special column will yield built-in behavior.
+
+
+**Tracking resource submitter: submitter_username**
+
+
+Setting up the following `foreign key`, in a table will automatically populate the field
+with requesting user's username creating the resource.
+
+
+.. code:: python
+
+    class MyTable(Base):
+        id:          Mapped[int]   = mapped_column(Integer(), primary_key=True)
+        ...
+        submitter_username:  Mapped[str] = mapped_column(ForeignKey("USER.username"), nullable=False)
+
+
 Schemas
 -------
 

docs/user_manual.rst

@@ -350,8 +350,8 @@ otherwise.
         "perm_files": {
             "write": {
                 "groups": [
-                    {"name": "genomics_team"},
-                    {"name": "IT_team"},
+                    {"path": "genomics_team"},
+                    {"path": "IT_team"},
                     {"..."}
                 ]
             },

src/biodm/api.py

@@ -8,7 +8,7 @@
 from typing import Callable, List, Optional, Dict, Any, Type
 
 from apispec import APISpec
-from apispec.ext.marshmallow import MarshmallowPlugin
+# from apispec.ext.marshmallow import MarshmallowPlugin
 from starlette_apispec import APISpecSchemaGenerator
 from starlette.applications import Starlette
 from starlette.middleware.base import BaseHTTPMiddleware
@@ -21,6 +21,7 @@
 
 from biodm import Scope, config
 from biodm.basics import CORE_CONTROLLERS, K8sController
+from biodm.components.controllers.resourcecontroller import ResourceController
 from biodm.components.k8smanifest import K8sManifest
 from biodm.managers import DatabaseManager, KeycloakManager, S3Manager, K8sManager
 from biodm.components.controllers import Controller
@@ -29,6 +30,7 @@
 from biodm.exceptions import RequestError
 from biodm.utils.security import AuthenticationMiddleware, PermissionLookupTables
 from biodm.utils.utils import to_it
+from biodm.utils.apispec import BDMarshmallowPlugin
 from biodm.tables import History, ListGroup, Upload, UploadPart
 from biodm import __version__ as CORE_VERSION
 
@@ -64,7 +66,7 @@ async def dispatch(self, request: Request, call_next: Callable) -> Any:
         endpoint = str(request.url).rsplit(self.server_host, maxsplit=1)[-1]
         body = await request.body()
         entry = {
-            'username_user': user_id,
+            'user_username': user_id,
             'endpoint': endpoint,
             'method': request.method,
             'content': str(body) if body else ""
@@ -150,7 +152,7 @@ def __init__(
                 title=config.API_NAME,
                 version=config.API_VERSION,
                 openapi_version="3.0.0",
-                plugins=[MarshmallowPlugin()],
+                plugins=[BDMarshmallowPlugin()],
                 info={"description": "", "backend": "biodm", "backend_version": CORE_VERSION},
                 security=[{'Authorization': []}] # Same name as security_scheme arg below.
             )
@@ -191,7 +193,7 @@ def __init__(
         # self.add_exception_handler(DatabaseError, on_error)
 
     @property
-    def server_endpoint(self) -> str:
+    def server_endpoint(cls) -> str:
         """Server address, useful to compute callbacks."""
         return f"{config.SERVER_SCHEME}{config.SERVER_HOST}:{config.SERVER_PORT}/"
 

src/biodm/basics/k8scontroller.py

@@ -8,9 +8,7 @@
 
 from biodm.components.controllers import Controller, HttpMethod# ResourceController
 from biodm.exceptions import ManifestError
-from biodm.tables import K8sInstance
 from biodm.components import K8sManifest
-from biodm.schemas import K8sinstanceSchema
 from biodm.utils.utils import json_response
 
 

src/biodm/components/controllers/resourcecontroller.py

@@ -1,7 +1,8 @@
 """Controller class for Tables acting as a Resource."""
 
 from __future__ import annotations
-from functools import partial, wraps
+from functools import partial
+from inspect import getmembers, ismethod
 from types import MethodType
 from typing import TYPE_CHECKING, Callable, List, Set, Any, Dict, Type
 
@@ -32,6 +33,7 @@
 )
 from biodm.utils.security import UserInfo
 from biodm.utils.utils import json_response
+from biodm.utils.apispec import register_runtime_schema, process_apispec_docstrings
 from biodm.components import Base
 from .controller import HttpMethod, EntityController
 
@@ -91,7 +93,10 @@ def __init__(
 
         self.pk = set(self.table.pk)
         self.svc: UnaryEntityService = self._infer_svc()(app=self.app, table=self.table)
-        self.__class__.schema = (schema if schema else self._infer_schema())(unknown=RAISE)
+        # Inst schema, and set custom registry for apispec.
+        schema_cls = schema if schema else self._infer_schema()
+        self.__class__.schema = schema_cls(unknown=RAISE)
+        register_runtime_schema(schema_cls, self.__class__.schema)
         self._infuse_schema_in_apispec_docstrings()
 
     @staticmethod
@@ -110,112 +115,22 @@ async def mirror(self, *args, **kwargs):
 
     def _infuse_schema_in_apispec_docstrings(self):
         """Substitute endpoint documentation template bits with adapted ones for this resource.
-
-        Essentially handling APIspec/Marshmallow/OpenAPISchema support for abstract endpoints.
-
-        Current patterns for abstract documentation:
-        - Marshmallow Schema |
-            schema: Schema -> schema: self.Schema.__class__.__name__
-        - key Attributes |
-            - in: path
-              name: id
-            ->
-            List of table primary keys, with their description from marshmallow schema if any.
-        - field conditions |
-            - in: query
-              name: field_conditions
-            ->
-            List of available fields to set conditions on.
+        Handling APIspec/Marshmallow/OpenAPISchema support for abstract endpoints.
         """
-        def process_apispec_docstrings(self, abs_doc):
-            # Use intance schema.
-            abs_doc = abs_doc.replace(
-                'schema: Schema', f"schema: {self.schema.__class__.__name__}"
+        for method, fct in getmembers(
+            self, predicate=lambda x: ( # Use typing anotations to identify endpoints.
+                ismethod(x) and hasattr(x, '__annotations__') and
+                x.__annotations__.get('request', '') == 'Request' and
+                x.__annotations__.get('return', '') == 'Response'
+            )
+        ):
+            # Replace with processed docstrings.
+            setattr(self, method, MethodType(
+                    ResourceController.replace_method_docstrings(
+                        method, process_apispec_docstrings(self, fct.__doc__ or "")
+                    ), self
+                )
             )
-
-            # Template replacement #1: path key.
-            path_key = []
-            for key in self.pk:
-                attr = []
-                attr.append("- in: path")
-                attr.append(f"name: {key}")
-                field = self.schema.declared_fields[key]
-                desc  = field.metadata.get("description", None)
-                attr.append("description: " + (desc or f"{self.resource} {key}"))
-                path_key.append(attr)
-
-            # Template replacement #2: field conditions.
-            field_conditions = []
-            for col in self.table.__table__.columns:
-                condition = []
-                condition.append("- in: query")
-                condition.append(f"name: {col.name}")
-                if col.type.python_type == str:
-                    condition.append(
-                        "description: text - key=val | key=pattern "
-                        "where pattern may contain '*' for wildcards"
-                    )
-                elif col.type.python_type in (int, float):
-                    condition.append(
-                        "description: numeric - key=val | key=val1,val2.. | key.op(val) "
-                        "for op in (le|lt|ge|gt)"
-                    )
-                else:
-                    condition.append(f"description: {self.resource} {col.name}")
-                field_conditions.append(condition)
-
-            # Split.
-            doc = abs_doc.split('---')
-            if len(doc) > 1:
-                sphinxdoc, apispec = doc
-                apispec = apispec.split('\n')
-                flattened = []
-                # Search and replace templates.
-                for i in range(len(apispec)):
-                    if '- in: path' in apispec[i-1] and 'name: id' in apispec[i]:
-                        # Work out same indentation level in order not to break the yaml.
-                        indent = len(apispec[i-1].split('- in: path')[0])
-                        for path_attribute in path_key:
-                            path_attribute[0] = " " * indent + path_attribute[0]
-                            path_attribute[1] = " " * (indent+2) + path_attribute[1]
-                            path_attribute[2] = " " * (indent+2) + path_attribute[2]
-                            flattened.extend(path_attribute)
-                        break
-                if flattened:
-                    apispec = apispec[:i-1] + flattened + apispec[i+1:]
-
-                flattened = []
-                for i in range(len(apispec)):
-                    if '- in: query' in apispec[i-1] and 'name: fields_conditions' in apispec[i]:
-                        indent = len(apispec[i-1].split('- in: query')[0])
-                        flattened = []
-                        for condition in field_conditions:
-                            condition[0] = " " * indent + condition[0]
-                            condition[1] = " " * (indent+2) + condition[1]
-                            condition[2] = " " * (indent+2) + condition[2]
-                            flattened.extend(condition)
-                        break
-                if flattened:
-                    apispec = apispec[:i-1] + flattened + apispec[i+1:]
-                # Join.
-                abs_doc = sphinxdoc + "\n---\n" + "\n".join(apispec)
-            return abs_doc
-
-        for method in dir(self):
-            if not method.startswith('_'):
-                fct = getattr(self, method, {})
-                if hasattr(fct, '__annotations__'):
-                    if ( # Use typing anotations to identify endpoints.
-                            fct.__annotations__.get('request', '') == 'Request' and
-                            fct.__annotations__.get('return', '') == 'Response'
-                        ):
-                            # Replace with processed docstrings.
-                            setattr(self, method, MethodType(
-                                    ResourceController.replace_method_docstrings(
-                                        method, process_apispec_docstrings(self, fct.__doc__ or "")
-                                    ), self
-                                )
-                            )
 
     def _infer_resource_name(self) -> str:
         """Infer entity name from controller name."""
@@ -440,7 +355,7 @@ async def read(self, request: Request) -> Response:
                 e.g. /datasets/1_1?name,description,contact,files
           - in: path
             name: attribute
-            description: Optional, nested collection name.
+            description: nested collection name
         responses:
             200:
                 description: Found matching item
@@ -526,21 +441,16 @@ async def update(self, request: Request) -> Response:
         # Plug in pk into the dict.
         validated_data.update(dict(zip(self.pk, pk_val))) # type: ignore [assignment]
 
-        try:
-            return json_response(
-                data=await self.svc.write(
-                    data=validated_data,
-                    stmt_only=False,
-                    user_info=request.state.user_info,
-                    serializer=partial(self.serialize, many=isinstance(validated_data, list)),
-                ),
-                status_code=201,
-            )
-        except IntegrityError as ie:
-            if 'UNIQUE' in ie.args[0] and 'version' in ie.args[0]: # Versioned case.
-                raise UpdateVersionedError(
-                    "Attempt at updating versioned resources via POST detected"
-                )
+        return json_response(
+            data=await self.svc.write(
+                data=validated_data,
+                stmt_only=False,
+                user_info=request.state.user_info,
+                serializer=partial(self.serialize, many=isinstance(validated_data, list)),
+            ),
+            status_code=201,
+        )
+
 
     async def delete(self, request: Request) -> Response:
         """Delete resource.

src/biodm/components/controllers/s3controller.py

@@ -21,6 +21,7 @@
 class S3Controller(ResourceController):
     """Controller for entities involving file management leveraging an S3Service."""
     svc: S3Service
+
     def __init__(
         self,
         app,