diff --git a/applications/visualizer/api/openapi.json b/applications/visualizer/api/openapi.json index 4e682cf9..15f3638d 100644 --- a/applications/visualizer/api/openapi.json +++ b/applications/visualizer/api/openapi.json @@ -143,11 +143,92 @@ ] } }, + "/api/cells/search": { + "get": { + "operationId": "search_cells", + "summary": "Search Cells", + "parameters": [ + { + "in": "query", + "name": "name", + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Name" + }, + "required": false + }, + { + "in": "query", + "name": "dataset_ids", + "schema": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Dataset Ids" + }, + "required": false + } + ], + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "items": { + "$ref": "#/components/schemas/Neuron" + }, + "title": "Response", + "type": "array" + } + } + } + } + }, + "tags": [ + "neurons" + ] + } + }, "/api/cells": { "get": { "operationId": "get_all_cells", "summary": "Get All Cells", "parameters": [ + { + "in": "query", + "name": "dataset_ids", + "schema": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Dataset Ids" + }, + "required": false + }, { "in": "query", "name": "page", @@ -400,6 +481,13 @@ "title": "Name", "type": "string" }, + "datasetIds": { + "items": { + "type": "string" + }, + "title": "Datasetids", + "type": "array" + }, "nclass": { "maxLength": 30, "title": "Nclass", @@ -433,6 +521,7 @@ }, "required": [ "name", + "datasetIds", "nclass", "neurotransmitter", "type" diff --git a/applications/visualizer/api/openapi.yaml b/applications/visualizer/api/openapi.yaml index 9f583052..a5bc1bc3 100644 --- a/applications/visualizer/api/openapi.yaml +++ b/applications/visualizer/api/openapi.yaml @@ -93,6 +93,11 @@ components: type: object Neuron: properties: + datasetIds: + items: + type: string + title: Datasetids + type: array embryonic: default: false title: Embryonic @@ -122,6 +127,7 @@ components: type: string required: - name + - datasetIds - nclass - neurotransmitter - type @@ -153,6 +159,16 @@ paths: description: Returns all the cells (neurons) from the DB operationId: get_all_cells parameters: + - in: query + name: dataset_ids + required: false + schema: + anyOf: + - items: + type: string + type: array + - type: 'null' + title: Dataset Ids - in: query name: page required: false @@ -171,6 +187,41 @@ paths: summary: Get All Cells tags: - neurons + /api/cells/search: + get: + operationId: search_cells + parameters: + - in: query + name: name + required: false + schema: + anyOf: + - type: string + - type: 'null' + title: Name + - in: query + name: dataset_ids + required: false + schema: + anyOf: + - items: + type: string + type: array + - type: 'null' + title: Dataset Ids + responses: + '200': + content: + application/json: + schema: + items: + $ref: '#/components/schemas/Neuron' + title: Response + type: array + description: OK + summary: Search Cells + tags: + - neurons /api/connections: get: description: Gets the connections of a dedicated Dataset diff --git a/applications/visualizer/backend/api/api.py b/applications/visualizer/backend/api/api.py index 2c87538a..5015339e 100644 --- a/applications/visualizer/backend/api/api.py +++ b/applications/visualizer/backend/api/api.py @@ -1,10 +1,13 @@ +from collections import defaultdict from typing import Optional from django.http import HttpResponse from ninja import NinjaAPI, Router, Schema, Query from ninja.pagination import paginate, PageNumberPagination from django.shortcuts import aget_object_or_404 -from django.db.models import Q +from django.db.models import Q, F, Value, CharField, Func, OuterRef +from django.db.models.manager import BaseManager +from django.db.models.functions import Coalesce, Concat from .schemas import Dataset, Neuron, Connection from .models import ( @@ -82,34 +85,94 @@ async def get_dataset(request, dataset: str): return await aget_object_or_404(DatasetModel, id=dataset) +def annotate_neurons_w_dataset_ids(neurons: BaseManager[NeuronModel]) -> None: + """Queries the datasets ids for each neuron.""" + neuron_names = neurons.values_list("name", flat=True).distinct() + pre = ( + ConnectionModel.objects.filter(pre__in=neuron_names) + .values_list("pre", "dataset") + .distinct() + ) + post = ( + ConnectionModel.objects.filter(post__in=neuron_names) + .values_list("post", "dataset") + .distinct() + ) + + # Filter out repeated dataset ids + neurons_dataset_ids = defaultdict(set) + for neuron, dataset in pre.union(post): + neurons_dataset_ids[neuron].add(dataset) + + for neuron in neurons: + neuron.dataset_ids = neurons_dataset_ids[neuron.name] # type: ignore + + +def neurons_from_datasets( + neurons: BaseManager[NeuronModel], dataset_ids: list[str] +) -> BaseManager[NeuronModel]: + """Filters neurons belonging to specific datasets.""" + return neurons.filter( + Q( + name__in=ConnectionModel.objects.filter( + dataset__id__in=dataset_ids + ).values_list("pre", flat=True) + ) + | Q( + name__in=ConnectionModel.objects.filter( + dataset__id__in=dataset_ids + ).values_list("post", flat=True) + ) + ) + + @api.get( "/datasets/{dataset}/neurons", response={200: list[Neuron], 404: ErrorMessage}, tags=["datasets"], ) -async def get_dataset_neurons(request, dataset: str): +def get_dataset_neurons(request, dataset: str): """Returns all the neurons of a dedicated dataset""" - return await to_list( - NeuronModel.objects.filter( - Q( - name__in=ConnectionModel.objects.filter( - dataset__id=dataset - ).values_list("pre", flat=True) - ) - | Q( - name__in=ConnectionModel.objects.filter( - dataset__id=dataset - ).values_list("post", flat=True) - ) - ) - ) + neurons = neurons_from_datasets(NeuronModel.objects, [dataset]) + annotate_neurons_w_dataset_ids(neurons) + return neurons + + +@api.get("/cells/search", response=list[Neuron], tags=["neurons"]) +def search_cells( + request, + name: Optional[str] = Query(None), + dataset_ids: Optional[list[str]] = Query(None), +): + neurons = NeuronModel.objects + + if name: + neurons = neurons.filter(name__istartswith=name) + + if dataset_ids: + neurons = neurons_from_datasets(neurons, dataset_ids) + else: + neurons = neurons.all() + + annotate_neurons_w_dataset_ids(neurons) + + return neurons @api.get("/cells", response=list[Neuron], tags=["neurons"]) -@paginate(PageNumberPagination, page_size=50) -def get_all_cells(request): +@paginate(PageNumberPagination, page_size=50) # BUG: this is not being applied +def get_all_cells(request, dataset_ids: Optional[list[str]] = Query(None)): """Returns all the cells (neurons) from the DB""" - return NeuronModel.objects.all() + neurons = NeuronModel.objects + + if dataset_ids: + neurons = neurons_from_datasets(neurons, dataset_ids) + else: + neurons = neurons.all() + + annotate_neurons_w_dataset_ids(neurons) + + return neurons # # @api.post("/connections", response=list[Connection], tags=["connectivity"]) diff --git a/applications/visualizer/backend/api/schemas.py b/applications/visualizer/backend/api/schemas.py index 9aac1a27..8124c08c 100644 --- a/applications/visualizer/backend/api/schemas.py +++ b/applications/visualizer/backend/api/schemas.py @@ -37,6 +37,7 @@ class Meta: class Neuron(ModelSchema, BilingualSchema): name: str + dataset_ids: list[str] class Meta: model = NeuronModel diff --git a/applications/visualizer/backend/api/tests.py b/applications/visualizer/backend/api/tests.py deleted file mode 100644 index 7ce503c2..00000000 --- a/applications/visualizer/backend/api/tests.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.test import TestCase - -# Create your tests here. diff --git a/applications/visualizer/backend/openapi/openapi.json b/applications/visualizer/backend/openapi/openapi.json index 4e682cf9..15f3638d 100644 --- a/applications/visualizer/backend/openapi/openapi.json +++ b/applications/visualizer/backend/openapi/openapi.json @@ -143,11 +143,92 @@ ] } }, + "/api/cells/search": { + "get": { + "operationId": "search_cells", + "summary": "Search Cells", + "parameters": [ + { + "in": "query", + "name": "name", + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Name" + }, + "required": false + }, + { + "in": "query", + "name": "dataset_ids", + "schema": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Dataset Ids" + }, + "required": false + } + ], + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "items": { + "$ref": "#/components/schemas/Neuron" + }, + "title": "Response", + "type": "array" + } + } + } + } + }, + "tags": [ + "neurons" + ] + } + }, "/api/cells": { "get": { "operationId": "get_all_cells", "summary": "Get All Cells", "parameters": [ + { + "in": "query", + "name": "dataset_ids", + "schema": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Dataset Ids" + }, + "required": false + }, { "in": "query", "name": "page", @@ -400,6 +481,13 @@ "title": "Name", "type": "string" }, + "datasetIds": { + "items": { + "type": "string" + }, + "title": "Datasetids", + "type": "array" + }, "nclass": { "maxLength": 30, "title": "Nclass", @@ -433,6 +521,7 @@ }, "required": [ "name", + "datasetIds", "nclass", "neurotransmitter", "type" diff --git a/applications/visualizer/backend/requirements-dev.txt b/applications/visualizer/backend/requirements-dev.txt index 5679bf19..364e9bf9 100644 --- a/applications/visualizer/backend/requirements-dev.txt +++ b/applications/visualizer/backend/requirements-dev.txt @@ -4,5 +4,6 @@ ipdb pytest pytest-django pytest-asyncio +pytest-unordered model-bakery black \ No newline at end of file diff --git a/applications/visualizer/backend/tests/test_neurons.py b/applications/visualizer/backend/tests/test_neurons.py new file mode 100644 index 00000000..860ff934 --- /dev/null +++ b/applications/visualizer/backend/tests/test_neurons.py @@ -0,0 +1,214 @@ +from ninja.testing import TestClient +import pytest +import warnings +from pytest_unordered import unordered + +from api.models import ( + Dataset as DatasetModel, + Neuron as NeuronModel, + Connection as ConnectionModel, +) +from api.api import api as celegans_api +from .utils import generate_instance + + +# Some test data +datasets = [ + { + "id": "ds1", + "name": "Dataset 1", + }, + { + "id": "ds2", + "name": "Gamma Goblin", + }, + { + "id": "ds3", + "name": "Dr. Seuss", + }, +] + +neurons = [ + { + "name": "ADAL", + "nclass": "ADA", + "neurotransmitter": "l", + "type": "i", + }, + { + "name": "ADAR", + "nclass": "ADA", + "neurotransmitter": "l", + "type": "i", + }, + { + "name": "ADEL", + "nclass": "ADA", + "neurotransmitter": "l", + "type": "i", + }, + { + "name": "ADER", + "nclass": "ADR", + "neurotransmitter": "d", + "type": "sn", + }, + { + "name": "ADFR", + "nclass": "ADF", + "neurotransmitter": "as", + "type": "sn", + }, + { + "name": "AFDL", + "nclass": "AFD", + "neurotransmitter": "l", + "type": "s", + }, +] + +connections = lambda: [ + { + "dataset": DatasetModel.objects.get(id="ds1"), + "pre": "ADAL", + "post": "ADAR", + }, + { + "dataset": DatasetModel.objects.get(id="ds2"), + "pre": "ADEL", + "post": "ADER", + }, + { + "dataset": DatasetModel.objects.get(id="ds2"), + "pre": "ADAR", + "post": "ADEL", + }, + { + "dataset": DatasetModel.objects.get(id="ds3"), + "pre": "ADFR", + "post": "ADAR", + }, +] + + +# Setup the db for this module with some data +# Data are baked with "baker", it allows to create dummy values automatically +# and also to specify some fields. It is used here to "fill" the fields which are +# marked as "non-null" in the model which we don't want to manually fill. +@pytest.fixture(scope="module") +def django_db_setup(django_db_setup, django_db_blocker): + with django_db_blocker.unblock(): + generate_instance(DatasetModel, datasets) + generate_instance(NeuronModel, neurons) + generate_instance(ConnectionModel, connections()) + + +# Fixture to access the test client in all test functions +@pytest.fixture +def api_client(): + client = TestClient(celegans_api.default_router) + return client + + +@pytest.mark.django_db # required to access the DB +def test__get_all_cells(api_client): + expected_dataset_ids = { + "ADAL": ["ds1"], + "ADAR": ["ds1", "ds2", "ds3"], + "ADEL": ["ds2"], + "ADER": ["ds2"], + "ADFR": ["ds3"], + "AFDL": [], + } + + response = api_client.get("/cells") + assert response.status_code == 200 + + neurons = response.json()["items"] + for neuron in neurons: + name = neuron["name"] + + if name not in expected_dataset_ids: + warnings.warn( + f"please, update test: neuron '{name}' not found in expected dataset ids" + ) + continue + + assert expected_dataset_ids[name] == unordered(neuron["datasetIds"]) + + +@pytest.mark.django_db # required to access the DB +def test__get_all_cells_from_specific_datasets(api_client): + dataset_ids = ["ds1", "ds2"] + expected_dataset_ids = { + "ADAL": ["ds1"], + "ADAR": ["ds1", "ds2", "ds3"], # ds3 should be present! + "ADEL": ["ds2"], + "ADER": ["ds2"], + # "ADFR": ["ds3"], not part of ds1 or ds2 + # "AFDL": [], not part of ds1 or ds2 + } + + query_params = "?" + "&".join([f"dataset_ids={ds}" for ds in dataset_ids]) + response = api_client.get("/cells" + query_params) + assert response.status_code == 200 + + neurons = response.json()["items"] + for neuron in neurons: + name = neuron["name"] + + assert name in expected_dataset_ids, f"unexpected neuron result: {neuron}" + assert expected_dataset_ids[name] == unordered(neuron["datasetIds"]) + + +@pytest.mark.django_db # required to access the DB +def test__search_cells(api_client): + search_query = "ada" + expected_neurons_names = ["ADAL", "ADAR"] + + response = api_client.get(f"/cells/search?name={search_query}") + assert response.status_code == 200 + + neurons = response.json() + assert len(neurons) == len( + expected_neurons_names + ), f"expected to query {len(expected_neurons_names)} and got {len(neurons)}" + + for neuron in neurons: + assert neuron["name"] in expected_neurons_names + + +@pytest.mark.django_db # required to access the DB +def test__search_cells_in_datasets(api_client): + search_query = "ade" + dataset_ids = ["ds1", "ds3"] + + # Search dataset that do not contain a match + query_params = f"?name={search_query}&" + "&".join( + [f"dataset_ids={ds}" for ds in dataset_ids] + ) + response = api_client.get(f"/cells/search" + query_params) + assert response.status_code == 200 + + neurons = response.json() + assert len(neurons) == 0, f"expected datasets to not contain search matches" + + # Search dataset with matching neurons + dataset_ids.append("ds2") + expected_neurons_names = ["ADEL", "ADER"] + + query_params = f"?name={search_query}&" + "&".join( + [f"dataset_ids={ds}" for ds in dataset_ids] + ) + print(query_params) + + response = api_client.get(f"/cells/search" + query_params) + assert response.status_code == 200 + + neurons = response.json() + assert len(neurons) == len( + expected_neurons_names + ), f"expected to query {len(expected_neurons_names)} and got {len(neurons)}" + + for neuron in neurons: + assert neuron["name"] in expected_neurons_names diff --git a/applications/visualizer/frontend/src/rest/core/OpenAPI.ts b/applications/visualizer/frontend/src/rest/core/OpenAPI.ts index 11de5934..a0a9ed4a 100644 --- a/applications/visualizer/frontend/src/rest/core/OpenAPI.ts +++ b/applications/visualizer/frontend/src/rest/core/OpenAPI.ts @@ -1,7 +1,7 @@ /* generated using openapi-typescript-codegen -- do not edit */ /* istanbul ignore file */ /* tslint:disable */ - +/* eslint-disable */ import type { ApiRequestOptions } from './ApiRequestOptions'; type Resolver = (options: ApiRequestOptions) => Promise; diff --git a/applications/visualizer/frontend/src/rest/index.ts b/applications/visualizer/frontend/src/rest/index.ts index f59e9102..6c3c26cd 100644 --- a/applications/visualizer/frontend/src/rest/index.ts +++ b/applications/visualizer/frontend/src/rest/index.ts @@ -1,7 +1,7 @@ /* generated using openapi-typescript-codegen -- do not edit */ /* istanbul ignore file */ /* tslint:disable */ - +/* eslint-disable */ export { ApiError } from './core/ApiError'; export { CancelablePromise, CancelError } from './core/CancelablePromise'; export { OpenAPI } from './core/OpenAPI'; diff --git a/applications/visualizer/frontend/src/rest/models/Connection.ts b/applications/visualizer/frontend/src/rest/models/Connection.ts index 48804b70..7149fdd6 100644 --- a/applications/visualizer/frontend/src/rest/models/Connection.ts +++ b/applications/visualizer/frontend/src/rest/models/Connection.ts @@ -1,7 +1,7 @@ /* generated using openapi-typescript-codegen -- do not edit */ /* istanbul ignore file */ /* tslint:disable */ - +/* eslint-disable */ export type Connection = { annotations?: Array; synapses?: Record; diff --git a/applications/visualizer/frontend/src/rest/models/ErrorMessage.ts b/applications/visualizer/frontend/src/rest/models/ErrorMessage.ts index 4ed9ab1e..0da43d21 100644 --- a/applications/visualizer/frontend/src/rest/models/ErrorMessage.ts +++ b/applications/visualizer/frontend/src/rest/models/ErrorMessage.ts @@ -1,7 +1,7 @@ /* generated using openapi-typescript-codegen -- do not edit */ /* istanbul ignore file */ /* tslint:disable */ - +/* eslint-disable */ export type ErrorMessage = { detail: string; }; diff --git a/applications/visualizer/frontend/src/rest/models/Input.ts b/applications/visualizer/frontend/src/rest/models/Input.ts index 03462318..36c98ebe 100644 --- a/applications/visualizer/frontend/src/rest/models/Input.ts +++ b/applications/visualizer/frontend/src/rest/models/Input.ts @@ -1,7 +1,7 @@ /* generated using openapi-typescript-codegen -- do not edit */ /* istanbul ignore file */ /* tslint:disable */ - +/* eslint-disable */ export type Input = { page?: number; }; diff --git a/applications/visualizer/frontend/src/rest/models/Neuron.ts b/applications/visualizer/frontend/src/rest/models/Neuron.ts index bb30a110..976f30fe 100644 --- a/applications/visualizer/frontend/src/rest/models/Neuron.ts +++ b/applications/visualizer/frontend/src/rest/models/Neuron.ts @@ -1,9 +1,10 @@ /* generated using openapi-typescript-codegen -- do not edit */ /* istanbul ignore file */ /* tslint:disable */ - +/* eslint-disable */ export type Neuron = { name: string; + datasetIds: Array; nclass: string; neurotransmitter: string; type: string; diff --git a/applications/visualizer/frontend/src/rest/models/PagedNeuron.ts b/applications/visualizer/frontend/src/rest/models/PagedNeuron.ts index 7ebc8f2a..dcc73430 100644 --- a/applications/visualizer/frontend/src/rest/models/PagedNeuron.ts +++ b/applications/visualizer/frontend/src/rest/models/PagedNeuron.ts @@ -1,7 +1,7 @@ /* generated using openapi-typescript-codegen -- do not edit */ /* istanbul ignore file */ /* tslint:disable */ - +/* eslint-disable */ import type { Neuron } from './Neuron'; export type PagedNeuron = { items: Array; diff --git a/applications/visualizer/frontend/src/rest/services/ConnectivityService.ts b/applications/visualizer/frontend/src/rest/services/ConnectivityService.ts index d9f05e78..c2ad0e9a 100644 --- a/applications/visualizer/frontend/src/rest/services/ConnectivityService.ts +++ b/applications/visualizer/frontend/src/rest/services/ConnectivityService.ts @@ -1,7 +1,7 @@ /* generated using openapi-typescript-codegen -- do not edit */ /* istanbul ignore file */ /* tslint:disable */ - +/* eslint-disable */ import type { Connection } from '../models/Connection'; import type { CancelablePromise } from '../core/CancelablePromise'; import { OpenAPI } from '../core/OpenAPI'; diff --git a/applications/visualizer/frontend/src/rest/services/DatasetsService.ts b/applications/visualizer/frontend/src/rest/services/DatasetsService.ts index 6ba01883..c0e3851b 100644 --- a/applications/visualizer/frontend/src/rest/services/DatasetsService.ts +++ b/applications/visualizer/frontend/src/rest/services/DatasetsService.ts @@ -1,7 +1,7 @@ /* generated using openapi-typescript-codegen -- do not edit */ /* istanbul ignore file */ /* tslint:disable */ - +/* eslint-disable */ import type { Dataset } from '../models/Dataset'; import type { Neuron } from '../models/Neuron'; import type { CancelablePromise } from '../core/CancelablePromise'; diff --git a/applications/visualizer/frontend/src/rest/services/NeuronsService.ts b/applications/visualizer/frontend/src/rest/services/NeuronsService.ts index fc3f710a..9fe03ce2 100644 --- a/applications/visualizer/frontend/src/rest/services/NeuronsService.ts +++ b/applications/visualizer/frontend/src/rest/services/NeuronsService.ts @@ -1,12 +1,34 @@ /* generated using openapi-typescript-codegen -- do not edit */ /* istanbul ignore file */ /* tslint:disable */ - +/* eslint-disable */ +import type { Neuron } from '../models/Neuron'; import type { PagedNeuron } from '../models/PagedNeuron'; import type { CancelablePromise } from '../core/CancelablePromise'; import { OpenAPI } from '../core/OpenAPI'; import { request as __request } from '../core/request'; export class NeuronsService { + /** + * Search Cells + * @returns Neuron OK + * @throws ApiError + */ + public static searchCells({ + name, + datasetIds, + }: { + name?: (string | null), + datasetIds?: (Array | null), + }): CancelablePromise> { + return __request(OpenAPI, { + method: 'GET', + url: '/api/cells/search', + query: { + 'name': name, + 'dataset_ids': datasetIds, + }, + }); + } /** * Get All Cells * Returns all the cells (neurons) from the DB @@ -14,14 +36,17 @@ export class NeuronsService { * @throws ApiError */ public static getAllCells({ + datasetIds, page = 1, }: { + datasetIds?: (Array | null), page?: number, }): CancelablePromise { return __request(OpenAPI, { method: 'GET', url: '/api/cells', query: { + 'dataset_ids': datasetIds, 'page': page, }, });