diff --git a/backend/composer/admin.py b/backend/composer/admin.py index 017a5604..90eddaeb 100644 --- a/backend/composer/admin.py +++ b/backend/composer/admin.py @@ -6,7 +6,6 @@ from fsm_admin.mixins import FSMTransitionMixin from composer.models import ( - AnatomicalEntity, Phenotype, Sex, ConnectivityStatement, @@ -166,8 +165,8 @@ class ConnectivityStatementAdmin( "sentence__pmid", "sentence__pmcid", "knowledge_statement", - "origins__name", - "destinations__anatomical_entities__name", + "origins", + "destinations__anatomical_entities", ) fieldsets = () diff --git a/backend/composer/services/cs_ingestion/cs_ingestion_services.py b/backend/composer/services/cs_ingestion/cs_ingestion_services.py index 78dc0882..5d668bca 100644 --- a/backend/composer/services/cs_ingestion/cs_ingestion_services.py +++ b/backend/composer/services/cs_ingestion/cs_ingestion_services.py @@ -1,5 +1,4 @@ import logging -from datetime import datetime from django.db import transaction @@ -12,7 +11,6 @@ from .models import LoggableAnomaly, Severity from .neurondm_script import main as get_statements_from_neurondm - logger_service = LoggerService() @@ -27,11 +25,14 @@ def ingest_statements(update_upstream=False, update_anatomical_entities=False): for statement in statements: sentence, _ = get_or_create_sentence(statement) create_or_update_connectivity_statement(statement, sentence, update_anatomical_entities) + update_forward_connections(statements) except Exception as e: logger_service.add_anomaly( - LoggableAnomaly(statement_id=None, entity_id=None, message=str(e), severity=Severity.ERROR)) + LoggableAnomaly(statement_id=None, entity_id=None, message=str(e), + severity=Severity.ERROR) + ) successful_transaction = False logging.error(f"Ingestion aborted due to {e}") @@ -41,6 +42,3 @@ def ingest_statements(update_upstream=False, update_anatomical_entities=False): if update_upstream: update_upstream_statements() logger_service.write_ingested_statements_to_file(statements) - - - diff --git a/backend/composer/services/cs_ingestion/helpers/changes_detector.py b/backend/composer/services/cs_ingestion/helpers/changes_detector.py index 28659f3f..301d5eae 100644 --- a/backend/composer/services/cs_ingestion/helpers/changes_detector.py +++ b/backend/composer/services/cs_ingestion/helpers/changes_detector.py @@ -1,5 +1,9 @@ +import rdflib +from neurondm import orders + +from composer.models import AnatomicalEntity from composer.services.cs_ingestion.helpers.common_helpers import VALIDATION_ERRORS, SPECIES, PROVENANCE, ID, \ - FORWARD_CONNECTION + FORWARD_CONNECTION, ORIGINS, VIAS, DESTINATIONS from composer.services.cs_ingestion.models import ValidationErrors @@ -40,64 +44,84 @@ def has_changes(connectivity_statement, statement, defaults): if current_forward_connections != new_forward_connections: return True - # TODO: Update - # # Check for changes in origins - # current_origins = set(origin.ontology_uri for origin in connectivity_statement.origins.all()) - # new_origins = set(uri for uri in statement[ORIGINS].anatomical_entities if uri not in validation_errors.entities) - # if current_origins != new_origins: - # return True - # - # # Check for changes in vias - # current_vias = [ - # { - # 'anatomical_entities': set(via.anatomical_entities.all().values_list('ontology_uri', flat=True)), - # 'from_entities': set(via.from_entities.all().values_list('ontology_uri', flat=True)) - # } - # for via in connectivity_statement.via_set.order_by('order').all() - # ] - # new_vias = statement[VIAS] - # - # if len(current_vias) != len(new_vias): - # return True - # - # for current_via, new_via in zip(current_vias, new_vias): - # new_via_anatomical_entities = set( - # uri for uri in new_via.anatomical_entities if uri not in validation_errors.entities) - # - # new_via_from_entities = set(uri for uri in new_via.from_entities if uri not in validation_errors.entities) - # - # if (new_via_anatomical_entities != current_via['anatomical_entities'] or - # new_via_from_entities != current_via['from_entities']): - # return True - # - # # Check for changes in destinations - # current_destinations = connectivity_statement.destinations.all() - # new_destinations = statement[DESTINATIONS] - # - # if len(current_destinations) != len(new_destinations): - # return True - # - # # We may need to change this algorithm when multi-destination is supported by neurondm - # - # current_destinations_anatomical_entities = set( - # uri for destination in current_destinations - # for uri in destination.anatomical_entities.all().values_list('ontology_uri', flat=True) - # ) - # current_destinations_from_entities = set( - # uri for destination in current_destinations - # for uri in destination.from_entities.all().values_list('ontology_uri', flat=True) - # ) - # - # new_destinations_anatomical_entities = {uri for new_dest in statement[DESTINATIONS] for uri in - # new_dest.anatomical_entities if uri not in validation_errors.entities} - # - # new_destinations_from_entities = {uri for new_dest in statement[DESTINATIONS] for uri in new_dest.from_entities if - # uri not in validation_errors.entities} - # - # if (current_destinations_anatomical_entities != new_destinations_anatomical_entities or - # current_destinations_from_entities != new_destinations_from_entities): - # return True - - return True + # Check for changes in origins + current_origins = set( + get_anatomical_entity_identifier_composer(origin) for origin in connectivity_statement.origins.all()) + new_origins = set( + get_anatomical_entity_identifier_neurondm(uri) for uri in statement[ORIGINS].anatomical_entities if + uri not in validation_errors.entities) + if current_origins != new_origins: + return True + + # Check for changes in vias + current_vias = [ + { + 'anatomical_entities': set( + get_anatomical_entity_identifier_composer(ae) for ae in via.anatomical_entities.all()), + 'from_entities': set(get_anatomical_entity_identifier_composer(ae) for ae in via.from_entities.all()) + } + for via in connectivity_statement.via_set.order_by('order').all() + ] + new_vias = statement[VIAS] + + if len(current_vias) != len(new_vias): + return True + + for current_via, new_via in zip(current_vias, new_vias): + new_via_anatomical_entities = set( + get_anatomical_entity_identifier_neurondm(uri) for uri in new_via.anatomical_entities if + uri not in validation_errors.entities) + + new_via_from_entities = set(get_anatomical_entity_identifier_neurondm(uri) for uri in new_via.from_entities if + uri not in validation_errors.entities) + + if (new_via_anatomical_entities != current_via['anatomical_entities'] or + new_via_from_entities != current_via['from_entities']): + return True + + # Check for changes in destinations + current_destinations = connectivity_statement.destinations.all() + new_destinations = statement[DESTINATIONS] + + if len(current_destinations) != len(new_destinations): + return True + + # We may need to change this algorithm when multi-destination is supported by neurondm + + current_destinations_anatomical_entities = set( + get_anatomical_entity_identifier_composer(uri) for destination in current_destinations + for uri in destination.anatomical_entities.all() + ) + current_destinations_from_entities = set( + get_anatomical_entity_identifier_composer(uri) for destination in current_destinations + for uri in destination.from_entities.all() + ) + + new_destinations_anatomical_entities = {get_anatomical_entity_identifier_neurondm(uri) for new_dest in + statement[DESTINATIONS] for uri in + new_dest.anatomical_entities if uri not in validation_errors.entities} + + new_destinations_from_entities = {get_anatomical_entity_identifier_neurondm(uri) for new_dest in + statement[DESTINATIONS] for uri in new_dest.from_entities if + uri not in validation_errors.entities} + + if (current_destinations_anatomical_entities != new_destinations_anatomical_entities or + current_destinations_from_entities != new_destinations_from_entities): + return True + # Not checking the Notes because they are kept - return False \ No newline at end of file + return False + + +def get_anatomical_entity_identifier_composer(entity: AnatomicalEntity): + if entity.region_layer: + layer_uri = entity.region_layer.layer.ontology_uri + region_uri = entity.region_layer.region.ontology_uri + return f"{region_uri}:{layer_uri}" + return entity.simple_entity.ontology_uri + + +def get_anatomical_entity_identifier_neurondm(entity: rdflib.term): + if isinstance(entity, orders.rl): + return f"{str(entity.region)}:{str(entity.layer)}" + return str(entity) diff --git a/backend/composer/services/cs_ingestion/models.py b/backend/composer/services/cs_ingestion/models.py index e46ff10c..0fd1cc40 100644 --- a/backend/composer/services/cs_ingestion/models.py +++ b/backend/composer/services/cs_ingestion/models.py @@ -3,21 +3,21 @@ class NeuronDMOrigin: - def __init__(self, anatomical_entities_uri: Set): - self.anatomical_entities = anatomical_entities_uri + def __init__(self, anatomical_entities: Set): + self.anatomical_entities = anatomical_entities class NeuronDMVia: - def __init__(self, anatomical_entities_uri: Set, from_entities: Set, order: int, type: str): - self.anatomical_entities = anatomical_entities_uri + def __init__(self, anatomical_entities: Set, from_entities: Set, order: int, type: str): + self.anatomical_entities = anatomical_entities self.from_entities = from_entities self.order = order self.type = type class NeuronDMDestination: - def __init__(self, anatomical_entities_uri: Set, from_entities: Set, type: str): - self.anatomical_entities = anatomical_entities_uri + def __init__(self, anatomical_entities: Set, from_entities: Set, type: str): + self.anatomical_entities = anatomical_entities self.from_entities = from_entities self.type = type