SCKAN-275 feat: Update changes detector

backend/composer/admin.py

@@ -6,7 +6,6 @@
 from fsm_admin.mixins import FSMTransitionMixin
 
 from composer.models import (
-    AnatomicalEntity,
     Phenotype,
     Sex,
     ConnectivityStatement,
@@ -166,8 +165,8 @@ class ConnectivityStatementAdmin(
         "sentence__pmid",
         "sentence__pmcid",
         "knowledge_statement",
-        "origins__name",
-        "destinations__anatomical_entities__name",
+        "origins",
+        "destinations__anatomical_entities",
     )
 
     fieldsets = ()

backend/composer/services/cs_ingestion/cs_ingestion_services.py

@@ -1,5 +1,4 @@
 import logging
-from datetime import datetime
 
 from django.db import transaction
 
@@ -12,7 +11,6 @@
 from .models import LoggableAnomaly, Severity
 from .neurondm_script import main as get_statements_from_neurondm
 
-
 logger_service = LoggerService()
 
 
@@ -27,11 +25,14 @@ def ingest_statements(update_upstream=False, update_anatomical_entities=False):
             for statement in statements:
                 sentence, _ = get_or_create_sentence(statement)
                 create_or_update_connectivity_statement(statement, sentence, update_anatomical_entities)
+
             update_forward_connections(statements)
 
     except Exception as e:
         logger_service.add_anomaly(
-            LoggableAnomaly(statement_id=None, entity_id=None, message=str(e), severity=Severity.ERROR))
+            LoggableAnomaly(statement_id=None, entity_id=None, message=str(e),
+                            severity=Severity.ERROR)
+        )
         successful_transaction = False
         logging.error(f"Ingestion aborted due to {e}")
 
@@ -41,6 +42,3 @@ def ingest_statements(update_upstream=False, update_anatomical_entities=False):
         if update_upstream:
             update_upstream_statements()
         logger_service.write_ingested_statements_to_file(statements)
-
-
-

backend/composer/services/cs_ingestion/helpers/changes_detector.py

@@ -1,5 +1,9 @@
+import rdflib
+from neurondm import orders
+
+from composer.models import AnatomicalEntity
 from composer.services.cs_ingestion.helpers.common_helpers import VALIDATION_ERRORS, SPECIES, PROVENANCE, ID, \
-    FORWARD_CONNECTION
+    FORWARD_CONNECTION, ORIGINS, VIAS, DESTINATIONS
 from composer.services.cs_ingestion.models import ValidationErrors
 
 
@@ -40,64 +44,84 @@ def has_changes(connectivity_statement, statement, defaults):
     if current_forward_connections != new_forward_connections:
         return True
 
-    # TODO: Update
-    # # Check for changes in origins
-    # current_origins = set(origin.ontology_uri for origin in connectivity_statement.origins.all())
-    # new_origins = set(uri for uri in statement[ORIGINS].anatomical_entities if uri not in validation_errors.entities)
-    # if current_origins != new_origins:
-    #     return True
-    #
-    # # Check for changes in vias
-    # current_vias = [
-    #     {
-    #         'anatomical_entities': set(via.anatomical_entities.all().values_list('ontology_uri', flat=True)),
-    #         'from_entities': set(via.from_entities.all().values_list('ontology_uri', flat=True))
-    #     }
-    #     for via in connectivity_statement.via_set.order_by('order').all()
-    # ]
-    # new_vias = statement[VIAS]
-    #
-    # if len(current_vias) != len(new_vias):
-    #     return True
-    #
-    # for current_via, new_via in zip(current_vias, new_vias):
-    #     new_via_anatomical_entities = set(
-    #         uri for uri in new_via.anatomical_entities if uri not in validation_errors.entities)
-    #
-    #     new_via_from_entities = set(uri for uri in new_via.from_entities if uri not in validation_errors.entities)
-    #
-    #     if (new_via_anatomical_entities != current_via['anatomical_entities'] or
-    #             new_via_from_entities != current_via['from_entities']):
-    #         return True
-    #
-    # # Check for changes in destinations
-    # current_destinations = connectivity_statement.destinations.all()
-    # new_destinations = statement[DESTINATIONS]
-    #
-    # if len(current_destinations) != len(new_destinations):
-    #     return True
-    #
-    # # We may need to change this algorithm when multi-destination is supported by neurondm
-    #
-    # current_destinations_anatomical_entities = set(
-    #     uri for destination in current_destinations
-    #     for uri in destination.anatomical_entities.all().values_list('ontology_uri', flat=True)
-    # )
-    # current_destinations_from_entities = set(
-    #     uri for destination in current_destinations
-    #     for uri in destination.from_entities.all().values_list('ontology_uri', flat=True)
-    # )
-    #
-    # new_destinations_anatomical_entities = {uri for new_dest in statement[DESTINATIONS] for uri in
-    #                                         new_dest.anatomical_entities if uri not in validation_errors.entities}
-    #
-    # new_destinations_from_entities = {uri for new_dest in statement[DESTINATIONS] for uri in new_dest.from_entities if
-    #                                   uri not in validation_errors.entities}
-    #
-    # if (current_destinations_anatomical_entities != new_destinations_anatomical_entities or
-    #         current_destinations_from_entities != new_destinations_from_entities):
-    #     return True
-
-    return True
+    # Check for changes in origins
+    current_origins = set(
+        get_anatomical_entity_identifier_composer(origin) for origin in connectivity_statement.origins.all())
+    new_origins = set(
+        get_anatomical_entity_identifier_neurondm(uri) for uri in statement[ORIGINS].anatomical_entities if
+        uri not in validation_errors.entities)
+    if current_origins != new_origins:
+        return True
+
+    # Check for changes in vias
+    current_vias = [
+        {
+            'anatomical_entities': set(
+                get_anatomical_entity_identifier_composer(ae) for ae in via.anatomical_entities.all()),
+            'from_entities': set(get_anatomical_entity_identifier_composer(ae) for ae in via.from_entities.all())
+        }
+        for via in connectivity_statement.via_set.order_by('order').all()
+    ]
+    new_vias = statement[VIAS]
+
+    if len(current_vias) != len(new_vias):
+        return True
+
+    for current_via, new_via in zip(current_vias, new_vias):
+        new_via_anatomical_entities = set(
+            get_anatomical_entity_identifier_neurondm(uri) for uri in new_via.anatomical_entities if
+            uri not in validation_errors.entities)
+
+        new_via_from_entities = set(get_anatomical_entity_identifier_neurondm(uri) for uri in new_via.from_entities if
+                                    uri not in validation_errors.entities)
+
+        if (new_via_anatomical_entities != current_via['anatomical_entities'] or
+                new_via_from_entities != current_via['from_entities']):
+            return True
+
+    # Check for changes in destinations
+    current_destinations = connectivity_statement.destinations.all()
+    new_destinations = statement[DESTINATIONS]
+
+    if len(current_destinations) != len(new_destinations):
+        return True
+
+    # We may need to change this algorithm when multi-destination is supported by neurondm
+
+    current_destinations_anatomical_entities = set(
+        get_anatomical_entity_identifier_composer(uri) for destination in current_destinations
+        for uri in destination.anatomical_entities.all()
+    )
+    current_destinations_from_entities = set(
+        get_anatomical_entity_identifier_composer(uri) for destination in current_destinations
+        for uri in destination.from_entities.all()
+    )
+
+    new_destinations_anatomical_entities = {get_anatomical_entity_identifier_neurondm(uri) for new_dest in
+                                            statement[DESTINATIONS] for uri in
+                                            new_dest.anatomical_entities if uri not in validation_errors.entities}
+
+    new_destinations_from_entities = {get_anatomical_entity_identifier_neurondm(uri) for new_dest in
+                                      statement[DESTINATIONS] for uri in new_dest.from_entities if
+                                      uri not in validation_errors.entities}
+
+    if (current_destinations_anatomical_entities != new_destinations_anatomical_entities or
+            current_destinations_from_entities != new_destinations_from_entities):
+        return True
+
     # Not checking the Notes because they are kept
-    return False
+    return False
+
+
+def get_anatomical_entity_identifier_composer(entity: AnatomicalEntity):
+    if entity.region_layer:
+        layer_uri = entity.region_layer.layer.ontology_uri
+        region_uri = entity.region_layer.region.ontology_uri
+        return f"{region_uri}:{layer_uri}"
+    return entity.simple_entity.ontology_uri
+
+
+def get_anatomical_entity_identifier_neurondm(entity: rdflib.term):
+    if isinstance(entity, orders.rl):
+        return f"{str(entity.region)}:{str(entity.layer)}"
+    return str(entity)

backend/composer/services/cs_ingestion/models.py

@@ -3,21 +3,21 @@
 
 
 class NeuronDMOrigin:
-    def __init__(self, anatomical_entities_uri: Set):
-        self.anatomical_entities = anatomical_entities_uri
+    def __init__(self, anatomical_entities: Set):
+        self.anatomical_entities = anatomical_entities
 
 
 class NeuronDMVia:
-    def __init__(self, anatomical_entities_uri: Set, from_entities: Set, order: int, type: str):
-        self.anatomical_entities = anatomical_entities_uri
+    def __init__(self, anatomical_entities: Set, from_entities: Set, order: int, type: str):
+        self.anatomical_entities = anatomical_entities
         self.from_entities = from_entities
         self.order = order
         self.type = type
 
 
 class NeuronDMDestination:
-    def __init__(self, anatomical_entities_uri: Set, from_entities: Set, type: str):
-        self.anatomical_entities = anatomical_entities_uri
+    def __init__(self, anatomical_entities: Set, from_entities: Set, type: str):
+        self.anatomical_entities = anatomical_entities
         self.from_entities = from_entities
         self.type = type