diff --git a/delta/plugins/composite-views/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/compositeviews/CompositeSink.scala b/delta/plugins/composite-views/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/compositeviews/CompositeSink.scala index 027242b678..03b93f80db 100644 --- a/delta/plugins/composite-views/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/compositeviews/CompositeSink.scala +++ b/delta/plugins/composite-views/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/compositeviews/CompositeSink.scala @@ -1,6 +1,7 @@ package ch.epfl.bluebrain.nexus.delta.plugins.compositeviews import cats.implicits._ +import ch.epfl.bluebrain.nexus.delta.kernel.effect.migration._ import ch.epfl.bluebrain.nexus.delta.kernel.kamon.KamonMetricComponent import ch.epfl.bluebrain.nexus.delta.kernel.syntax.kamonSyntax import ch.epfl.bluebrain.nexus.delta.plugins.blazegraph.client.BlazegraphClient @@ -15,7 +16,7 @@ import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.client.{ElasticSearch import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.indexing.{ElasticSearchSink, GraphResourceToDocument} import ch.epfl.bluebrain.nexus.delta.rdf.RdfError import ch.epfl.bluebrain.nexus.delta.rdf.graph.Graph -import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.api.{JsonLdApi, JsonLdJavaApi} +import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.api.{JsonLdApi, JsonLdJavaApi, JsonLdOptions} import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.{ContextValue, RemoteContextResolution} import ch.epfl.bluebrain.nexus.delta.rdf.query.SparqlQuery.SparqlConstructQuery import ch.epfl.bluebrain.nexus.delta.rdf.syntax.iriStringContextSyntax @@ -25,7 +26,6 @@ import ch.epfl.bluebrain.nexus.delta.sourcing.state.GraphResource import ch.epfl.bluebrain.nexus.delta.sourcing.stream.Elem import ch.epfl.bluebrain.nexus.delta.sourcing.stream.Elem.{DroppedElem, FailedElem, SuccessElem} import ch.epfl.bluebrain.nexus.delta.sourcing.stream.Operation.Sink -import ch.epfl.bluebrain.nexus.delta.kernel.effect.migration._ import fs2.Chunk import monix.bio.Task import shapeless.Typeable @@ -199,7 +199,8 @@ object CompositeSink { common: String, cfg: CompositeViewsConfig )(implicit rcr: RemoteContextResolution): ElasticSearchProjection => CompositeSink = { target => - val esSink = + implicit val jsonLdOptions: JsonLdOptions = JsonLdOptions.AlwaysEmbed + val esSink = ElasticSearchSink.states( esClient, cfg.elasticsearchBatch.maxElements, diff --git a/delta/plugins/elasticsearch/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/elasticsearch/indexing/GraphResourceToDocument.scala b/delta/plugins/elasticsearch/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/elasticsearch/indexing/GraphResourceToDocument.scala index 51bc9ddf5f..1c418f98a5 100644 --- a/delta/plugins/elasticsearch/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/elasticsearch/indexing/GraphResourceToDocument.scala +++ b/delta/plugins/elasticsearch/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/elasticsearch/indexing/GraphResourceToDocument.scala @@ -1,6 +1,6 @@ package ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.indexing -import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.api.{JsonLdApi, JsonLdJavaApi} +import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.api.{JsonLdApi, JsonLdJavaApi, JsonLdOptions} import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.JsonLdContext.keywords import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.{ContextValue, RemoteContextResolution} import ch.epfl.bluebrain.nexus.delta.sdk.implicits._ @@ -19,7 +19,8 @@ import shapeless.Typeable * a context to compute the compacted JSON-LD for of the [[GraphResource]] */ final class GraphResourceToDocument(context: ContextValue, includeContext: Boolean)(implicit - cr: RemoteContextResolution + cr: RemoteContextResolution, + jsonLdOptions: JsonLdOptions ) extends Pipe { override type In = GraphResource override type Out = Json diff --git a/delta/rdf/src/main/scala/ch/epfl/bluebrain/nexus/delta/rdf/jsonld/api/JsonLdOptions.scala b/delta/rdf/src/main/scala/ch/epfl/bluebrain/nexus/delta/rdf/jsonld/api/JsonLdOptions.scala index 87d55591e8..96b08b3a21 100644 --- a/delta/rdf/src/main/scala/ch/epfl/bluebrain/nexus/delta/rdf/jsonld/api/JsonLdOptions.scala +++ b/delta/rdf/src/main/scala/ch/epfl/bluebrain/nexus/delta/rdf/jsonld/api/JsonLdOptions.scala @@ -29,4 +29,5 @@ final case class JsonLdOptions( object JsonLdOptions { implicit val defaults: JsonLdOptions = JsonLdOptions() + val AlwaysEmbed: JsonLdOptions = defaults.copy(embed = "@always") } diff --git a/delta/rdf/src/test/scala/ch/epfl/bluebrain/nexus/delta/rdf/jsonld/CompactedJsonLdSpec.scala b/delta/rdf/src/test/scala/ch/epfl/bluebrain/nexus/delta/rdf/jsonld/CompactedJsonLdSpec.scala index f2cb11d5cc..8f18dd8eae 100644 --- a/delta/rdf/src/test/scala/ch/epfl/bluebrain/nexus/delta/rdf/jsonld/CompactedJsonLdSpec.scala +++ b/delta/rdf/src/test/scala/ch/epfl/bluebrain/nexus/delta/rdf/jsonld/CompactedJsonLdSpec.scala @@ -1,10 +1,10 @@ package ch.epfl.bluebrain.nexus.delta.rdf.jsonld -import ch.epfl.bluebrain.nexus.delta.rdf.{Fixtures, GraphHelpers} import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.BNode import ch.epfl.bluebrain.nexus.delta.rdf.implicits._ import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.ContextValue import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.JsonLdContext.keywords +import ch.epfl.bluebrain.nexus.delta.rdf.{Fixtures, GraphHelpers} import ch.epfl.bluebrain.nexus.testkit.scalatest.ce.CatsEffectSpec class CompactedJsonLdSpec extends CatsEffectSpec with Fixtures with GraphHelpers { diff --git a/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/IndexingAction.scala b/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/IndexingAction.scala index 4ab0bf685f..5a1aed08b4 100644 --- a/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/IndexingAction.scala +++ b/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/IndexingAction.scala @@ -39,9 +39,7 @@ trait IndexingAction { */ def projections(project: ProjectRef, elem: Elem[GraphResource]): ElemStream[CompiledProjection] - def apply(project: ProjectRef, elem: Elem[GraphResource])(implicit - contextShift: ContextShift[IO] - ): IO[List[FailedElem]] = { + def apply(project: ProjectRef, elem: Elem[GraphResource]): IO[List[FailedElem]] = { for { // To collect the errors errorsRef <- Ref.of[IO, List[FailedElem]](List.empty) diff --git a/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/ResourceShift.scala b/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/ResourceShift.scala index 47f36b9125..7dcb443ca3 100644 --- a/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/ResourceShift.scala +++ b/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/ResourceShift.scala @@ -2,15 +2,15 @@ package ch.epfl.bluebrain.nexus.delta.sdk import cats.effect.IO import cats.syntax.all._ +import ch.epfl.bluebrain.nexus.delta.kernel.effect.migration._ import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri -import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.api.{JsonLdApi, JsonLdJavaApi} +import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.api.{JsonLdApi, JsonLdJavaApi, JsonLdOptions} import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.JsonLdContext.keywords import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.RemoteContextResolution import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.encoder.JsonLdEncoder import ch.epfl.bluebrain.nexus.delta.sdk.jsonld.JsonLdContent import ch.epfl.bluebrain.nexus.delta.sdk.model.{BaseUri, ResourceF} import ch.epfl.bluebrain.nexus.delta.sdk.syntax._ -import ch.epfl.bluebrain.nexus.delta.kernel.effect.migration._ import ch.epfl.bluebrain.nexus.delta.sourcing.Serializer import ch.epfl.bluebrain.nexus.delta.sourcing.model.{EntityType, ProjectRef, ResourceRef} import ch.epfl.bluebrain.nexus.delta.sourcing.offset.Offset @@ -114,7 +114,7 @@ abstract class ResourceShift[State <: ScopedState, A, M]( ) } - private def encodeMetadata(id: Iri, metadata: Option[M])(implicit cr: RemoteContextResolution) = + private def encodeMetadata(id: Iri, metadata: Option[M])(implicit cr: RemoteContextResolution, opts: JsonLdOptions) = (metadata, metadataEncoder) match { case (Some(m), Some(e)) => e.graph(m).toCatsIO.map { g => Some(g.replaceRootNode(id)) } case (_, _) => IO.none diff --git a/delta/testkit/src/main/scala/ch/epfl/bluebrain/nexus/testkit/CirceEq.scala b/delta/testkit/src/main/scala/ch/epfl/bluebrain/nexus/testkit/CirceEq.scala index e200617a5b..b00367ea29 100644 --- a/delta/testkit/src/main/scala/ch/epfl/bluebrain/nexus/testkit/CirceEq.scala +++ b/delta/testkit/src/main/scala/ch/epfl/bluebrain/nexus/testkit/CirceEq.scala @@ -3,10 +3,20 @@ package ch.epfl.bluebrain.nexus.testkit import ch.epfl.bluebrain.nexus.testkit.CirceEq.IgnoredArrayOrder import io.circe._ import io.circe.syntax._ -import org.scalatest.matchers.{MatchResult, Matcher} +import org.scalatest.matchers.{HavePropertyMatchResult, HavePropertyMatcher, MatchResult, Matcher} trait CirceEq { def equalIgnoreArrayOrder(json: Json): IgnoredArrayOrder = IgnoredArrayOrder(json) + + def field(fieldName: String, expectedValue: Json): HavePropertyMatcher[Json, Json] = HavePropertyMatcher(left => { + val actualValue = left.hcursor.downField(fieldName).as[Json].getOrElse(Json.Null) + HavePropertyMatchResult( + actualValue == expectedValue, + fieldName, + expectedValue, + actualValue + ) + }) } object CirceEq { diff --git a/tests/docker/config/construct-query.sparql b/tests/docker/config/construct-query.sparql index a1d3d79fb0..ac381d6b48 100644 --- a/tests/docker/config/construct-query.sparql +++ b/tests/docker/config/construct-query.sparql @@ -183,6 +183,18 @@ CONSTRUCT { ?alias :campaign ?campaignId . ?campaignId :identifier ?campaignId ; :name ?campaignName . + + ## Synapses + ?alias :postSynapticPathway ?postSynaptic . + ?postSynaptic :about ?postSynapticAbout ; + :label ?postSynapticLabel ; + :notation ?postSynapticNotation . + + ?alias :preSynapticPathway ?preSynaptic . + ?preSynaptic :about ?preSynapticAbout ; + :label ?preSynapticLabel ; + :notation ?preSynapticNotation . + } WHERE { VALUES ?id { {resource_id} } . BIND( IRI(concat(str(?id), '/', 'alias')) AS ?alias ) . @@ -633,4 +645,16 @@ CONSTRUCT { } . } . + # Synapses + OPTIONAL { + ?id bmo:synapticPathway / nsg:postSynaptic ?postSynaptic . + ?postSynaptic schema:about ?postSynapticAbout ; + rdfs:label ?postSynapticLabel . + OPTIONAL { ?postSynaptic skos:notation ?postSynapticNotation . } . + ?id bmo:synapticPathway / nsg:preSynaptic ?preSynaptic . + ?preSynaptic schema:about ?preSynapticAbout ; + rdfs:label ?preSynapticLabel . + OPTIONAL { ?preSynaptic skos:notation ?preSynapticNotation . } . + + } . } \ No newline at end of file diff --git a/tests/docker/config/search-context.json b/tests/docker/config/search-context.json index 729d113c73..8b46b1612e 100644 --- a/tests/docker/config/search-context.json +++ b/tests/docker/config/search-context.json @@ -30,6 +30,12 @@ "protocol": { "@container": "@set" }, + "preSynapticPathway": { + "@container": "@set" + }, + "postSynapticPathway": { + "@container": "@set" + }, "project": { "@type": "@id" }, diff --git a/tests/src/test/resources/kg/search/id-query-single-field.json b/tests/src/test/resources/kg/search/id-query-single-field.json new file mode 100644 index 0000000000..587fb3cd4a --- /dev/null +++ b/tests/src/test/resources/kg/search/id-query-single-field.json @@ -0,0 +1,16 @@ +{ + "_source": [ + "{{field}}" + ], + "query": { + "bool": { + "filter": [ + { + "term": { + "@id.keyword": "{{id}}" + } + } + ] + } + } +} \ No newline at end of file diff --git a/tests/src/test/resources/kg/search/id-query.json b/tests/src/test/resources/kg/search/id-query.json index 587fb3cd4a..7ef892df70 100644 --- a/tests/src/test/resources/kg/search/id-query.json +++ b/tests/src/test/resources/kg/search/id-query.json @@ -1,7 +1,4 @@ { - "_source": [ - "{{field}}" - ], "query": { "bool": { "filter": [ diff --git a/tests/src/test/resources/kg/search/synapse-two-pathways.json b/tests/src/test/resources/kg/search/synapse-two-pathways.json new file mode 100644 index 0000000000..e075c8907d --- /dev/null +++ b/tests/src/test/resources/kg/search/synapse-two-pathways.json @@ -0,0 +1,31 @@ +{ + "@context": "https://bbp.neuroshapes.org", + "@id": "https://bbp.epfl.ch/data/synapse-two-pathways", + "@type": [ + "Entity", + "Parameter", + "SchafferCollateralAnatomyParameter", + "SynapsesPerConnection", + "ExperimentalSynapsesPerConnection" + ], + "description": "SynapseDensity parameter from Schaffer axon collateral (pre-synaptic) to PV+ (post-synaptic) neurons. These data are part of the Schaffer collateral anatomy data.", + "name": "SynapseDensity parameter from Schaffer axon collateral (pre-synaptic) to PV+ (post-synaptic) neurons", + "synapticPathway": { + "postSynaptic": [ + { + "@id": "http://api.brain-map.org/api/v2/data/Structure/454", + "about": "OtherBrainRegion", + "label": "Other somatosensory areas", + "notation": "OSS" + } + ], + "preSynaptic": [ + { + "@id": "http://api.brain-map.org/api/v2/data/Structure/453", + "about": "BrainRegion", + "label": "Somatosensory areas", + "notation": "SS" + } + ] + } +} \ No newline at end of file diff --git a/tests/src/test/resources/kg/search/synapse.json b/tests/src/test/resources/kg/search/synapse.json new file mode 100644 index 0000000000..7e42f964f4 --- /dev/null +++ b/tests/src/test/resources/kg/search/synapse.json @@ -0,0 +1,31 @@ +{ + "@context": "https://bbp.neuroshapes.org", + "@id": "https://bbp.epfl.ch/data/synapse", + "@type": [ + "Entity", + "Parameter", + "SchafferCollateralAnatomyParameter", + "SynapsesPerConnection", + "ExperimentalSynapsesPerConnection" + ], + "description": "SynapseDensity parameter from Schaffer axon collateral (pre-synaptic) to PV+ (post-synaptic) neurons. These data are part of the Schaffer collateral anatomy data.", + "name": "SynapseDensity parameter from Schaffer axon collateral (pre-synaptic) to PV+ (post-synaptic) neurons", + "synapticPathway": { + "postSynaptic": [ + { + "@id": "http://api.brain-map.org/api/v2/data/Structure/453", + "about": "BrainRegion", + "label": "Somatosensory areas", + "notation": "SS" + } + ], + "preSynaptic": [ + { + "@id": "http://api.brain-map.org/api/v2/data/Structure/453", + "about": "BrainRegion", + "label": "Somatosensory areas", + "notation": "SS" + } + ] + } +} \ No newline at end of file diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/SearchConfigSpec.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/SearchConfigIndexingSpec.scala similarity index 89% rename from tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/SearchConfigSpec.scala rename to tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/SearchConfigIndexingSpec.scala index 84a4c761bd..a9a1999c50 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/SearchConfigSpec.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/SearchConfigIndexingSpec.scala @@ -7,12 +7,13 @@ import ch.epfl.bluebrain.nexus.tests.BaseIntegrationSpec import ch.epfl.bluebrain.nexus.tests.Identity.resources.Rick import ch.epfl.bluebrain.nexus.tests.iam.types.Permission.{Organizations, Resources} import io.circe.Json +import org.scalactic.source.Position import org.scalatest.Assertion import java.time.Instant import concurrent.duration._ -class SearchConfigSpec extends BaseIntegrationSpec { +class SearchConfigIndexingSpec extends BaseIntegrationSpec { implicit override def patienceConfig: PatienceConfig = PatienceConfig(config.patience * 2, 300.millis) @@ -30,6 +31,8 @@ class SearchConfigSpec extends BaseIntegrationSpec { private val boutonDensityId = "https://bbp.epfl.ch/data/bouton-density" private val simulationCampaignId = "https://bbp.epfl.ch/data/simulation-campaign" private val simulationId = "https://bbp.epfl.ch/data/simulation" + private val synapseId = "https://bbp.epfl.ch/data/synapse" + private val synapseTwoPathwaysId = "https://bbp.epfl.ch/data/synapse-two-pathways" private val detailedCircuitId = "https://bbp.epfl.ch/data/detailed-circuit" // the resources that should appear in the search index @@ -40,6 +43,8 @@ class SearchConfigSpec extends BaseIntegrationSpec { "/kg/search/unassessed-trace.json", "/kg/search/neuron-morphology.json", "/kg/search/neuron-density.json", + "/kg/search/synapse.json", + "/kg/search/synapse-two-pathways.json", "/kg/search/layer-thickness.json", "/kg/search/bouton-density.json", "/kg/search/detailed-circuit.json", @@ -784,6 +789,55 @@ class SearchConfigSpec extends BaseIntegrationSpec { } } + "have the correct synaptic pathways when they are different" in { + assertOneSource(queryDocument(synapseTwoPathwaysId)) { json => + json should have( + field( + "preSynapticPathway", + json"""[ + { + "@id": "http://api.brain-map.org/api/v2/data/Structure/453", + "about": "https://bbp.epfl.ch/neurosciencegraph/data/BrainRegion", + "label": "Somatosensory areas", + "notation": "SS" + } + ]""" + ) + ) + json should have( + field( + "postSynapticPathway", + json"""[ + { + "@id": "http://api.brain-map.org/api/v2/data/Structure/454", + "about": "https://bbp.epfl.ch/neurosciencegraph/data/OtherBrainRegion", + "label": "Other somatosensory areas", + "notation": "OSS" + } + ]""" + ) + ) + } + + } + + "have the correct synaptic pathways when they are the same" in { + val singlePathway = + json""" + [ + { + "@id": "http://api.brain-map.org/api/v2/data/Structure/453", + "about": "https://bbp.epfl.ch/neurosciencegraph/data/BrainRegion", + "label": "Somatosensory areas", + "notation": "SS" + } + ]""" + + assertOneSource(queryDocument(synapseId)) { json => + json should have(field("preSynapticPathway", singlePathway)) + json should have(field("postSynapticPathway", singlePathway)) + } + } } /** @@ -791,7 +845,10 @@ class SearchConfigSpec extends BaseIntegrationSpec { * the requested field */ private def queryField(id: String, field: String) = - jsonContentOf("/kg/search/id-query.json", "id" -> id, "field" -> field) + jsonContentOf("/kg/search/id-query-single-field.json", "id" -> id, "field" -> field) + + private def queryDocument(id: String) = + jsonContentOf("/kg/search/id-query.json", "id" -> id) /** Post a resource across all defined projects in the suite */ private def postResource(resourcePath: String): IO[List[Assertion]] = { @@ -809,20 +866,35 @@ class SearchConfigSpec extends BaseIntegrationSpec { * Queries ES using the provided query. Asserts that there is only on result in _source. Runs the provided assertion * on the _source. */ - private def assertOneSource(query: Json)(assertion: Json => Assertion): IO[Assertion] = + private def assertOneSource(query: Json)(assertion: Json => Assertion)(implicit pos: Position): IO[Assertion] = eventually { deltaClient.post[Json]("/search/query", query, Rick) { (body, response) => response.status shouldEqual StatusCodes.OK - val sources = Json.fromValues(body.findAllByKey("_source")) - val source = sources.hcursor.downArray.as[Json] - val actual = source.getOrElse(Json.Null) - sources.asArray.value.size shouldBe 1 - assertion(actual) + val results = Json + .fromValues(body.findAllByKey("_source")) + .hcursor + .as[List[Json]] + .getOrElse(Nil) + + results match { + case single :: Nil => assertion(single) + case Nil => + fail( + s"Expected exactly 1 source to match query, got 0.\n " + + s"Query was ${query.spaces2}" + ) + case many => + fail( + s"Expected exactly 1 source to match query, got ${many.size}.\n" + + s"Query was: ${query.spaces2}\n" + + s"Results were: $results" + ) + } } } - private def assertEmpty(query: Json): IO[Assertion] = + private def assertEmpty(query: Json)(implicit pos: Position): IO[Assertion] = assertOneSource(query)(j => assert(j == json"""{ }""")) /** Check that a given field in the json can be parsed as [[Instant]] */