Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add pre and post synaptic pathways to BG -> ES projection #4424

Merged
merged 5 commits into from
Oct 30, 2023
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package ch.epfl.bluebrain.nexus.delta.plugins.compositeviews

import cats.implicits._
import ch.epfl.bluebrain.nexus.delta.kernel.effect.migration._
import ch.epfl.bluebrain.nexus.delta.kernel.kamon.KamonMetricComponent
import ch.epfl.bluebrain.nexus.delta.kernel.syntax.kamonSyntax
import ch.epfl.bluebrain.nexus.delta.plugins.blazegraph.client.BlazegraphClient
@@ -15,7 +16,7 @@ import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.client.{ElasticSearch
import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.indexing.{ElasticSearchSink, GraphResourceToDocument}
import ch.epfl.bluebrain.nexus.delta.rdf.RdfError
import ch.epfl.bluebrain.nexus.delta.rdf.graph.Graph
import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.api.{JsonLdApi, JsonLdJavaApi}
import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.api.{JsonLdApi, JsonLdJavaApi, JsonLdOptions}
import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.{ContextValue, RemoteContextResolution}
import ch.epfl.bluebrain.nexus.delta.rdf.query.SparqlQuery.SparqlConstructQuery
import ch.epfl.bluebrain.nexus.delta.rdf.syntax.iriStringContextSyntax
@@ -25,7 +26,6 @@ import ch.epfl.bluebrain.nexus.delta.sourcing.state.GraphResource
import ch.epfl.bluebrain.nexus.delta.sourcing.stream.Elem
import ch.epfl.bluebrain.nexus.delta.sourcing.stream.Elem.{DroppedElem, FailedElem, SuccessElem}
import ch.epfl.bluebrain.nexus.delta.sourcing.stream.Operation.Sink
import ch.epfl.bluebrain.nexus.delta.kernel.effect.migration._
import fs2.Chunk
import monix.bio.Task
import shapeless.Typeable
@@ -199,7 +199,8 @@ object CompositeSink {
common: String,
cfg: CompositeViewsConfig
)(implicit rcr: RemoteContextResolution): ElasticSearchProjection => CompositeSink = { target =>
val esSink =
implicit val jsonLdOptions: JsonLdOptions = JsonLdOptions.AlwaysEmbed
val esSink =
ElasticSearchSink.states(
esClient,
cfg.elasticsearchBatch.maxElements,
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
package ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.indexing

import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.api.{JsonLdApi, JsonLdJavaApi}
import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.api.{JsonLdApi, JsonLdJavaApi, JsonLdOptions}
import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.JsonLdContext.keywords
import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.{ContextValue, RemoteContextResolution}
import ch.epfl.bluebrain.nexus.delta.sdk.implicits._
@@ -19,7 +19,8 @@ import shapeless.Typeable
* a context to compute the compacted JSON-LD for of the [[GraphResource]]
*/
final class GraphResourceToDocument(context: ContextValue, includeContext: Boolean)(implicit
cr: RemoteContextResolution
cr: RemoteContextResolution,
jsonLdOptions: JsonLdOptions
) extends Pipe {
override type In = GraphResource
override type Out = Json
Original file line number Diff line number Diff line change
@@ -29,4 +29,5 @@ final case class JsonLdOptions(

object JsonLdOptions {
implicit val defaults: JsonLdOptions = JsonLdOptions()
val AlwaysEmbed: JsonLdOptions = defaults.copy(embed = "@always")
}
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
package ch.epfl.bluebrain.nexus.delta.rdf.jsonld

import ch.epfl.bluebrain.nexus.delta.rdf.{Fixtures, GraphHelpers}
import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.BNode
import ch.epfl.bluebrain.nexus.delta.rdf.implicits._
import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.ContextValue
import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.JsonLdContext.keywords
import ch.epfl.bluebrain.nexus.delta.rdf.{Fixtures, GraphHelpers}
import ch.epfl.bluebrain.nexus.testkit.scalatest.ce.CatsEffectSpec

class CompactedJsonLdSpec extends CatsEffectSpec with Fixtures with GraphHelpers {
Original file line number Diff line number Diff line change
@@ -39,9 +39,7 @@ trait IndexingAction {
*/
def projections(project: ProjectRef, elem: Elem[GraphResource]): ElemStream[CompiledProjection]

def apply(project: ProjectRef, elem: Elem[GraphResource])(implicit
contextShift: ContextShift[IO]
): IO[List[FailedElem]] = {
def apply(project: ProjectRef, elem: Elem[GraphResource]): IO[List[FailedElem]] = {
for {
// To collect the errors
errorsRef <- Ref.of[IO, List[FailedElem]](List.empty)
Original file line number Diff line number Diff line change
@@ -2,15 +2,15 @@ package ch.epfl.bluebrain.nexus.delta.sdk

import cats.effect.IO
import cats.syntax.all._
import ch.epfl.bluebrain.nexus.delta.kernel.effect.migration._
import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri
import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.api.{JsonLdApi, JsonLdJavaApi}
import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.api.{JsonLdApi, JsonLdJavaApi, JsonLdOptions}
import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.JsonLdContext.keywords
import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.RemoteContextResolution
import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.encoder.JsonLdEncoder
import ch.epfl.bluebrain.nexus.delta.sdk.jsonld.JsonLdContent
import ch.epfl.bluebrain.nexus.delta.sdk.model.{BaseUri, ResourceF}
import ch.epfl.bluebrain.nexus.delta.sdk.syntax._
import ch.epfl.bluebrain.nexus.delta.kernel.effect.migration._
import ch.epfl.bluebrain.nexus.delta.sourcing.Serializer
import ch.epfl.bluebrain.nexus.delta.sourcing.model.{EntityType, ProjectRef, ResourceRef}
import ch.epfl.bluebrain.nexus.delta.sourcing.offset.Offset
@@ -114,7 +114,7 @@ abstract class ResourceShift[State <: ScopedState, A, M](
)
}

private def encodeMetadata(id: Iri, metadata: Option[M])(implicit cr: RemoteContextResolution) =
private def encodeMetadata(id: Iri, metadata: Option[M])(implicit cr: RemoteContextResolution, opts: JsonLdOptions) =
(metadata, metadataEncoder) match {
case (Some(m), Some(e)) => e.graph(m).toCatsIO.map { g => Some(g.replaceRootNode(id)) }
case (_, _) => IO.none
Original file line number Diff line number Diff line change
@@ -3,10 +3,20 @@ package ch.epfl.bluebrain.nexus.testkit
import ch.epfl.bluebrain.nexus.testkit.CirceEq.IgnoredArrayOrder
import io.circe._
import io.circe.syntax._
import org.scalatest.matchers.{MatchResult, Matcher}
import org.scalatest.matchers.{HavePropertyMatchResult, HavePropertyMatcher, MatchResult, Matcher}

trait CirceEq {
def equalIgnoreArrayOrder(json: Json): IgnoredArrayOrder = IgnoredArrayOrder(json)

def field(fieldName: String, expectedValue: Json): HavePropertyMatcher[Json, Json] = HavePropertyMatcher(left => {
val actualValue = left.hcursor.downField(fieldName).as[Json].getOrElse(Json.Null)
HavePropertyMatchResult(
actualValue == expectedValue,
fieldName,
expectedValue,
actualValue
)
})
}

object CirceEq {
24 changes: 24 additions & 0 deletions tests/docker/config/construct-query.sparql
Original file line number Diff line number Diff line change
@@ -183,6 +183,18 @@ CONSTRUCT {
?alias :campaign ?campaignId .
?campaignId :identifier ?campaignId ;
:name ?campaignName .

## Synapses
?alias :postSynapticPathway ?postSynaptic .
?postSynaptic :about ?postSynapticAbout ;
:label ?postSynapticLabel ;
:notation ?postSynapticNotation .

?alias :preSynapticPathway ?preSynaptic .
?preSynaptic :about ?preSynapticAbout ;
:label ?preSynapticLabel ;
:notation ?preSynapticNotation .

} WHERE {
VALUES ?id { {resource_id} } .
BIND( IRI(concat(str(?id), '/', 'alias')) AS ?alias ) .
@@ -633,4 +645,16 @@ CONSTRUCT {
} .
} .

# Synapses
OPTIONAL {
?id bmo:synapticPathway / nsg:postSynaptic ?postSynaptic .
?postSynaptic schema:about ?postSynapticAbout ;
rdfs:label ?postSynapticLabel .
OPTIONAL { ?postSynaptic skos:notation ?postSynapticNotation . } .
?id bmo:synapticPathway / nsg:preSynaptic ?preSynaptic .
?preSynaptic schema:about ?preSynapticAbout ;
rdfs:label ?preSynapticLabel .
OPTIONAL { ?preSynaptic skos:notation ?preSynapticNotation . } .

} .
}
6 changes: 6 additions & 0 deletions tests/docker/config/search-context.json
Original file line number Diff line number Diff line change
@@ -30,6 +30,12 @@
"protocol": {
"@container": "@set"
},
"preSynapticPathway": {
"@container": "@set"
},
"postSynapticPathway": {
"@container": "@set"
},
"project": {
"@type": "@id"
},
16 changes: 16 additions & 0 deletions tests/src/test/resources/kg/search/id-query-single-field.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
{
"_source": [
"{{field}}"
],
"query": {
"bool": {
"filter": [
{
"term": {
"@id.keyword": "{{id}}"
}
}
]
}
}
}
3 changes: 0 additions & 3 deletions tests/src/test/resources/kg/search/id-query.json
Original file line number Diff line number Diff line change
@@ -1,7 +1,4 @@
{
"_source": [
"{{field}}"
],
"query": {
"bool": {
"filter": [
31 changes: 31 additions & 0 deletions tests/src/test/resources/kg/search/synapse-two-pathways.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
{
"@context": "https://bbp.neuroshapes.org",
"@id": "https://bbp.epfl.ch/data/synapse-two-pathways",
"@type": [
"Entity",
"Parameter",
"SchafferCollateralAnatomyParameter",
"SynapsesPerConnection",
"ExperimentalSynapsesPerConnection"
],
"description": "SynapseDensity parameter from Schaffer axon collateral (pre-synaptic) to PV+ (post-synaptic) neurons. These data are part of the Schaffer collateral anatomy data.",
"name": "SynapseDensity parameter from Schaffer axon collateral (pre-synaptic) to PV+ (post-synaptic) neurons",
"synapticPathway": {
"postSynaptic": [
{
"@id": "http://api.brain-map.org/api/v2/data/Structure/454",
"about": "OtherBrainRegion",
"label": "Other somatosensory areas",
"notation": "OSS"
}
],
"preSynaptic": [
{
"@id": "http://api.brain-map.org/api/v2/data/Structure/453",
"about": "BrainRegion",
"label": "Somatosensory areas",
"notation": "SS"
}
]
}
}
31 changes: 31 additions & 0 deletions tests/src/test/resources/kg/search/synapse.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
{
"@context": "https://bbp.neuroshapes.org",
"@id": "https://bbp.epfl.ch/data/synapse",
"@type": [
"Entity",
"Parameter",
"SchafferCollateralAnatomyParameter",
"SynapsesPerConnection",
"ExperimentalSynapsesPerConnection"
],
"description": "SynapseDensity parameter from Schaffer axon collateral (pre-synaptic) to PV+ (post-synaptic) neurons. These data are part of the Schaffer collateral anatomy data.",
"name": "SynapseDensity parameter from Schaffer axon collateral (pre-synaptic) to PV+ (post-synaptic) neurons",
"synapticPathway": {
"postSynaptic": [
{
"@id": "http://api.brain-map.org/api/v2/data/Structure/453",
"about": "BrainRegion",
"label": "Somatosensory areas",
"notation": "SS"
}
],
"preSynaptic": [
{
"@id": "http://api.brain-map.org/api/v2/data/Structure/453",
"about": "BrainRegion",
"label": "Somatosensory areas",
"notation": "SS"
}
]
}
}
Original file line number Diff line number Diff line change
@@ -7,12 +7,13 @@ import ch.epfl.bluebrain.nexus.tests.BaseIntegrationSpec
import ch.epfl.bluebrain.nexus.tests.Identity.resources.Rick
import ch.epfl.bluebrain.nexus.tests.iam.types.Permission.{Organizations, Resources}
import io.circe.Json
import org.scalactic.source.Position
import org.scalatest.Assertion

import java.time.Instant
import concurrent.duration._

class SearchConfigSpec extends BaseIntegrationSpec {
class SearchConfigIndexingSpec extends BaseIntegrationSpec {

implicit override def patienceConfig: PatienceConfig = PatienceConfig(config.patience * 2, 300.millis)

@@ -30,6 +31,8 @@ class SearchConfigSpec extends BaseIntegrationSpec {
private val boutonDensityId = "https://bbp.epfl.ch/data/bouton-density"
private val simulationCampaignId = "https://bbp.epfl.ch/data/simulation-campaign"
private val simulationId = "https://bbp.epfl.ch/data/simulation"
private val synapseId = "https://bbp.epfl.ch/data/synapse"
private val synapseTwoPathwaysId = "https://bbp.epfl.ch/data/synapse-two-pathways"
private val detailedCircuitId = "https://bbp.epfl.ch/data/detailed-circuit"

// the resources that should appear in the search index
@@ -40,6 +43,8 @@ class SearchConfigSpec extends BaseIntegrationSpec {
"/kg/search/unassessed-trace.json",
"/kg/search/neuron-morphology.json",
"/kg/search/neuron-density.json",
"/kg/search/synapse.json",
"/kg/search/synapse-two-pathways.json",
"/kg/search/layer-thickness.json",
"/kg/search/bouton-density.json",
"/kg/search/detailed-circuit.json",
@@ -784,14 +789,66 @@ class SearchConfigSpec extends BaseIntegrationSpec {
}
}

"have the correct synaptic pathways when they are different" in {
assertOneSource(queryDocument(synapseTwoPathwaysId)) { json =>
json should have(
field(
"preSynapticPathway",
json"""[
{
"@id": "http://api.brain-map.org/api/v2/data/Structure/453",
"about": "https://bbp.epfl.ch/neurosciencegraph/data/BrainRegion",
"label": "Somatosensory areas",
"notation": "SS"
}
]"""
)
)
json should have(
field(
"postSynapticPathway",
json"""[
{
"@id": "http://api.brain-map.org/api/v2/data/Structure/454",
"about": "https://bbp.epfl.ch/neurosciencegraph/data/OtherBrainRegion",
"label": "Other somatosensory areas",
"notation": "OSS"
}
]"""
)
)
}

}

"have the correct synaptic pathways when they are the same" in {
val singlePathway =
json"""
[
{
"@id": "http://api.brain-map.org/api/v2/data/Structure/453",
"about": "https://bbp.epfl.ch/neurosciencegraph/data/BrainRegion",
"label": "Somatosensory areas",
"notation": "SS"
}
]"""

assertOneSource(queryDocument(synapseId)) { json =>
json should have(field("preSynapticPathway", singlePathway))
json should have(field("postSynapticPathway", singlePathway))
}
}
}

/**
* Defines an ES query that searches for the document with the provided id and limits the resulting source to just
* the requested field
*/
private def queryField(id: String, field: String) =
jsonContentOf("/kg/search/id-query.json", "id" -> id, "field" -> field)
jsonContentOf("/kg/search/id-query-single-field.json", "id" -> id, "field" -> field)

private def queryDocument(id: String) =
jsonContentOf("/kg/search/id-query.json", "id" -> id)

/** Post a resource across all defined projects in the suite */
private def postResource(resourcePath: String): IO[List[Assertion]] = {
@@ -809,20 +866,35 @@ class SearchConfigSpec extends BaseIntegrationSpec {
* Queries ES using the provided query. Asserts that there is only on result in _source. Runs the provided assertion
* on the _source.
*/
private def assertOneSource(query: Json)(assertion: Json => Assertion): IO[Assertion] =
private def assertOneSource(query: Json)(assertion: Json => Assertion)(implicit pos: Position): IO[Assertion] =
eventually {
deltaClient.post[Json]("/search/query", query, Rick) { (body, response) =>
response.status shouldEqual StatusCodes.OK
val sources = Json.fromValues(body.findAllByKey("_source"))
val source = sources.hcursor.downArray.as[Json]
val actual = source.getOrElse(Json.Null)

sources.asArray.value.size shouldBe 1
assertion(actual)
val results = Json
.fromValues(body.findAllByKey("_source"))
.hcursor
.as[List[Json]]
.getOrElse(Nil)

results match {
case single :: Nil => assertion(single)
case Nil =>
fail(
s"Expected exactly 1 source to match query, got 0.\n " +
s"Query was ${query.spaces2}"
)
case many =>
fail(
s"Expected exactly 1 source to match query, got ${many.size}.\n" +
s"Query was: ${query.spaces2}\n" +
s"Results were: $results"
)
}
}
}

private def assertEmpty(query: Json): IO[Assertion] =
private def assertEmpty(query: Json)(implicit pos: Position): IO[Assertion] =
assertOneSource(query)(j => assert(j == json"""{ }"""))

/** Check that a given field in the json can be parsed as [[Instant]] */