From 317b485f373a068c61a91de52a98994df149b34b Mon Sep 17 00:00:00 2001 From: Benjamin Benoist Date: Wed, 12 Jun 2024 09:42:54 +0200 Subject: [PATCH] Address review feedback --- .../common/enrichments/AtomicFields.scala | 166 +++++++++--------- .../enrichments/EnrichmentManager.scala | 10 +- .../common/enrichments/Failure.scala | 23 ++- .../enrichments/AtomicFieldsSpec.scala | 28 +-- .../enrichments/EnrichmentManagerSpec.scala | 9 +- .../utils/IgluUtilsSpec.scala | 66 ++++--- 6 files changed, 159 insertions(+), 143 deletions(-) diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/AtomicFields.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/AtomicFields.scala index a34b028bb..0d4c4b9ab 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/AtomicFields.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/AtomicFields.scala @@ -41,90 +41,90 @@ object AtomicFields { // format: off val supportedFields: List[AtomicField] = List( - AtomicField(name = "app_id", _.app_id, _.app_id = null), - AtomicField(name = "platform", _.platform, _.platform = null), - AtomicField(name = "event", _.event, _.event = null), - AtomicField(name = "event_id", _.event_id, _ => ()), // required in loading - AtomicField(name = "name_tracker", _.name_tracker, _.name_tracker = null), - AtomicField(name = "v_tracker", _.v_tracker, _.v_tracker = null), - AtomicField(name = "v_collector", _.v_collector, _ => ()), // required in loading - AtomicField(name = "v_etl", _.v_etl, _ => ()), // required in loading - AtomicField(name = "user_id", _.user_id, _.user_id = null), - AtomicField(name = "user_ipaddress", _.user_ipaddress, _.user_ipaddress = null), - AtomicField(name = "user_fingerprint", _.user_fingerprint, _.user_fingerprint = null), - AtomicField(name = "domain_userid", _.domain_userid, _.domain_userid = null), - AtomicField(name = "network_userid", _.network_userid, _.network_userid = null), - AtomicField(name = "geo_country", _.geo_country, _.geo_country = null), - AtomicField(name = "geo_region", _.geo_region, _.geo_region = null), - AtomicField(name = "geo_city", _.geo_city, _.geo_city = null), - AtomicField(name = "geo_zipcode", _.geo_zipcode, _.geo_zipcode = null), - AtomicField(name = "geo_region_name", _.geo_region_name, _.geo_region_name = null), - AtomicField(name = "ip_isp", _.ip_isp, _.ip_isp = null), - AtomicField(name = "ip_organization", _.ip_organization, _.ip_organization = null), - AtomicField(name = "ip_domain", _.ip_domain, _.ip_domain = null), - AtomicField(name = "ip_netspeed", _.ip_netspeed, _.ip_netspeed = null), - AtomicField(name = "page_url", _.page_url, _.page_url = null), - AtomicField(name = "page_title", _.page_title, _.page_title = null), - AtomicField(name = "page_referrer", _.page_referrer, _.page_referrer = null), - AtomicField(name = "page_urlscheme", _.page_urlscheme, _.page_urlscheme = null), - AtomicField(name = "page_urlhost", _.page_urlhost, _.page_urlhost = null), - AtomicField(name = "page_urlpath", _.page_urlpath, _.page_urlpath = null), - AtomicField(name = "page_urlquery", _.page_urlquery, _.page_urlquery = null), - AtomicField(name = "page_urlfragment", _.page_urlfragment, _.page_urlfragment = null), - AtomicField(name = "refr_urlscheme", _.refr_urlscheme, _.refr_urlscheme = null), - AtomicField(name = "refr_urlhost", _.refr_urlhost, _.refr_urlhost = null), - AtomicField(name = "refr_urlpath", _.refr_urlpath, _.refr_urlpath = null), - AtomicField(name = "refr_urlquery", _.refr_urlquery, _.refr_urlquery = null), - AtomicField(name = "refr_urlfragment", _.refr_urlfragment, _.refr_urlfragment = null), - AtomicField(name = "refr_medium", _.refr_medium, _.refr_medium = null), - AtomicField(name = "refr_source", _.refr_source, _.refr_source = null), - AtomicField(name = "refr_term", _.refr_term, _.refr_term = null), - AtomicField(name = "mkt_medium", _.mkt_medium, _.mkt_medium = null), - AtomicField(name = "mkt_source", _.mkt_source, _.mkt_source = null), - AtomicField(name = "mkt_term", _.mkt_term, _.mkt_term = null), - AtomicField(name = "mkt_content", _.mkt_content, _.mkt_content = null), - AtomicField(name = "mkt_campaign", _.mkt_campaign, _.mkt_campaign = null), - AtomicField(name = "se_category", _.se_category, _.se_category = null), - AtomicField(name = "se_action", _.se_action, _.se_action = null), - AtomicField(name = "se_label", _.se_label, _.se_label = null), - AtomicField(name = "se_property", _.se_property, _.se_property = null), - AtomicField(name = "tr_orderid", _.tr_orderid, _.tr_orderid = null), - AtomicField(name = "tr_affiliation", _.tr_affiliation, _.tr_affiliation = null), - AtomicField(name = "tr_city", _.tr_city, _.tr_city = null), - AtomicField(name = "tr_state", _.tr_state, _.tr_state = null), - AtomicField(name = "tr_country", _.tr_country, _.tr_country = null), - AtomicField(name = "ti_orderid", _.ti_orderid, _.ti_orderid = null), - AtomicField(name = "ti_sku", _.ti_sku, _.ti_sku = null), - AtomicField(name = "ti_name", _.ti_name, _.ti_name = null), - AtomicField(name = "ti_category", _.ti_category, _.ti_category = null), - AtomicField(name = "useragent", _.useragent, _.useragent = null), - AtomicField(name = "br_name", _.br_name, _.br_name = null), - AtomicField(name = "br_family", _.br_family, _.br_family = null), - AtomicField(name = "br_version", _.br_version, _.br_version = null), - AtomicField(name = "br_type", _.br_type, _.br_type = null), - AtomicField(name = "br_renderengine", _.br_renderengine, _.br_renderengine = null), - AtomicField(name = "br_lang", _.br_lang, _.br_lang = null), - AtomicField(name = "br_colordepth", _.br_colordepth, _.br_colordepth = null), - AtomicField(name = "os_name", _.os_name, _.os_name = null), - AtomicField(name = "os_family", _.os_family, _.os_family = null), - AtomicField(name = "os_manufacturer", _.os_manufacturer, _.os_manufacturer = null), - AtomicField(name = "os_timezone", _.os_timezone, _.os_timezone = null), - AtomicField(name = "dvce_type", _.dvce_type, _.dvce_type = null), - AtomicField(name = "doc_charset", _.doc_charset, _.doc_charset = null), - AtomicField(name = "tr_currency", _.tr_currency, _.tr_currency = null), - AtomicField(name = "ti_currency", _.ti_currency, _.ti_currency = null), - AtomicField(name = "base_currency", _.base_currency, _.base_currency = null), - AtomicField(name = "geo_timezone", _.geo_timezone, _.geo_timezone = null), - AtomicField(name = "mkt_clickid", _.mkt_clickid, _.mkt_clickid = null), - AtomicField(name = "mkt_network", _.mkt_network, _.mkt_network = null), - AtomicField(name = "etl_tags", _.etl_tags, _.etl_tags = null), + AtomicField(name = "app_id", _.app_id , _.app_id = null ), + AtomicField(name = "platform", _.platform , _.platform = null ), + AtomicField(name = "event", _.event , _.event = null ), + AtomicField(name = "event_id", _.event_id , _ => () ), // required in loading + AtomicField(name = "name_tracker", _.name_tracker , _.name_tracker = null ), + AtomicField(name = "v_tracker", _.v_tracker , _.v_tracker = null ), + AtomicField(name = "v_collector", _.v_collector , _ => () ), // required in loading + AtomicField(name = "v_etl", _.v_etl , _ => () ), // required in loading + AtomicField(name = "user_id", _.user_id , _.user_id = null ), + AtomicField(name = "user_ipaddress", _.user_ipaddress , _.user_ipaddress = null ), + AtomicField(name = "user_fingerprint", _.user_fingerprint , _.user_fingerprint = null ), + AtomicField(name = "domain_userid", _.domain_userid , _.domain_userid = null ), + AtomicField(name = "network_userid", _.network_userid , _.network_userid = null ), + AtomicField(name = "geo_country", _.geo_country , _.geo_country = null ), + AtomicField(name = "geo_region", _.geo_region , _.geo_region = null ), + AtomicField(name = "geo_city", _.geo_city , _.geo_city = null ), + AtomicField(name = "geo_zipcode", _.geo_zipcode , _.geo_zipcode = null ), + AtomicField(name = "geo_region_name", _.geo_region_name , _.geo_region_name = null ), + AtomicField(name = "ip_isp", _.ip_isp , _.ip_isp = null ), + AtomicField(name = "ip_organization", _.ip_organization , _.ip_organization = null ), + AtomicField(name = "ip_domain", _.ip_domain , _.ip_domain = null ), + AtomicField(name = "ip_netspeed", _.ip_netspeed , _.ip_netspeed = null ), + AtomicField(name = "page_url", _.page_url , _.page_url = null ), + AtomicField(name = "page_title", _.page_title , _.page_title = null ), + AtomicField(name = "page_referrer", _.page_referrer , _.page_referrer = null ), + AtomicField(name = "page_urlscheme", _.page_urlscheme , _.page_urlscheme = null ), + AtomicField(name = "page_urlhost", _.page_urlhost , _.page_urlhost = null ), + AtomicField(name = "page_urlpath", _.page_urlpath , _.page_urlpath = null ), + AtomicField(name = "page_urlquery", _.page_urlquery , _.page_urlquery = null ), + AtomicField(name = "page_urlfragment", _.page_urlfragment , _.page_urlfragment = null ), + AtomicField(name = "refr_urlscheme", _.refr_urlscheme , _.refr_urlscheme = null ), + AtomicField(name = "refr_urlhost", _.refr_urlhost , _.refr_urlhost = null ), + AtomicField(name = "refr_urlpath", _.refr_urlpath , _.refr_urlpath = null ), + AtomicField(name = "refr_urlquery", _.refr_urlquery , _.refr_urlquery = null ), + AtomicField(name = "refr_urlfragment", _.refr_urlfragment , _.refr_urlfragment = null ), + AtomicField(name = "refr_medium", _.refr_medium , _.refr_medium = null ), + AtomicField(name = "refr_source", _.refr_source , _.refr_source = null ), + AtomicField(name = "refr_term", _.refr_term , _.refr_term = null ), + AtomicField(name = "mkt_medium", _.mkt_medium , _.mkt_medium = null ), + AtomicField(name = "mkt_source", _.mkt_source , _.mkt_source = null ), + AtomicField(name = "mkt_term", _.mkt_term , _.mkt_term = null ), + AtomicField(name = "mkt_content", _.mkt_content , _.mkt_content = null ), + AtomicField(name = "mkt_campaign", _.mkt_campaign , _.mkt_campaign = null ), + AtomicField(name = "se_category", _.se_category , _.se_category = null ), + AtomicField(name = "se_action", _.se_action , _.se_action = null ), + AtomicField(name = "se_label", _.se_label , _.se_label = null ), + AtomicField(name = "se_property", _.se_property , _.se_property = null ), + AtomicField(name = "tr_orderid", _.tr_orderid , _.tr_orderid = null ), + AtomicField(name = "tr_affiliation", _.tr_affiliation , _.tr_affiliation = null ), + AtomicField(name = "tr_city", _.tr_city , _.tr_city = null ), + AtomicField(name = "tr_state", _.tr_state , _.tr_state = null ), + AtomicField(name = "tr_country", _.tr_country , _.tr_country = null ), + AtomicField(name = "ti_orderid", _.ti_orderid , _.ti_orderid = null ), + AtomicField(name = "ti_sku", _.ti_sku , _.ti_sku = null ), + AtomicField(name = "ti_name", _.ti_name , _.ti_name = null ), + AtomicField(name = "ti_category", _.ti_category , _.ti_category = null ), + AtomicField(name = "useragent", _.useragent , _.useragent = null ), + AtomicField(name = "br_name", _.br_name , _.br_name = null ), + AtomicField(name = "br_family", _.br_family , _.br_family = null ), + AtomicField(name = "br_version", _.br_version , _.br_version = null ), + AtomicField(name = "br_type", _.br_type , _.br_type = null ), + AtomicField(name = "br_renderengine", _.br_renderengine , _.br_renderengine = null ), + AtomicField(name = "br_lang", _.br_lang , _.br_lang = null ), + AtomicField(name = "br_colordepth", _.br_colordepth , _.br_colordepth = null ), + AtomicField(name = "os_name", _.os_name , _.os_name = null ), + AtomicField(name = "os_family", _.os_family , _.os_family = null ), + AtomicField(name = "os_manufacturer", _.os_manufacturer , _.os_manufacturer = null ), + AtomicField(name = "os_timezone", _.os_timezone , _.os_timezone = null ), + AtomicField(name = "dvce_type", _.dvce_type , _.dvce_type = null ), + AtomicField(name = "doc_charset", _.doc_charset , _.doc_charset = null ), + AtomicField(name = "tr_currency", _.tr_currency , _.tr_currency = null ), + AtomicField(name = "ti_currency", _.ti_currency , _.ti_currency = null ), + AtomicField(name = "base_currency", _.base_currency , _.base_currency = null ), + AtomicField(name = "geo_timezone", _.geo_timezone , _.geo_timezone = null ), + AtomicField(name = "mkt_clickid", _.mkt_clickid , _.mkt_clickid = null ), + AtomicField(name = "mkt_network", _.mkt_network , _.mkt_network = null ), + AtomicField(name = "etl_tags", _.etl_tags , _.etl_tags = null ), AtomicField(name = "refr_domain_userid", _.refr_domain_userid, _.refr_domain_userid = null), - AtomicField(name = "domain_sessionid", _.domain_sessionid, _.domain_sessionid = null), - AtomicField(name = "event_vendor", _.event_vendor, _.event_vendor = null), - AtomicField(name = "event_name", _.event_name, _.event_name = null), - AtomicField(name = "event_format", _.event_format, _.event_format = null), - AtomicField(name = "event_version", _.event_version, _.event_version = null), - AtomicField(name = "event_fingerprint", _.event_fingerprint, _.event_fingerprint = null), + AtomicField(name = "domain_sessionid", _.domain_sessionid , _.domain_sessionid = null ), + AtomicField(name = "event_vendor", _.event_vendor , _.event_vendor = null ), + AtomicField(name = "event_name", _.event_name , _.event_name = null ), + AtomicField(name = "event_format", _.event_format , _.event_format = null ), + AtomicField(name = "event_version", _.event_version , _.event_version = null ), + AtomicField(name = "event_fingerprint", _.event_fingerprint , _.event_fingerprint = null ) ) // format: on diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/EnrichmentManager.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/EnrichmentManager.scala index b1c006d7a..1bacdc4e3 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/EnrichmentManager.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/EnrichmentManager.scala @@ -119,15 +119,14 @@ object EnrichmentManager { for { enrichedEvent <- Sync[F].delay(new EnrichedEvent) enrichmentResult <- enrich(enrichedEvent).value - now = Instant.now() - _ = setDerivedContexts(enrichedEvent, enrichmentResult, now, processor) + _ = setDerivedContexts(enrichedEvent, enrichmentResult, processor) result = enrichmentResult .leftMap { fe => createBadRow( fe, EnrichedEvent.toPartiallyEnrichedEvent(enrichedEvent), RawEvent.toRawEvent(raw), - now, + Instant.now(), processor ) } @@ -165,12 +164,11 @@ object EnrichmentManager { def setDerivedContexts( enriched: EnrichedEvent, enrichmentResult: Ior[NonEmptyList[NonEmptyList[Failure]], List[SelfDescribingData[Json]]], - timestamp: Instant, processor: Processor ): Unit = { val derivedContexts = enrichmentResult.leftMap { ll => ll.flatten.toList - .map(_.toSDJ(timestamp, processor)) + .map(_.toSDJ(processor)) }.merge ME.formatContexts(derivedContexts).foreach(c => enriched.derived_contexts = c) } @@ -213,7 +211,7 @@ object EnrichmentManager { failures.toNel match { case Some(nel) => Ior.both( - nel.map(Failure.EnrichmentFailure), + nel.map(f => Failure.EnrichmentFailure(f)), contexts ) case None => diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/Failure.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/Failure.scala index 3991528cb..3e0aa44cb 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/Failure.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/Failure.scala @@ -23,7 +23,7 @@ import com.snowplowanalytics.snowplow.badrows._ import com.snowplowanalytics.iglu.client.ClientError import com.snowplowanalytics.iglu.client.validator.ValidatorError -import com.snowplowanalytics.iglu.core.{SchemaKey, SchemaVer, SelfDescribingData} +import com.snowplowanalytics.iglu.core.{ParseError, SchemaKey, SchemaVer, SelfDescribingData} import com.snowplowanalytics.iglu.core.circe.implicits.schemaKeyCirceJsonEncoder /** @@ -31,7 +31,8 @@ import com.snowplowanalytics.iglu.core.circe.implicits.schemaKeyCirceJsonEncoder * Failure entities will be attached to incomplete events as derived contexts. */ sealed trait Failure { - def toSDJ(timestamp: Instant, processor: Processor): SelfDescribingData[Json] + def timestamp: Instant + def toSDJ(processor: Processor): SelfDescribingData[Json] } object Failure { @@ -41,9 +42,10 @@ object Failure { case class SchemaViolation( schemaViolation: FailureDetails.SchemaViolation, source: String, - data: Json + data: Json, + timestamp: Instant = Instant.now() ) extends Failure { - def toSDJ(timestamp: Instant, processor: Processor): SelfDescribingData[Json] = { + def toSDJ(processor: Processor): SelfDescribingData[Json] = { val feJson = fromSchemaViolation(this, timestamp, processor) SelfDescribingData(failureSchemaKey, feJson.asJson) } @@ -51,9 +53,10 @@ object Failure { } case class EnrichmentFailure( - enrichmentFailure: FailureDetails.EnrichmentFailure + enrichmentFailure: FailureDetails.EnrichmentFailure, + timestamp: Instant = Instant.now() ) extends Failure { - def toSDJ(timestamp: Instant, processor: Processor): SelfDescribingData[Json] = { + def toSDJ(processor: Processor): SelfDescribingData[Json] = { val feJson = fromEnrichmentFailure(this, timestamp, processor) SelfDescribingData(failureSchemaKey, feJson.asJson) } @@ -133,7 +136,13 @@ object Failure { val error = Json.obj("message" := err, "source" := v.source) ("NotJSON", List(error), None, Json.obj(v.source := v.data).some) case FailureDetails.SchemaViolation.NotIglu(_, err) => - val message = err.message("").split(":").headOption + val message = err match { + case ParseError.InvalidSchemaVer => "Invalid schema version" + case ParseError.InvalidIgluUri => "Invalid Iglu URI" + case ParseError.InvalidData => "Invalid data payload" + case ParseError.InvalidSchema => "Invalid schema" + case ParseError.InvalidMetaschema => "Invalid metashema" + } val error = Json.obj("message" := message, "source" := v.source) ("NotIglu", List(error), None, v.data.some) case FailureDetails.SchemaViolation.CriterionMismatch(schemaKey, schemaCriterion) => diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/AtomicFieldsSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/AtomicFieldsSpec.scala index 1bfb44d14..8117eaa12 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/AtomicFieldsSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/AtomicFieldsSpec.scala @@ -42,21 +42,23 @@ class AtomicFieldsSpec extends Specification { ValidatorReport(error4.message, error4.field.some, Nil, AtomicError.keywordParse.some) ) ) - val expected = Failure.SchemaViolation( - schemaViolation = FailureDetails.SchemaViolation.IgluError( - schemaKey = AtomicFields.atomicSchema, - error = ValidationError(ValidatorError.InvalidData(vrList), None) - ), - source = AtomicError.source, - data = Json.obj( - error1.field := error1.value, - error2.field := error2.value, - error3.field := error3.value, - error4.field := Json.Null - ) + + val schemaViolation = FailureDetails.SchemaViolation.IgluError( + schemaKey = AtomicFields.atomicSchema, + error = ValidationError(ValidatorError.InvalidData(vrList), None) ) + val source = AtomicError.source + val data = Json.obj( + error1.field := error1.value, + error2.field := error2.value, + error3.field := error3.value, + error4.field := Json.Null + ) + val result = AtomicFields.errorsToSchemaViolation(NonEmptyList(error1, List(error2, error3, error4))) - result must beEqualTo(expected) + result.schemaViolation must beEqualTo(schemaViolation) + result.source must beEqualTo(source) + result.data must beEqualTo(data) } } } diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/EnrichmentManagerSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/EnrichmentManagerSpec.scala index fa1f350db..fbc74f052 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/EnrichmentManagerSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/EnrichmentManagerSpec.scala @@ -10,8 +10,6 @@ */ package com.snowplowanalytics.snowplow.enrich.common.enrichments -import java.time.Instant - import org.apache.commons.codec.digest.DigestUtils import org.specs2.mutable.Specification @@ -2463,11 +2461,10 @@ class EnrichmentManagerSpec extends Specification with EitherMatchers with CatsE ) ) val emailSentSDJ = SelfDescribingData.parse[Json](jparse(emailSent).toOption.get).toOption.get - val timestamp = Instant.now() "set derived contexts correctly if enrichment result is Ior.Left" >> { val enriched = new EnrichedEvent() val enrichmentResult = Ior.Left(NonEmptyList.of(NonEmptyList.of(sv, ef), NonEmptyList.of(sv, ef))) - EnrichmentManager.setDerivedContexts(enriched, enrichmentResult, timestamp, processor) + EnrichmentManager.setDerivedContexts(enriched, enrichmentResult, processor) val schemas = SpecHelpers.listContextsSchemas(enriched.derived_contexts) schemas.size must beEqualTo(4) forall(schemas)(s => s must beEqualTo(Failure.failureSchemaKey)) @@ -2475,7 +2472,7 @@ class EnrichmentManagerSpec extends Specification with EitherMatchers with CatsE "set derived contexts correctly if enrichment result is Ior.Right" >> { val enriched = new EnrichedEvent() val enrichmentResult = Ior.Right(List(emailSentSDJ, emailSentSDJ)) - EnrichmentManager.setDerivedContexts(enriched, enrichmentResult, timestamp, processor) + EnrichmentManager.setDerivedContexts(enriched, enrichmentResult, processor) val schemas = SpecHelpers.listContextsSchemas(enriched.derived_contexts) schemas.size must beEqualTo(2) forall(schemas)(s => s must beEqualTo(emailSentSchema)) @@ -2486,7 +2483,7 @@ class EnrichmentManagerSpec extends Specification with EitherMatchers with CatsE NonEmptyList.of(NonEmptyList.of(sv, ef), NonEmptyList.of(sv, ef)), List(emailSentSDJ, emailSentSDJ) ) - EnrichmentManager.setDerivedContexts(enriched, enrichmentResult, timestamp, processor) + EnrichmentManager.setDerivedContexts(enriched, enrichmentResult, processor) val schemas = SpecHelpers.listContextsSchemas(enriched.derived_contexts) schemas.size must beEqualTo(6) schemas.count(_ == Failure.failureSchemaKey) must beEqualTo(4) diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/IgluUtilsSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/IgluUtilsSpec.scala index d9d8736ae..1f8f492a6 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/IgluUtilsSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/IgluUtilsSpec.scala @@ -163,7 +163,7 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect .map { case Ior.Both( NonEmptyList( - Failure.SchemaViolation(_: FailureDetails.SchemaViolation.NotJson, `unstructFieldName`, `jsonNotJson`), + Failure.SchemaViolation(_: FailureDetails.SchemaViolation.NotJson, `unstructFieldName`, `jsonNotJson`, _), _ ), None @@ -183,7 +183,7 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect .value .map { case Ior.Both( - NonEmptyList(Failure.SchemaViolation(_: FailureDetails.SchemaViolation.NotIglu, `unstructFieldName`, `json`), _), + NonEmptyList(Failure.SchemaViolation(_: FailureDetails.SchemaViolation.NotIglu, `unstructFieldName`, `json`, _), _), None ) => ok @@ -202,7 +202,7 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect .map { case Ior.Both( NonEmptyList( - Failure.SchemaViolation(_: FailureDetails.SchemaViolation.CriterionMismatch, `unstructFieldName`, `json`), + Failure.SchemaViolation(_: FailureDetails.SchemaViolation.CriterionMismatch, `unstructFieldName`, `json`, _), _ ), None @@ -223,7 +223,7 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect .value .map { case Ior.Both(NonEmptyList( - Failure.SchemaViolation(_: FailureDetails.SchemaViolation.NotJson, `unstructFieldName`, `ueJson`), + Failure.SchemaViolation(_: FailureDetails.SchemaViolation.NotJson, `unstructFieldName`, `ueJson`, _), _ ), None @@ -245,7 +245,8 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect .map { case Ior.Both(NonEmptyList(Failure.SchemaViolation(FailureDetails.SchemaViolation.IgluError(_, _: ValidationError), `unstructFieldName`, - `ueJson` + `ueJson`, + _ ), _ ), @@ -265,13 +266,12 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect .extractAndValidateUnstructEvent(input, SpecHelpers.client, SpecHelpers.registryLookup) .value .map { - case Ior.Both(NonEmptyList(Failure.SchemaViolation(FailureDetails.SchemaViolation.IgluError(_, _: ValidationError), - `unstructFieldName`, - `json` - ), - _ - ), - None + case Ior.Both( + NonEmptyList( + Failure.SchemaViolation(FailureDetails.SchemaViolation.IgluError(_, _: ValidationError), `unstructFieldName`, `json`, _), + _ + ), + None ) => ok case other => ko(s"[$other] is not expected one") @@ -289,7 +289,7 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect .map { case Ior.Both( NonEmptyList( - Failure.SchemaViolation(FailureDetails.SchemaViolation.IgluError(_, _: ResolutionError), `unstructFieldName`, `json`), + Failure.SchemaViolation(FailureDetails.SchemaViolation.IgluError(_, _: ResolutionError), `unstructFieldName`, `json`, _), _ ), None @@ -376,7 +376,9 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect .value .map { case Ior.Both( - NonEmptyList(Failure.SchemaViolation(_: FailureDetails.SchemaViolation.NotJson, `contextsFieldName`, `jsonNotJson`), Nil), + NonEmptyList(Failure.SchemaViolation(_: FailureDetails.SchemaViolation.NotJson, `contextsFieldName`, `jsonNotJson`, _), + Nil + ), Nil ) => ok @@ -394,7 +396,7 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect .value .map { case Ior.Both( - NonEmptyList(Failure.SchemaViolation(_: FailureDetails.SchemaViolation.NotIglu, `contextsFieldName`, `json`), Nil), + NonEmptyList(Failure.SchemaViolation(_: FailureDetails.SchemaViolation.NotIglu, `contextsFieldName`, `json`, _), Nil), Nil ) => ok @@ -412,7 +414,7 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect .value .map { case Ior.Both(NonEmptyList( - Failure.SchemaViolation(_: FailureDetails.SchemaViolation.CriterionMismatch, `contextsFieldName`, `json`), + Failure.SchemaViolation(_: FailureDetails.SchemaViolation.CriterionMismatch, `contextsFieldName`, `json`, _), Nil ), Nil @@ -435,7 +437,7 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect .map { case Ior.Both( NonEmptyList( - Failure.SchemaViolation(FailureDetails.SchemaViolation.IgluError(_, _: ValidationError), `contextsFieldName`, `json`), + Failure.SchemaViolation(FailureDetails.SchemaViolation.IgluError(_, _: ValidationError), `contextsFieldName`, `json`, _), Nil ), Nil @@ -456,7 +458,7 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect .map { case Ior.Both( NonEmptyList( - Failure.SchemaViolation(FailureDetails.SchemaViolation.IgluError(_, _: ValidationError), `contextsFieldName`, `json`), + Failure.SchemaViolation(FailureDetails.SchemaViolation.IgluError(_, _: ValidationError), `contextsFieldName`, `json`, _), Nil ), Nil @@ -477,7 +479,7 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect .map { case Ior.Both( NonEmptyList( - Failure.SchemaViolation(FailureDetails.SchemaViolation.IgluError(_, _: ResolutionError), `contextsFieldName`, `json`), + Failure.SchemaViolation(FailureDetails.SchemaViolation.IgluError(_, _: ResolutionError), `contextsFieldName`, `json`, _), Nil ), Nil @@ -500,12 +502,14 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect case Ior.Both(NonEmptyList( Failure.SchemaViolation(FailureDetails.SchemaViolation.IgluError(_, _: ValidationError), `contextsFieldName`, - `invalidEmailSentJson` + `invalidEmailSentJson`, + _ ), List( Failure.SchemaViolation(FailureDetails.SchemaViolation.IgluError(_, _: ResolutionError), `contextsFieldName`, - `noSchemaJson` + `noSchemaJson`, + _ ) ) ), @@ -526,7 +530,7 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect .value .map { case Ior.Both(NonEmptyList( - Failure.SchemaViolation(_: FailureDetails.SchemaViolation.IgluError, `contextsFieldName`, `noSchemaJson`), + Failure.SchemaViolation(_: FailureDetails.SchemaViolation.IgluError, `contextsFieldName`, `noSchemaJson`, _), Nil ), List(extract) @@ -596,7 +600,8 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect case Ior.Both( NonEmptyList(Failure.SchemaViolation(FailureDetails.SchemaViolation.IgluError(_, _: ValidationError), `derivedContextsFieldName`, - `json` + `json`, + _ ), Nil ), @@ -622,12 +627,14 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect case Ior.Both(NonEmptyList( Failure.SchemaViolation(FailureDetails.SchemaViolation.IgluError(_, _: ValidationError), `derivedContextsFieldName`, - `invalidEmailSentJson` + `invalidEmailSentJson`, + _ ), List( Failure.SchemaViolation(FailureDetails.SchemaViolation.IgluError(_, _: ResolutionError), `derivedContextsFieldName`, - `noSchemaJson` + `noSchemaJson`, + _ ) ) ), @@ -652,7 +659,8 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect case Ior.Both(NonEmptyList( Failure.SchemaViolation(FailureDetails.SchemaViolation.IgluError(_, _: ValidationError), `derivedContextsFieldName`, - `invalidEmailSentJson` + `invalidEmailSentJson`, + _ ), Nil ), @@ -795,7 +803,8 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect case Ior.Both( NonEmptyList(Failure.SchemaViolation(FailureDetails.SchemaViolation.IgluError(_, _: ValidationError), `contextsFieldName`, - `invalidEmailSentJson` + `invalidEmailSentJson`, + _ ), _ ), @@ -826,7 +835,8 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect case Ior.Both( NonEmptyList(Failure.SchemaViolation(FailureDetails.SchemaViolation.IgluError(_, _: ValidationError), `unstructFieldName`, - `invalidEmailSentJson` + `invalidEmailSentJson`, + _ ), _ ),