Set data field of incomplete event failure entity to SDJ's data field…

… only At the moment, we set all self describing json to data field of incomplete event failure entity. However, failure entity has separate schema field and therefore we don't need to set all SDJ to data field. It is enough to use only data field of SDJ.
snowplow · Apr 19, 2024 · f5d611a · f5d611a
1 parent ebe793f
commit f5d611a
Show file tree

Hide file tree

Showing 3 changed files with 62 additions and 50 deletions.
diff --git a/.../common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/IgluUtils.scala b/.../common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/IgluUtils.scala
@@ -156,7 +156,7 @@ object IgluUtils {
             Failure.SchemaViolation(
               schemaViolation = FailureDetails.SchemaViolation.IgluError(sdj.schema, clientError),
               source = "derived_contexts",
-              data = sdj.asJson
+              data = sdj.data
             )
         }
       )
@@ -201,7 +201,7 @@ object IgluUtils {
                  Failure.SchemaViolation(
                    schemaViolation = FailureDetails.SchemaViolation.CriterionMismatch(sdj.schema, expectedCriterion),
                    source = field,
-                   data = sdj.asJson
+                   data = sdj.data
                  )
                )
       // Check that the SDJ holding the .data is valid
@@ -211,7 +211,7 @@ object IgluUtils {
                  Failure.SchemaViolation(
                    schemaViolation = FailureDetails.SchemaViolation.IgluError(schemaKey, clientError),
                    source = field,
-                   data = sdj.asJson
+                   data = sdj.data
                  )
              }
       // Extract .data of SelfDescribingData[Json]
@@ -276,7 +276,7 @@ object IgluUtils {
                                    schemaViolation = FailureDetails.SchemaViolation
                                      .IgluError(schemaKey, clientError): FailureDetails.SchemaViolation,
                                    source = field,
-                                   data = json.asJson
+                                   data = sdj.data
                                  )
                              }
                              .leftMap(NonEmptyList.one)

diff --git a/...cala/com.snowplowanalytics.snowplow.enrich.common/enrichments/EnrichmentManagerSpec.scala b/...cala/com.snowplowanalytics.snowplow.enrich.common/enrichments/EnrichmentManagerSpec.scala
@@ -1241,14 +1241,16 @@ class EnrichmentManagerSpec extends Specification with EitherMatchers with CatsE
           JavascriptScriptEnrichment(schemaKey, script)
         )
       )
-      val invalidUe =
+      val invalidUeData =
         """{
+             "emailAddress": "[email protected]",
+             "emailAddress2": "[email protected]",
+             "unallowedAdditionalField": "[email protected]"
+           }"""
+      val invalidUe =
+        s"""{
              "schema":"iglu:com.acme/email_sent/jsonschema/1-0-0",
-             "data": {
-               "emailAddress": "[email protected]",
-               "emailAddress2": "[email protected]",
-               "unallowedAdditionalField": "[email protected]"
-             }
+             "data": $invalidUeData
            }"""
 
       val parameters = Map(
@@ -1287,7 +1289,7 @@ class EnrichmentManagerSpec extends Specification with EitherMatchers with CatsE
                   )
                 ) &&
                 feJson.field("schema") == emailSentSchema.asJson &&
-                feJson.field("data") == jparse(invalidUe).toOption.get =>
+                feJson.field("data") == jparse(invalidUeData).toOption.get =>
             true
           case _ => false
         }
@@ -1333,13 +1335,15 @@ class EnrichmentManagerSpec extends Specification with EitherMatchers with CatsE
           JavascriptScriptEnrichment(schemaKey, script)
         )
       )
-      val invalidContext =
+      val invalidContextData =
         """{
+            "foo": "[email protected]",
+            "emailAddress2": "[email protected]"
+          }"""
+      val invalidContext =
+        s"""{
             "schema":"iglu:com.acme/email_sent/jsonschema/1-0-0",
-            "data": {
-              "foo": "[email protected]",
-              "emailAddress2": "[email protected]"
-            }
+            "data": $invalidContextData
           }"""
 
       val parameters = Map(
@@ -1383,7 +1387,7 @@ class EnrichmentManagerSpec extends Specification with EitherMatchers with CatsE
                   )
                 ) &&
                 feJson.field("schema") == emailSentSchema.asJson &&
-                feJson.field("data") == jparse(invalidContext).toOption.get =>
+                feJson.field("data") == jparse(invalidContextData).toOption.get =>
             true
           case _ => false
         }
@@ -1428,14 +1432,17 @@ class EnrichmentManagerSpec extends Specification with EitherMatchers with CatsE
           JavascriptScriptEnrichment(schemaKey, script)
         )
       )
-      val invalidContext =
+      val invalidContextData =
         """
+          {
+            "foo": "[email protected]",
+            "emailAddress2": "[email protected]"
+          }"""
+      val invalidContext =
+        s"""
           {
             "schema":"iglu:com.acme/email_sent/jsonschema/1-0-0",
-            "data": {
-              "foo": "[email protected]",
-              "emailAddress2": "[email protected]"
-            }
+            "data": $invalidContextData
           }"""
 
       val parameters = Map(
@@ -1482,7 +1489,7 @@ class EnrichmentManagerSpec extends Specification with EitherMatchers with CatsE
                   )
                 ) &&
                 feJson.field("schema") == emailSentSchema.asJson &&
-                feJson.field("data") == jparse(invalidContext).toOption.get =>
+                feJson.field("data") == jparse(invalidContextData).toOption.get =>
             true
           case _ => false
         }
@@ -1662,14 +1669,17 @@ class EnrichmentManagerSpec extends Specification with EitherMatchers with CatsE
     }
 
     "remove an invalid enrichment context and return the enriched event if emitIncomplete is set to true" >> {
-      val invalidContext =
+      val invalidContextData =
         """
+          {
+            "foo": "[email protected]",
+            "emailAddress2": "[email protected]"
+          }"""
+      val invalidContext =
+        s"""
           {
             "schema":"iglu:com.acme/email_sent/jsonschema/1-0-0",
-            "data": {
-              "foo": "[email protected]",
-              "emailAddress2": "[email protected]"
-            }
+            "data": $invalidContextData
           }"""
       val script =
         s"""
@@ -1727,7 +1737,7 @@ class EnrichmentManagerSpec extends Specification with EitherMatchers with CatsE
                   )
                 ) &&
                 feJson.field("schema") == emailSentSchema.asJson &&
-                feJson.field("data") == jparse(invalidContext).toOption.get =>
+                feJson.field("data") == jparse(invalidContextData).toOption.get =>
             true
           case _ => false
         }

diff --git a/...mon/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/IgluUtilsSpec.scala b/...mon/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/IgluUtilsSpec.scala
@@ -103,11 +103,12 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect
       "emailAddress2": "[email protected]"
     }
   }"""
+  val invalidEmailSentData = s"""{
+      "emailAddress": "[email protected]"
+  }"""
   val invalidEmailSent = s"""{
     "schema": "${emailSentSchema.toSchemaUri}",
-    "data": {
-      "emailAddress": "[email protected]"
-    }
+    "data": $invalidEmailSentData
   }"""
   val supersedingExample1 =
     s"""{
@@ -137,8 +138,9 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect
       "userId": "20d631b8-7837-49df-a73e-6da73154e6fd"
     }
   }"""
+  val noSchemaData = "{}"
   val noSchema =
-    """{"schema":"iglu:com.snowplowanalytics.snowplow/foo/jsonschema/1-0-0", "data": {}}"""
+    s"""{"schema":"iglu:com.snowplowanalytics.snowplow/foo/jsonschema/1-0-0", "data": $noSchemaData}"""
 
   "extractAndValidateUnstructEvent" should {
     "return None if unstruct_event field is empty" >> {
@@ -190,7 +192,7 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect
     }
 
     "return a FailureDetails.SchemaViolation.CriterionMismatch if unstruct_event contains a self-describing JSON but not with the expected schema for unstructured events" >> {
-      val json = noSchema.toJson
+      val json = noSchemaData.toJson
       val input = new EnrichedEvent
       input.setUnstruct_event(noSchema)
 
@@ -234,7 +236,7 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect
     "return a FailureDetails.SchemaViolation.IgluError containing a ValidationError if the JSON in .data is not self-describing" >> {
       val input = new EnrichedEvent
       val ue = buildUnstruct(notIglu)
-      val ueJson = ue.toJson
+      val ueJson = notIglu.toJson
       input.setUnstruct_event(ue)
 
       IgluUtils
@@ -256,7 +258,7 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect
 
     "return a FailureDetails.SchemaViolation.IgluError containing a ValidationError if the JSON in .data is not a valid SDJ" >> {
       val input = new EnrichedEvent
-      val json = invalidEmailSent.toJson
+      val json = invalidEmailSentData.toJson
       input.setUnstruct_event(buildUnstruct(invalidEmailSent))
 
       IgluUtils
@@ -278,7 +280,7 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect
 
     "return a FailureDetails.SchemaViolation.IgluError containing a ResolutionError if the schema of the SDJ in .data can't be resolved" >> {
       val input = new EnrichedEvent
-      val json = noSchema.toJson
+      val json = noSchemaData.toJson
       input.setUnstruct_event(buildUnstruct(noSchema))
 
       IgluUtils
@@ -402,7 +404,7 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect
 
     "return a FailureDetails.SchemaViolation.CriterionMismatch if .contexts contains a self-describing JSON but not with the right schema" >> {
       val input = new EnrichedEvent
-      val json = noSchema.toJson
+      val json = noSchemaData.toJson
       input.setContexts(noSchema)
 
       IgluUtils
@@ -424,7 +426,7 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect
       val input = new EnrichedEvent
       val notArrayContexts =
         s"""{"schema": "${inputContextsSchema.toSchemaUri}", "data": ${emailSent1}}"""
-      val json = notArrayContexts.toJson
+      val json = emailSent1.toJson
       input.setContexts(notArrayContexts)
 
       IgluUtils
@@ -445,7 +447,7 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect
 
     "return a FailureDetails.SchemaViolation.IgluError containing a ValidationError if .data contains one invalid context" >> {
       val input = new EnrichedEvent
-      val json = invalidEmailSent.toJson
+      val json = invalidEmailSentData.toJson
       input.setContexts(buildInputContexts(List(invalidEmailSent)))
 
       IgluUtils
@@ -466,7 +468,7 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect
 
     "return a FailureDetails.SchemaViolation.IgluError containing a ResolutionError if .data contains one context whose schema can't be resolved" >> {
       val input = new EnrichedEvent
-      val json = noSchema.toJson
+      val json = noSchemaData.toJson
       input.setContexts(buildInputContexts(List(noSchema)))
 
       IgluUtils
@@ -487,8 +489,8 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect
 
     "return 2 expected failures for 2 invalid contexts" >> {
       val input = new EnrichedEvent
-      val invalidEmailSentJson = invalidEmailSent.toJson
-      val noSchemaJson = noSchema.toJson
+      val invalidEmailSentJson = invalidEmailSentData.toJson
+      val noSchemaJson = noSchemaData.toJson
       input.setContexts(buildInputContexts(List(invalidEmailSent, noSchema)))
 
       IgluUtils
@@ -516,7 +518,7 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect
 
     "return an expected failure and an expected SDJ if one context is invalid and one is valid" >> {
       val input = new EnrichedEvent
-      val noSchemaJson = noSchema.toJson
+      val noSchemaJson = noSchemaData.toJson
       input.setContexts(buildInputContexts(List(emailSent1, noSchema)))
 
       IgluUtils
@@ -582,7 +584,7 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect
 
   "validateEnrichmentsContexts" should {
     "return one expected SchemaViolation for one invalid context" >> {
-      val json = invalidEmailSent.toJson
+      val json = invalidEmailSentData.toJson
       val contexts = List(
         SpecHelpers.jsonStringToSDJ(invalidEmailSent).right.get
       )
@@ -606,8 +608,8 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect
     }
 
     "return two expected SchemaViolation for two invalid contexts" >> {
-      val invalidEmailSentJson = invalidEmailSent.toJson
-      val noSchemaJson = noSchema.toJson
+      val invalidEmailSentJson = invalidEmailSentData.toJson
+      val noSchemaJson = noSchemaData.toJson
       val contexts = List(
         SpecHelpers.jsonStringToSDJ(invalidEmailSent).right.get,
         SpecHelpers.jsonStringToSDJ(noSchema).right.get
@@ -637,7 +639,7 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect
     }
 
     "return one expected SchemaViolation for one invalid context and one valid" >> {
-      val invalidEmailSentJson = invalidEmailSent.toJson
+      val invalidEmailSentJson = invalidEmailSentData.toJson
       val contexts = List(
         SpecHelpers.jsonStringToSDJ(invalidEmailSent).right.get,
         SpecHelpers.jsonStringToSDJ(emailSent1).right.get
@@ -778,7 +780,7 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect
 
     "return the SchemaViolation of the invalid context in the Left and the extracted unstructured event in the Right" >> {
       val input = new EnrichedEvent
-      val invalidEmailSentJson = invalidEmailSent.toJson
+      val invalidEmailSentJson = invalidEmailSentData.toJson
       input.setUnstruct_event(buildUnstruct(emailSent1))
       input.setContexts(buildInputContexts(List(invalidEmailSent)))
 
@@ -809,7 +811,7 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect
 
     "return the SchemaViolation of the invalid unstructured event in the Left and the valid context in the Right" >> {
       val input = new EnrichedEvent
-      val invalidEmailSentJson = invalidEmailSent.toJson
+      val invalidEmailSentJson = invalidEmailSentData.toJson
       input.setUnstruct_event(buildUnstruct(invalidEmailSent))
       input.setContexts(buildInputContexts(List(emailSent1)))