#2172 - update hermes json files (#2173)

* #2172 - update hermes json files - Updated json files. - Removed spline configuration properties.
AbsaOSS · Mar 22, 2023 · d2aa209 · d2aa209
1 parent 98b15a2
commit d2aa209
Show file tree

Hide file tree

Showing 12 changed files with 44 additions and 44 deletions.
diff --git a/examples/data/e2e_tests/test_jsons/castingConformanceRuleTest.json b/examples/data/e2e_tests/test_jsons/castingConformanceRuleTest.json
@@ -2,9 +2,9 @@
 {
     "vars": {
       "spark-submit": "spark-submit --num-executors 2 --executor-memory 2G --deploy-mode client",
-      "spark-conf": "--conf 'spark.driver.extraJavaOptions=-Denceladus.rest.uri=http://localhost:8080/rest_api/api -Dspline.mongodb.name=spline -Dspline.mongodb.url=mongodb://127.0.0.1:27017/ -Denceladus.recordId.generation.strategy=stableHashId'",
-      "enceladus-job-jar": "spark-jobs/target/spark-jobs-2.19.0-SNAPSHOT.jar",
-      "credentials": "--rest-api-credentials-file ~/.ssh/menasCredential.properties",
+      "spark-conf": "--conf 'spark.driver.extraJavaOptions=-Denceladus.rest.uri=http://localhost:8080/rest_api/api -Denceladus.recordId.generation.strategy=stableHashId'",
+      "enceladus-job-jar": "spark-jobs/target/spark-jobs-3.0.0-SNAPSHOT.jar",
+      "credentials": "--rest-api-credentials-file ~/.ssh/menas-credential.properties",
       "ref-std-data-path": "/ref/castingConformanceRule/std",
       "new-std-data-path": "/tmp/conformance-output/standardized-castingConformanceRule-1-2020-03-23-1",
       "ref-publish-data-path": "/ref/castingConformanceRule/publish",

diff --git a/examples/data/e2e_tests/test_jsons/coalesceConformanceRuleTest.json b/examples/data/e2e_tests/test_jsons/coalesceConformanceRuleTest.json
@@ -2,9 +2,9 @@
 {
     "vars": {
       "spark-submit": "spark-submit --num-executors 2 --executor-memory 2G --deploy-mode client",
-      "spark-conf": "--conf 'spark.driver.extraJavaOptions=-Denceladus.rest.uri=http://localhost:8080/rest_api/api -Dspline.mongodb.name=spline -Dspline.mongodb.url=mongodb://127.0.0.1:27017/ -Denceladus.recordId.generation.strategy=stableHashId'",
-      "enceladus-job-jar": "spark-jobs/target/spark-jobs-2.19.0-SNAPSHOT.jar",
-      "credentials": "--rest-api-credentials-file ~/.ssh/menasCredential.properties",
+      "spark-conf": "--conf 'spark.driver.extraJavaOptions=-Denceladus.rest.uri=http://localhost:8080/rest_api/api -Denceladus.recordId.generation.strategy=stableHashId'",
+      "enceladus-job-jar": "spark-jobs/target/spark-jobs-3.0.0-SNAPSHOT.jar",
+      "credentials": "--rest-api-credentials-file ~/.ssh/menas-credential.properties",
       "ref-std-data-path": "/ref/coalesceConformanceRule/std",
       "new-std-data-path": "/tmp/conformance-output/standardized-coalesceConformanceRule-1-2020-03-23-1",
       "ref-publish-data-path": "/ref/coalesceConformanceRule/publish",

diff --git a/examples/data/e2e_tests/test_jsons/concatenationConformanceRuleTest.json b/examples/data/e2e_tests/test_jsons/concatenationConformanceRuleTest.json
@@ -2,9 +2,9 @@
 {
     "vars": {
       "spark-submit": "spark-submit --num-executors 2 --executor-memory 2G --deploy-mode client",
-      "spark-conf": "--conf 'spark.driver.extraJavaOptions=-Denceladus.rest.uri=http://localhost:8080/rest_api/api -Dspline.mongodb.name=spline -Dspline.mongodb.url=mongodb://127.0.0.1:27017/ -Denceladus.recordId.generation.strategy=stableHashId'",
-      "enceladus-job-jar": "spark-jobs/target/spark-jobs-2.19.0-SNAPSHOT.jar",
-      "credentials": "--rest-api-credentials-file ~/.ssh/menasCredential.properties",
+      "spark-conf": "--conf 'spark.driver.extraJavaOptions=-Denceladus.rest.uri=http://localhost:8080/rest_api/api -Denceladus.recordId.generation.strategy=stableHashId'",
+      "enceladus-job-jar": "spark-jobs/target/spark-jobs-3.0.0-SNAPSHOT.jar",
+      "credentials": "--rest-api-credentials-file ~/.ssh/menas-credential.properties",
       "ref-std-data-path": "/ref/concatenationConformanceRule/std",
       "new-std-data-path": "/tmp/conformance-output/standardized-concatenationConformanceRule-1-2020-03-23-1",
       "ref-publish-data-path": "/ref/concatenationConformanceRule/publish",

diff --git a/examples/data/e2e_tests/test_jsons/literalConformanceRuleTest.json b/examples/data/e2e_tests/test_jsons/literalConformanceRuleTest.json
@@ -2,9 +2,9 @@
 {
     "vars": {
       "spark-submit": "spark-submit --num-executors 2 --executor-memory 2G --deploy-mode client",
-      "spark-conf": "--conf 'spark.driver.extraJavaOptions=-Denceladus.rest.uri=http://localhost:8080/rest_api/api -Dspline.mongodb.name=spline -Dspline.mongodb.url=mongodb://127.0.0.1:27017/ -Denceladus.recordId.generation.strategy=stableHashId'",
-      "enceladus-job-jar": "spark-jobs/target/spark-jobs-2.19.0-SNAPSHOT.jar",
-      "credentials": "--rest-api-credentials-file ~/.ssh/menasCredential.properties",
+      "spark-conf": "--conf 'spark.driver.extraJavaOptions=-Denceladus.rest.uri=http://localhost:8080/rest_api/api -Denceladus.recordId.generation.strategy=stableHashId'",
+      "enceladus-job-jar": "spark-jobs/target/spark-jobs-3.0.0-SNAPSHOT.jar",
+      "credentials": "--rest-api-credentials-file ~/.ssh/menas-credential.properties",
       "ref-std-data-path": "/ref/literalConformanceRule/std",
       "new-std-data-path": "/tmp/conformance-output/standardized-literalConformanceRule-1-2020-03-23-1",
       "ref-publish-data-path": "/ref/literalConformanceRule/publish",

diff --git a/examples/data/e2e_tests/test_jsons/mappingConformanceRuleTest.json b/examples/data/e2e_tests/test_jsons/mappingConformanceRuleTest.json
@@ -2,9 +2,9 @@
 {
     "vars": {
       "spark-submit": "spark-submit --num-executors 2 --executor-memory 2G --deploy-mode client",
-      "spark-conf": "--conf 'spark.driver.extraJavaOptions=-Denceladus.rest.uri=http://localhost:8080/rest_api/api -Dspline.mongodb.name=spline -Dspline.mongodb.url=mongodb://127.0.0.1:27017/ -Denceladus.recordId.generation.strategy=stableHashId'",
-      "enceladus-job-jar": "spark-jobs/target/spark-jobs-2.19.0-SNAPSHOT.jar",
-      "credentials": "--rest-api-credentials-file ~/.ssh/menasCredential.properties",
+      "spark-conf": "--conf 'spark.driver.extraJavaOptions=-Denceladus.rest.uri=http://localhost:8080/rest_api/api -Denceladus.recordId.generation.strategy=stableHashId'",
+      "enceladus-job-jar": "spark-jobs/target/spark-jobs-3.0.0-SNAPSHOT.jar",
+      "credentials": "--rest-api-credentials-file ~/.ssh/menas-credential.properties",
       "ref-std-data-path": "/ref/mappingConformanceRule/std",
       "new-std-data-path": "/tmp/conformance-output/standardized-mappingConformanceRule-1-2020-03-23-1",
       "ref-publish-data-path": "/ref/mappingConformanceRule/publish",

diff --git a/examples/data/e2e_tests/test_jsons/negationConformanceRuleTest.json b/examples/data/e2e_tests/test_jsons/negationConformanceRuleTest.json
@@ -2,9 +2,9 @@
 {
     "vars": {
       "spark-submit": "spark-submit --num-executors 2 --executor-memory 2G --deploy-mode client",
-      "spark-conf": "--conf 'spark.driver.extraJavaOptions=-Denceladus.rest.uri=http://localhost:8080/rest_api/api -Dspline.mongodb.name=spline -Dspline.mongodb.url=mongodb://127.0.0.1:27017/ -Denceladus.recordId.generation.strategy=stableHashId'",
-      "enceladus-job-jar": "spark-jobs/target/spark-jobs-2.19.0-SNAPSHOT.jar",
-      "credentials": "--rest-api-credentials-file ~/.ssh/menasCredential.properties",
+      "spark-conf": "--conf 'spark.driver.extraJavaOptions=-Denceladus.rest.uri=http://localhost:8080/rest_api/api -Denceladus.recordId.generation.strategy=stableHashId'",
+      "enceladus-job-jar": "spark-jobs/target/spark-jobs-3.0.0-SNAPSHOT.jar",
+      "credentials": "--rest-api-credentials-file ~/.ssh/menas-credential.properties",
       "ref-std-data-path": "/ref/negationConformanceRule/std",
       "new-std-data-path": "/tmp/conformance-output/standardized-negationConformanceRule-1-2020-03-23-1",
       "ref-publish-data-path": "/ref/negationConformanceRule/publish",

diff --git a/examples/data/e2e_tests/test_jsons/singleColumnConformanceRuleTest.json b/examples/data/e2e_tests/test_jsons/singleColumnConformanceRuleTest.json
@@ -2,9 +2,9 @@
 {
     "vars": {
       "spark-submit": "spark-submit --num-executors 2 --executor-memory 2G --deploy-mode client",
-      "spark-conf": "--conf 'spark.driver.extraJavaOptions=-Denceladus.rest.uri=http://localhost:8080/rest_api/api -Dspline.mongodb.name=spline -Dspline.mongodb.url=mongodb://127.0.0.1:27017/ -Denceladus.recordId.generation.strategy=stableHashId'",
-      "enceladus-job-jar": "spark-jobs/target/spark-jobs-2.19.0-SNAPSHOT.jar",
-      "credentials": "--rest-api-credentials-file ~/.ssh/menasCredential.properties",
+      "spark-conf": "--conf 'spark.driver.extraJavaOptions=-Denceladus.rest.uri=http://localhost:8080/rest_api/api -Denceladus.recordId.generation.strategy=stableHashId'",
+      "enceladus-job-jar": "spark-jobs/target/spark-jobs-3.0.0-SNAPSHOT.jar",
+      "credentials": "--rest-api-credentials-file ~/.ssh/menas-credential.properties",
       "ref-std-data-path": "/ref/singleColumnConformanceRule/std",
       "new-std-data-path": "/tmp/conformance-output/standardized-singleColumnConformanceRule-1-2020-03-23-1",
       "ref-publish-data-path": "/ref/singleColumnConformanceRule/publish",

diff --git a/examples/data/e2e_tests/test_jsons/stdNfDnTest.json b/examples/data/e2e_tests/test_jsons/stdNfDnTest.json
@@ -1,10 +1,10 @@
 
 {
     "vars": {
-      "spark-submit": "spark-submit --num-executors 2 --executor-memory 2G --deploy-mode client",
+      "spark-submit": "spark-submit --num-executors 2 --executor-memory 2G --deploy-mode client --conf spark.sql.parquet.datetimeRebaseModeInRead=LEGACY --conf spark.sql.parquet.datetimeRebaseModeInWrite=LEGACY",
       "spark-conf": "--conf 'spark.driver.extraJavaOptions=-Denceladus.rest.uri=http://localhost:8080/rest_api/api -Dspline.mongodb.name=spline -Dspline.mongodb.url=mongodb://127.0.0.1:27017/ -Denceladus.recordId.generation.strategy=stableHashId'",
-      "enceladus-job-jar": "spark-jobs/target/spark-jobs-2.19.0-SNAPSHOT.jar",
-      "credentials": "--rest-api-credentials-file ~/.ssh/menasCredential.properties",
+      "enceladus-job-jar": "spark-jobs/target/spark-jobs-3.0.0-SNAPSHOT.jar",
+      "credentials": "--rest-api-credentials-file ~/.ssh/menas-credential.properties",
       "ref-std-data-path": "/ref/std_nf_dn/std",
       "new-std-data-path": "/tmp/conformance-output/standardized-std_nf_dn-1-2019-11-27-1",
       "results-log-path": "/std/std_nf_dn/results",
@@ -15,7 +15,7 @@
         "pluginName" : "BashPlugin",
         "name": "Standardization",
         "order" : 0,
-        "args" : ["#{spark-submit}# #{spark-conf}# --class za.co.absa.enceladus.standardization.StandardizationJob #{enceladus-job-jar}# #{credentials}# #{dataset}# --raw-format csv --header true "],
+        "args" : ["#{spark-submit}# #{spark-conf}# --class za.co.absa.enceladus.standardization.StandardizationJob #{enceladus-job-jar}# #{credentials}# #{dataset}# --raw-format json"],
         "writeArgs": []
     },
       {
@@ -30,7 +30,7 @@
         "pluginName" : "DatasetComparison",
         "name": "DatasetComparison",
         "order" : 1,
-        "args" : ["--format", "parquet", "--new-path", "#{new-std-data-path}#", "--ref-path", "#{ref-std-data-path}#", "--keys", "name" ],
+        "args" : ["--format", "parquet", "--new-path", "#{new-std-data-path}#", "--ref-path", "#{ref-std-data-path}#", "--keys", "property"],
         "writeArgs": ["--out-path", "#{results-log-path}#/stdDataDiff"],
         "dependsOn": "Standardization"
       }

diff --git a/examples/data/e2e_tests/test_jsons/stdNfDyTest.json b/examples/data/e2e_tests/test_jsons/stdNfDyTest.json
@@ -1,10 +1,10 @@
 
 {
     "vars": {
-      "spark-submit": "spark-submit --num-executors 2 --executor-memory 2G --deploy-mode client",
+      "spark-submit": "spark-submit --num-executors 2 --executor-memory 2G --deploy-mode client --conf spark.sql.parquet.datetimeRebaseModeInRead=LEGACY --conf spark.sql.parquet.datetimeRebaseModeInWrite=LEGACY",
       "spark-conf": "--conf 'spark.driver.extraJavaOptions=-Denceladus.rest.uri=http://localhost:8080/rest_api/api -Dspline.mongodb.name=spline -Dspline.mongodb.url=mongodb://127.0.0.1:27017/ -Denceladus.recordId.generation.strategy=stableHashId'",
-      "enceladus-job-jar": "spark-jobs/target/spark-jobs-2.19.0-SNAPSHOT.jar",
-      "credentials": "--rest-api-credentials-file ~/.ssh/menasCredential.properties",
+      "enceladus-job-jar": "spark-jobs/target/spark-jobs-3.0.0-SNAPSHOT.jar",
+      "credentials": "--rest-api-credentials-file ~/.ssh/menas-credential.properties",
       "ref-std-data-path": "/ref/std_nf_dy/std",
       "new-std-data-path": "/tmp/conformance-output/standardized-std_nf_dy-1-2019-11-27-1",
       "results-log-path": "/std/std_nf_dy/results",
@@ -15,7 +15,7 @@
         "pluginName" : "BashPlugin",
         "name": "Standardization",
         "order" : 0,
-        "args" : ["#{spark-submit}# #{spark-conf}# --class za.co.absa.enceladus.standardization.StandardizationJob #{enceladus-job-jar}# #{credentials}# #{dataset}# --raw-format csv --header true "],
+        "args" : ["#{spark-submit}# #{spark-conf}# --class za.co.absa.enceladus.standardization.StandardizationJob #{enceladus-job-jar}# #{credentials}# #{dataset}# --raw-format json"],
         "writeArgs": []
     },
       {
@@ -30,7 +30,7 @@
         "pluginName" : "DatasetComparison",
         "name": "DatasetComparison",
         "order" : 1,
-        "args" : ["--format", "parquet", "--new-path", "#{new-std-data-path}#", "--ref-path", "#{ref-std-data-path}#", "--keys", "name" ],
+        "args" : ["--format", "parquet", "--new-path", "#{new-std-data-path}#", "--ref-path", "#{ref-std-data-path}#", "--keys", "property"],
         "writeArgs": ["--out-path", "#{results-log-path}#/stdDataDiff"],
         "dependsOn": "Standardization"
       }

diff --git a/examples/data/e2e_tests/test_jsons/stdNtDnTest.json b/examples/data/e2e_tests/test_jsons/stdNtDnTest.json
@@ -1,10 +1,10 @@
 
 {
     "vars": {
-      "spark-submit": "spark-submit --num-executors 2 --executor-memory 2G --deploy-mode client",
+      "spark-submit": "spark-submit --num-executors 2 --executor-memory 2G --deploy-mode client --conf spark.sql.parquet.datetimeRebaseModeInRead=LEGACY --conf spark.sql.parquet.datetimeRebaseModeInWrite=LEGACY",
       "spark-conf": "--conf 'spark.driver.extraJavaOptions=-Denceladus.rest.uri=http://localhost:8080/rest_api/api -Dspline.mongodb.name=spline -Dspline.mongodb.url=mongodb://127.0.0.1:27017/ -Denceladus.recordId.generation.strategy=stableHashId'",
-      "enceladus-job-jar": "spark-jobs/target/spark-jobs-2.19.0-SNAPSHOT.jar",
-      "credentials": "--rest-api-credentials-file ~/.ssh/menasCredential.properties",
+      "enceladus-job-jar": "spark-jobs/target/spark-jobs-3.0.0-SNAPSHOT.jar",
+      "credentials": "--rest-api-credentials-file ~/.ssh/menas-credential.properties",
       "ref-std-data-path": "/ref/std_nt_dn/std",
       "new-std-data-path": "/tmp/conformance-output/standardized-std_nt_dn-1-2019-11-27-1",
       "results-log-path": "/std/std_nt_dn/results",
@@ -15,7 +15,7 @@
         "pluginName" : "BashPlugin",
         "name": "Standardization",
         "order" : 0,
-        "args" : ["#{spark-submit}# #{spark-conf}# --class za.co.absa.enceladus.standardization.StandardizationJob #{enceladus-job-jar}# #{credentials}# #{dataset}# --raw-format csv --header true "],
+        "args" : ["#{spark-submit}# #{spark-conf}# --class za.co.absa.enceladus.standardization.StandardizationJob #{enceladus-job-jar}# #{credentials}# #{dataset}# --raw-format json"],
         "writeArgs": []
     },
       {
@@ -30,7 +30,7 @@
         "pluginName" : "DatasetComparison",
         "name": "DatasetComparison",
         "order" : 1,
-        "args" : ["--format", "parquet", "--new-path", "#{new-std-data-path}#", "--ref-path", "#{ref-std-data-path}#", "--keys", "name" ],
+        "args" : ["--format", "parquet", "--new-path", "#{new-std-data-path}#", "--ref-path", "#{ref-std-data-path}#", "--keys", "property"],
         "writeArgs": ["--out-path", "#{results-log-path}#/stdDataDiff"],
         "dependsOn": "Standardization"
       }

diff --git a/examples/data/e2e_tests/test_jsons/stdNtDyTest.json b/examples/data/e2e_tests/test_jsons/stdNtDyTest.json
@@ -1,10 +1,10 @@
 
 {
     "vars": {
-      "spark-submit": "spark-submit --num-executors 2 --executor-memory 2G --deploy-mode client",
+      "spark-submit": "spark-submit --num-executors 2 --executor-memory 2G --deploy-mode client --conf spark.sql.parquet.datetimeRebaseModeInRead=LEGACY --conf spark.sql.parquet.datetimeRebaseModeInWrite=LEGACY",
       "spark-conf": "--conf 'spark.driver.extraJavaOptions=-Denceladus.rest.uri=http://localhost:8080/rest_api/api -Dspline.mongodb.name=spline -Dspline.mongodb.url=mongodb://127.0.0.1:27017/ -Denceladus.recordId.generation.strategy=stableHashId'",
-      "enceladus-job-jar": "spark-jobs/target/spark-jobs-2.19.0-SNAPSHOT.jar",
-      "credentials": "--rest-api-credentials-file ~/.ssh/menasCredential.properties",
+      "enceladus-job-jar": "spark-jobs/target/spark-jobs-3.0.0-SNAPSHOT.jar",
+      "credentials": "--rest-api-credentials-file ~/.ssh/menas-credential.properties",
       "ref-std-data-path": "/ref/std_nt_dy/std",
       "new-std-data-path": "/tmp/conformance-output/standardized-std_nt_dy-1-2019-11-27-1",
       "results-log-path": "/std/std_nt_dy/results",
@@ -15,7 +15,7 @@
         "pluginName" : "BashPlugin",
         "name": "Standardization",
         "order" : 0,
-        "args" : ["#{spark-submit}# #{spark-conf}# --class za.co.absa.enceladus.standardization.StandardizationJob #{enceladus-job-jar}# #{credentials}# #{dataset}# --raw-format csv --header true "],
+        "args" : ["#{spark-submit}# #{spark-conf}# --class za.co.absa.enceladus.standardization.StandardizationJob #{enceladus-job-jar}# #{credentials}# #{dataset}# --raw-format json"],
         "writeArgs": []
     },
       {
@@ -30,7 +30,7 @@
         "pluginName" : "DatasetComparison",
         "name": "DatasetComparison",
         "order" : 1,
-        "args" : ["--format", "parquet", "--new-path", "#{new-std-data-path}#", "--ref-path", "#{ref-std-data-path}#", "--keys", "name" ],
+        "args" : ["--format", "parquet", "--new-path", "#{new-std-data-path}#", "--ref-path", "#{ref-std-data-path}#", "--keys", "property"],
         "writeArgs": ["--out-path", "#{results-log-path}#/stdDataDiff"],
         "dependsOn": "Standardization"
       }

diff --git a/examples/data/e2e_tests/test_jsons/uppercaseConformanceRuleTest.json b/examples/data/e2e_tests/test_jsons/uppercaseConformanceRuleTest.json
@@ -2,9 +2,9 @@
 {
     "vars": {
       "spark-submit": "spark-submit --num-executors 2 --executor-memory 2G --deploy-mode client",
-      "spark-conf": "--conf 'spark.driver.extraJavaOptions=-Denceladus.rest.uri=http://localhost:8080/rest_api/api -Dspline.mongodb.name=spline -Dspline.mongodb.url=mongodb://127.0.0.1:27017/ -Denceladus.recordId.generation.strategy=stableHashId'",
-      "enceladus-job-jar": "spark-jobs/target/spark-jobs-2.19.0-SNAPSHOT.jar",
-      "credentials": "--rest-api-credentials-file ~/.ssh/menasCredential.properties",
+      "spark-conf": "--conf 'spark.driver.extraJavaOptions=-Denceladus.rest.uri=http://localhost:8080/rest_api/api -Denceladus.recordId.generation.strategy=stableHashId'",
+      "enceladus-job-jar": "spark-jobs/target/spark-jobs-3.0.0-SNAPSHOT.jar",
+      "credentials": "--rest-api-credentials-file ~/.ssh/menas-credential.properties",
       "ref-std-data-path": "/ref/uppercaseConformanceRule/std",
       "new-std-data-path": "/tmp/conformance-output/standardized-uppercaseConformanceRule-1-2020-03-23-1",
       "ref-publish-data-path": "/ref/uppercaseConformanceRule/publish",