From b57227702e009b05254f685e8fbd57e0707f403e Mon Sep 17 00:00:00 2001 From: Ghislain Fourny Date: Tue, 14 Jun 2022 14:21:10 +0200 Subject: [PATCH 01/20] Update main screen. --- src/main/resources/assets/defaultscreen.txt | 29 +++++---------------- 1 file changed, 6 insertions(+), 23 deletions(-) diff --git a/src/main/resources/assets/defaultscreen.txt b/src/main/resources/assets/defaultscreen.txt index 68306781ac..e47238739b 100644 --- a/src/main/resources/assets/defaultscreen.txt +++ b/src/main/resources/assets/defaultscreen.txt @@ -9,7 +9,7 @@ If you do not want to install Spark, then you need to use the standalone jar instead from www.rumbledb.org. Usage: -spark-submit +docker run -i rumbledb/rumble The first optional argument specifies the mode: **** run **** @@ -17,11 +17,11 @@ for directly running a query from an input file or (with -q) provided directly o It is the default mode. -spark-submit rumbledb-1.19.0.jar run my-query.jq -spark-submit rumbledb-1.19.0.jar run -q '1+1' +docker run -i rumbledb/rumble run my-query.jq +docker run -i rumbledb/rumble run -q '1+1' You can specify an output path with -o like so: -spark-submit rumbledb-1.19.0.jar run -q '1+1' -o my-output.txt +docker run -i rumbledb/rumble run -q '1+1' -o my-output.txt **** serve **** for running as an HTTP server listening on the specified port (-p) and host (-h). @@ -39,24 +39,7 @@ spark-submit rumbledb-1.19.0.jar repl **** resource use configuration **** -For a local use, you can control the number of cores, as well as allocated -memory, with: -spark-submit --master local[*] rumbledb-1.19.0.jar repl -spark-submit --master local[*] rumbledb-1.19.0.jar repl -spark-submit --master local[2] rumbledb-1.19.0.jar repl -spark-submit --master local[*] --driver-memory 10G rumbledb-1.19.0.jar repl - -You can use RumbleDB remotely with: -spark-submit --master yarn rumbledb-1.19.0.jar repl - -(Although for clusters provided as a service, --master yarn is often implicit -and unnecessary). - -For remote use (e.g., logged in on the Spark cluster with ssh), you can set the -number of executors, cores and memory, you can use: -spark-submit --executor-cores 3 --executor-memory 5G rumbledb-1.19.0.jar repl - -For remote use, you can also use other file system paths such as S3, HDFS, etc: -spark-submit rumbledb-1.19.0.jar run hdfs://server:port/my-query.jq -o hdfs://server:port/my-output.json +The docker edition of RumbleDB cannot change the number of cores or the memory allocation. +If you wish to do so, you could consider using the standalone RumbleDB jar. More documentation on available CLI parameters is available on https://www.rumbledb.org/ From 52a5a1e146b66797bd73349546f64296ddff67f0 Mon Sep 17 00:00:00 2001 From: Ghislain Fourny Date: Tue, 14 Jun 2022 14:25:17 +0200 Subject: [PATCH 02/20] Update info for brew. --- src/main/resources/assets/defaultscreen.txt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/main/resources/assets/defaultscreen.txt b/src/main/resources/assets/defaultscreen.txt index e47238739b..03069aefe9 100644 --- a/src/main/resources/assets/defaultscreen.txt +++ b/src/main/resources/assets/defaultscreen.txt @@ -9,7 +9,7 @@ If you do not want to install Spark, then you need to use the standalone jar instead from www.rumbledb.org. Usage: -docker run -i rumbledb/rumble +rumbledb The first optional argument specifies the mode: **** run **** @@ -17,16 +17,16 @@ for directly running a query from an input file or (with -q) provided directly o It is the default mode. -docker run -i rumbledb/rumble run my-query.jq -docker run -i rumbledb/rumble run -q '1+1' +rumbledb run my-query.jq +rumbledb run -q '1+1' You can specify an output path with -o like so: -docker run -i rumbledb/rumble run -q '1+1' -o my-output.txt +rumbledb run -q '1+1' -o my-output.txt **** serve **** for running as an HTTP server listening on the specified port (-p) and host (-h). -spark-submit rumbledb-1.19.0.jar serve -p 9090 +rumbledb serve -p 9090 RumbleDB also supports Apache Livy for use in Jupyter notebooks, which may be even more convenient if you are using a cluster. @@ -34,7 +34,7 @@ even more convenient if you are using a cluster. **** repl **** for shell mode. -spark-submit rumbledb-1.19.0.jar repl +rumbledb repl **** resource use configuration **** From 3bebc9ebb952d96aebc2b479af11f74786da616d Mon Sep 17 00:00:00 2001 From: Ghislain Fourny Date: Tue, 16 May 2023 14:10:42 +0200 Subject: [PATCH 03/20] Fix parsing. --- src/main/java/org/rumbledb/items/Base64BinaryItem.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/rumbledb/items/Base64BinaryItem.java b/src/main/java/org/rumbledb/items/Base64BinaryItem.java index 30ea447849..73bdce1f5c 100644 --- a/src/main/java/org/rumbledb/items/Base64BinaryItem.java +++ b/src/main/java/org/rumbledb/items/Base64BinaryItem.java @@ -41,7 +41,7 @@ public Base64BinaryItem() { public Base64BinaryItem(String stringValue) { this.value = parseBase64BinaryString(stringValue); - this.stringValue = StringUtils.chomp(Base64.encodeBase64String(this.value)); + this.stringValue = StringUtils.chomp(Base64.getEncoder().encodeToString(this.value)); } @Override From 4dc86cd5ba74c18a60d900544cb024c55e5422c5 Mon Sep 17 00:00:00 2001 From: Ghislain Fourny Date: Wed, 17 May 2023 14:15:48 +0200 Subject: [PATCH 04/20] Remove java version check because of Java 20 dependency on Brew. --- src/main/java/org/rumbledb/cli/Main.java | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/main/java/org/rumbledb/cli/Main.java b/src/main/java/org/rumbledb/cli/Main.java index 436a14dc69..db73e690b2 100644 --- a/src/main/java/org/rumbledb/cli/Main.java +++ b/src/main/java/org/rumbledb/cli/Main.java @@ -36,16 +36,6 @@ public class Main { public static RumbleJLineShell terminal = null; public static void main(String[] args) throws IOException { - String javaVersion = System.getProperty("java.version"); - if (!javaVersion.startsWith("1.8") && !javaVersion.startsWith("11.")) { - System.err.println("[Error] RumbleDB requires Java 8 or Java 11."); - System.err.println("Your Java version: " + System.getProperty("java.version")); - System.err.println("You can download Java 8 or 11 from https://adoptium.net/"); - System.err.println( - "If you do have Java 8 or 11, but the wrong version appears above, then it means you need to set your JAVA_HOME environment variable properly to point to Java 8 or 11." - ); - System.exit(43); - } RumbleRuntimeConfiguration sparksoniqConf = null; // Parse arguments try { From b73a128ea8c0ce6a73c9b4e9936a3724c8d85924 Mon Sep 17 00:00:00 2001 From: Ghislain Fourny Date: Thu, 21 Dec 2023 10:53:23 +0100 Subject: [PATCH 05/20] Update to Spark 3.5. --- pom.xml | 14 +++++++------- .../java/org/rumbledb/items/Base64BinaryItem.java | 2 +- src/main/java/org/rumbledb/items/DoubleItem.java | 2 +- src/main/java/org/rumbledb/items/FloatItem.java | 2 +- .../strings/NormalizeSpaceFunctionIterator.java | 2 +- .../org/rumbledb/runtime/typing/CastIterator.java | 2 +- .../spark/ml/ApplyEstimatorRuntimeIterator.java | 2 +- .../spark/ml/ApplyTransformerRuntimeIterator.java | 2 +- src/test/java/iq/base/AnnotationsTestsBase.java | 2 +- .../test_files/RumbleML/RumbleML/MLPipeline1.jq | 2 +- .../test_files/RumbleML/RumbleML/MLPipeline2.jq | 2 +- .../test_files/RumbleML/RumbleML/MLPipeline3.jq | 2 +- 12 files changed, 18 insertions(+), 18 deletions(-) diff --git a/pom.xml b/pom.xml index 55c12b0628..991bcd648c 100644 --- a/pom.xml +++ b/pom.xml @@ -200,31 +200,31 @@ org.apache.spark spark-core_2.12 - 3.3.2 + 3.5.0 provided org.apache.spark spark-sql_2.12 - 3.3.2 + 3.5.0 provided org.apache.spark spark-mllib_2.12 - 3.3.2 + 3.5.0 provided org.apache.hadoop hadoop-aws - 3.3.2 + 3.3.6 provided org.apache.spark spark-avro_2.12 - 3.3.2 + 3.4.0 org.antlr @@ -250,12 +250,12 @@ org.apache.commons commons-text - 1.6 + 1.10.0 org.apache.commons commons-lang3 - 3.9 + 3.12.0 commons-io diff --git a/src/main/java/org/rumbledb/items/Base64BinaryItem.java b/src/main/java/org/rumbledb/items/Base64BinaryItem.java index 73bdce1f5c..5e2b8dc3ad 100644 --- a/src/main/java/org/rumbledb/items/Base64BinaryItem.java +++ b/src/main/java/org/rumbledb/items/Base64BinaryItem.java @@ -3,7 +3,7 @@ import com.esotericsoftware.kryo.Kryo; import com.esotericsoftware.kryo.io.Input; import com.esotericsoftware.kryo.io.Output; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import java.util.Base64; import org.rumbledb.api.Item; import org.rumbledb.exceptions.ExceptionMetadata; diff --git a/src/main/java/org/rumbledb/items/DoubleItem.java b/src/main/java/org/rumbledb/items/DoubleItem.java index fab38f6247..a82018813c 100644 --- a/src/main/java/org/rumbledb/items/DoubleItem.java +++ b/src/main/java/org/rumbledb/items/DoubleItem.java @@ -24,7 +24,7 @@ import com.esotericsoftware.kryo.io.Input; import com.esotericsoftware.kryo.io.Output; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import org.rumbledb.api.Item; import org.rumbledb.exceptions.ExceptionMetadata; import org.rumbledb.exceptions.IteratorFlowException; diff --git a/src/main/java/org/rumbledb/items/FloatItem.java b/src/main/java/org/rumbledb/items/FloatItem.java index d9dca106c3..62f6a2aaee 100644 --- a/src/main/java/org/rumbledb/items/FloatItem.java +++ b/src/main/java/org/rumbledb/items/FloatItem.java @@ -24,7 +24,7 @@ import com.esotericsoftware.kryo.io.Input; import com.esotericsoftware.kryo.io.Output; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import org.rumbledb.api.Item; import org.rumbledb.exceptions.ExceptionMetadata; import org.rumbledb.exceptions.IteratorFlowException; diff --git a/src/main/java/org/rumbledb/runtime/functions/strings/NormalizeSpaceFunctionIterator.java b/src/main/java/org/rumbledb/runtime/functions/strings/NormalizeSpaceFunctionIterator.java index 9b19d0b89d..13b704934d 100644 --- a/src/main/java/org/rumbledb/runtime/functions/strings/NormalizeSpaceFunctionIterator.java +++ b/src/main/java/org/rumbledb/runtime/functions/strings/NormalizeSpaceFunctionIterator.java @@ -20,7 +20,7 @@ package org.rumbledb.runtime.functions.strings; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import org.rumbledb.api.Item; import org.rumbledb.context.DynamicContext; import org.rumbledb.context.Name; diff --git a/src/main/java/org/rumbledb/runtime/typing/CastIterator.java b/src/main/java/org/rumbledb/runtime/typing/CastIterator.java index 082b9fb499..b96022e0dc 100644 --- a/src/main/java/org/rumbledb/runtime/typing/CastIterator.java +++ b/src/main/java/org/rumbledb/runtime/typing/CastIterator.java @@ -2,7 +2,7 @@ import org.apache.commons.codec.binary.Base64; import org.apache.commons.codec.binary.Hex; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import org.rumbledb.api.Item; import org.rumbledb.context.DynamicContext; import org.rumbledb.context.RuntimeStaticContext; diff --git a/src/main/java/sparksoniq/spark/ml/ApplyEstimatorRuntimeIterator.java b/src/main/java/sparksoniq/spark/ml/ApplyEstimatorRuntimeIterator.java index 05dad6ca1d..f6b655db4b 100644 --- a/src/main/java/sparksoniq/spark/ml/ApplyEstimatorRuntimeIterator.java +++ b/src/main/java/sparksoniq/spark/ml/ApplyEstimatorRuntimeIterator.java @@ -1,6 +1,6 @@ package sparksoniq.spark.ml; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.spark.ml.Estimator; import org.apache.spark.ml.Transformer; import org.apache.spark.ml.linalg.VectorUDT; diff --git a/src/main/java/sparksoniq/spark/ml/ApplyTransformerRuntimeIterator.java b/src/main/java/sparksoniq/spark/ml/ApplyTransformerRuntimeIterator.java index 81e288c594..8c6607f08e 100644 --- a/src/main/java/sparksoniq/spark/ml/ApplyTransformerRuntimeIterator.java +++ b/src/main/java/sparksoniq/spark/ml/ApplyTransformerRuntimeIterator.java @@ -1,6 +1,6 @@ package sparksoniq.spark.ml; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.spark.ml.Transformer; import org.apache.spark.ml.linalg.VectorUDT; import org.apache.spark.ml.param.ParamMap; diff --git a/src/test/java/iq/base/AnnotationsTestsBase.java b/src/test/java/iq/base/AnnotationsTestsBase.java index 30d4353d19..c3a7dff39a 100644 --- a/src/test/java/iq/base/AnnotationsTestsBase.java +++ b/src/test/java/iq/base/AnnotationsTestsBase.java @@ -20,7 +20,7 @@ package iq.base; -import org.apache.commons.lang.exception.ExceptionUtils; +import org.apache.commons.lang3.exception.ExceptionUtils; import org.junit.Assert; import org.rumbledb.api.Item; import org.rumbledb.api.Rumble; diff --git a/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline1.jq b/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline1.jq index 4a5b07f1d2..2979627829 100644 --- a/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline1.jq +++ b/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline1.jq @@ -1,4 +1,4 @@ -(:JIQS: ShouldRun; Output="({ "id" : 0, "label" : 1, "col1" : -1, "col2" : 1.5, "col3" : 1.3, "features" : [ -1, 1.5, 1.3 ], "features2" : [ 0.6811443247067492, -1.6574062675838126 ], "rawPrediction" : [ -37.806767087840996, 37.806767087840996 ], "probability" : [ 3.808287043115909E-17, 1 ], "ocol1" : 1 }, { "id" : 1, "label" : 0, "col1" : 3, "col2" : 2, "col3" : -0.1, "features" : [ 3, 2, -0.1 ], "features2" : [ -2.93898570550277, 0.48441694826197534 ], "rawPrediction" : [ 37.56770709258572, -37.56770709258572 ], "probability" : [ 1, 0 ], "ocol1" : 0 }, { "id" : 2, "label" : 1, "col1" : 0, "col2" : 2.2, "col3" : -1.5, "features" : [ 0, 2.2, -1.5 ], "features2" : [ 0.28609394877170796, 1.1822492465709793 ], "rawPrediction" : [ -19.16835344100661, 19.16835344100661 ], "probability" : [ 4.734671708106428E-9, 0.9999999952653283 ], "ocol1" : 1 })" :) +(:JIQS: ShouldRun; Output="({ "id" : 0, "label" : 1, "col1" : -1, "col2" : 1.5, "col3" : 1.3, "features" : [ -1, 1.5, 1.3 ], "features2" : [ 0.6811443247067493, -1.6574062675838126 ], "rawPrediction" : [ -37.806767087841095, 37.806767087841095 ], "probability" : [ 3.80828704311553E-17, 1 ], "ocol1" : 1 }, { "id" : 1, "label" : 0, "col1" : 3, "col2" : 2, "col3" : -0.1, "features" : [ 3, 2, -0.1 ], "features2" : [ -2.9389857055027706, 0.48441694826197546 ], "rawPrediction" : [ 37.56770709258579, -37.56770709258579 ], "probability" : [ 1, 0 ], "ocol1" : 0 }, { "id" : 2, "label" : 1, "col1" : 0, "col2" : 2.2, "col3" : -1.5, "features" : [ 0, 2.2, -1.5 ], "features2" : [ 0.28609394877170796, 1.1822492465709793 ], "rawPrediction" : [ -19.168353441006694, 19.168353441006694 ], "probability" : [ 4.734671708106025E-9, 0.9999999952653283 ], "ocol1" : 1 })" :) let $vector-assembler := get-transformer("VectorAssembler") let $training-data := ( {"id": 0, "label": 1, "col1": 0.0, "col2": 1.1, "col3": 0.1}, diff --git a/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline2.jq b/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline2.jq index 5bdd8da1a7..57b2c3bc09 100644 --- a/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline2.jq +++ b/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline2.jq @@ -1,4 +1,4 @@ -(:JIQS: ShouldRun; Output="({ "id" : 0, "label" : 1, "col1" : -1, "col2" : 1.5, "col3" : 1.3, "features" : [ -1, 1.5, 1.3 ], "features2" : [ 0.6811443247067492, -1.6574062675838126 ], "rawPrediction" : [ -37.806767087840996, 37.806767087840996 ], "probability" : [ 3.808287043115909E-17, 1 ], "ocol1" : 1 }, { "id" : 1, "label" : 0, "col1" : 3, "col2" : 2, "col3" : -0.1, "features" : [ 3, 2, -0.1 ], "features2" : [ -2.93898570550277, 0.48441694826197534 ], "rawPrediction" : [ 37.56770709258572, -37.56770709258572 ], "probability" : [ 1, 0 ], "ocol1" : 0 }, { "id" : 2, "label" : 1, "col1" : 0, "col2" : 2.2, "col3" : -1.5, "features" : [ 0, 2.2, -1.5 ], "features2" : [ 0.28609394877170796, 1.1822492465709793 ], "rawPrediction" : [ -19.16835344100661, 19.16835344100661 ], "probability" : [ 4.734671708106428E-9, 0.9999999952653283 ], "ocol1" : 1 })" :) +(:JIQS: ShouldRun; Output="({ "id" : 0, "label" : 1, "col1" : -1, "col2" : 1.5, "col3" : 1.3, "features" : [ -1, 1.5, 1.3 ], "features2" : [ 0.6811443247067493, -1.6574062675838126 ], "rawPrediction" : [ -37.806767087841095, 37.806767087841095 ], "probability" : [ 3.80828704311553E-17, 1 ], "ocol1" : 1 }, { "id" : 1, "label" : 0, "col1" : 3, "col2" : 2, "col3" : -0.1, "features" : [ 3, 2, -0.1 ], "features2" : [ -2.9389857055027706, 0.48441694826197546 ], "rawPrediction" : [ 37.56770709258579, -37.56770709258579 ], "probability" : [ 1, 0 ], "ocol1" : 0 }, { "id" : 2, "label" : 1, "col1" : 0, "col2" : 2.2, "col3" : -1.5, "features" : [ 0, 2.2, -1.5 ], "features2" : [ 0.28609394877170796, 1.1822492465709793 ], "rawPrediction" : [ -19.168353441006694, 19.168353441006694 ], "probability" : [ 4.734671708106025E-9, 0.9999999952653283 ], "ocol1" : 1 })" :) let $vector-assembler := get-transformer("VectorAssembler")(?, { "inputCols" : [ "col1", "col2", "col3" ], "outputCol" : "features" }) let $training-data := ( {"id": 0, "label": 1, "col1": 0.0, "col2": 1.1, "col3": 0.1}, diff --git a/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline3.jq b/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline3.jq index 43aefcb264..37e99a3949 100644 --- a/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline3.jq +++ b/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline3.jq @@ -1,4 +1,4 @@ -(:JIQS: ShouldRun; Output="({ "id" : 0, "label" : 1, "col1" : -1, "col2" : 1.5, "col3" : 1.3, "features" : [ -1, 1.5, 1.3 ], "features2" : [ 0.6811443247067492, -1.6574062675838126 ], "rawPrediction" : [ -37.806767087840996, 37.806767087840996 ], "probability" : [ 3.808287043115909E-17, 1 ], "ocol1" : 1 }, { "id" : 1, "label" : 0, "col1" : 3, "col2" : 2, "col3" : -0.1, "features" : [ 3, 2, -0.1 ], "features2" : [ -2.93898570550277, 0.48441694826197534 ], "rawPrediction" : [ 37.56770709258572, -37.56770709258572 ], "probability" : [ 1, 0 ], "ocol1" : 0 }, { "id" : 2, "label" : 1, "col1" : 0, "col2" : 2.2, "col3" : -1.5, "features" : [ 0, 2.2, -1.5 ], "features2" : [ 0.28609394877170796, 1.1822492465709793 ], "rawPrediction" : [ -19.16835344100661, 19.16835344100661 ], "probability" : [ 4.734671708106428E-9, 0.9999999952653283 ], "ocol1" : 1 })" :) +(:JIQS: ShouldRun; Output="({ "id" : 0, "label" : 1, "col1" : -1, "col2" : 1.5, "col3" : 1.3, "features" : [ -1, 1.5, 1.3 ], "features2" : [ 0.6811443247067493, -1.6574062675838126 ], "rawPrediction" : [ -37.806767087841095, 37.806767087841095 ], "probability" : [ 3.80828704311553E-17, 1 ], "ocol1" : 1 }, { "id" : 1, "label" : 0, "col1" : 3, "col2" : 2, "col3" : -0.1, "features" : [ 3, 2, -0.1 ], "features2" : [ -2.9389857055027706, 0.48441694826197546 ], "rawPrediction" : [ 37.56770709258579, -37.56770709258579 ], "probability" : [ 1, 0 ], "ocol1" : 0 }, { "id" : 2, "label" : 1, "col1" : 0, "col2" : 2.2, "col3" : -1.5, "features" : [ 0, 2.2, -1.5 ], "features2" : [ 0.28609394877170796, 1.1822492465709793 ], "rawPrediction" : [ -19.168353441006694, 19.168353441006694 ], "probability" : [ 4.734671708106025E-9, 0.9999999952653283 ], "ocol1" : 1 })" :) declare type local:mytype as { "id": "integer", "label": "integer", "col1": "decimal", "col2": "decimal", "col3": "decimal" From 9d0bfc4a242551501d6be357a6f6f9b190462028 Mon Sep 17 00:00:00 2001 From: Ghislain Fourny Date: Thu, 21 Dec 2023 11:04:05 +0100 Subject: [PATCH 06/20] Upgrade to AntLR 4.9.3. --- pom.xml | 2 +- .../rumbledb/parser/JsoniqBaseVisitor.java | 2 +- .../java/org/rumbledb/parser/JsoniqLexer.java | 4 +-- .../org/rumbledb/parser/JsoniqParser.java | 4 +-- .../org/rumbledb/parser/JsoniqVisitor.java | 2 +- .../java/org/rumbledb/parser/XQueryLexer.java | 4 +-- .../org/rumbledb/parser/XQueryParser.java | 29 ++++++------------- .../parser/XQueryParserBaseVisitor.java | 2 +- .../rumbledb/parser/XQueryParserVisitor.java | 2 +- 9 files changed, 20 insertions(+), 31 deletions(-) diff --git a/pom.xml b/pom.xml index 991bcd648c..aeb0a23d9a 100644 --- a/pom.xml +++ b/pom.xml @@ -229,7 +229,7 @@ org.antlr antlr4-runtime - 4.8 + 4.9.3 org.jline diff --git a/src/main/java/org/rumbledb/parser/JsoniqBaseVisitor.java b/src/main/java/org/rumbledb/parser/JsoniqBaseVisitor.java index 2944705596..378b338ba7 100644 --- a/src/main/java/org/rumbledb/parser/JsoniqBaseVisitor.java +++ b/src/main/java/org/rumbledb/parser/JsoniqBaseVisitor.java @@ -1,4 +1,4 @@ -// Generated from ./src/main/java/org/rumbledb/parser/Jsoniq.g4 by ANTLR 4.8 +// Generated from ./src/main/java/org/rumbledb/parser/Jsoniq.g4 by ANTLR 4.9.3 // Java header package org.rumbledb.parser; diff --git a/src/main/java/org/rumbledb/parser/JsoniqLexer.java b/src/main/java/org/rumbledb/parser/JsoniqLexer.java index 888f964a1d..10855f70de 100644 --- a/src/main/java/org/rumbledb/parser/JsoniqLexer.java +++ b/src/main/java/org/rumbledb/parser/JsoniqLexer.java @@ -1,4 +1,4 @@ -// Generated from ./src/main/java/org/rumbledb/parser/Jsoniq.g4 by ANTLR 4.8 +// Generated from ./src/main/java/org/rumbledb/parser/Jsoniq.g4 by ANTLR 4.9.3 // Java header package org.rumbledb.parser; @@ -16,7 +16,7 @@ @SuppressWarnings({"all", "warnings", "unchecked", "unused", "cast"}) public class JsoniqLexer extends Lexer { - static { RuntimeMetaData.checkVersion("4.8", RuntimeMetaData.VERSION); } + static { RuntimeMetaData.checkVersion("4.9.3", RuntimeMetaData.VERSION); } protected static final DFA[] _decisionToDFA; protected static final PredictionContextCache _sharedContextCache = diff --git a/src/main/java/org/rumbledb/parser/JsoniqParser.java b/src/main/java/org/rumbledb/parser/JsoniqParser.java index 299fcb8a32..f68fcbf4c0 100644 --- a/src/main/java/org/rumbledb/parser/JsoniqParser.java +++ b/src/main/java/org/rumbledb/parser/JsoniqParser.java @@ -1,4 +1,4 @@ -// Generated from ./src/main/java/org/rumbledb/parser/Jsoniq.g4 by ANTLR 4.8 +// Generated from ./src/main/java/org/rumbledb/parser/Jsoniq.g4 by ANTLR 4.9.3 // Java header package org.rumbledb.parser; @@ -25,7 +25,7 @@ @SuppressWarnings({"all", "warnings", "unchecked", "unused", "cast"}) public class JsoniqParser extends Parser { - static { RuntimeMetaData.checkVersion("4.8", RuntimeMetaData.VERSION); } + static { RuntimeMetaData.checkVersion("4.9.3", RuntimeMetaData.VERSION); } protected static final DFA[] _decisionToDFA; protected static final PredictionContextCache _sharedContextCache = diff --git a/src/main/java/org/rumbledb/parser/JsoniqVisitor.java b/src/main/java/org/rumbledb/parser/JsoniqVisitor.java index e4a5acf0df..b774486780 100644 --- a/src/main/java/org/rumbledb/parser/JsoniqVisitor.java +++ b/src/main/java/org/rumbledb/parser/JsoniqVisitor.java @@ -1,4 +1,4 @@ -// Generated from ./src/main/java/org/rumbledb/parser/Jsoniq.g4 by ANTLR 4.8 +// Generated from ./src/main/java/org/rumbledb/parser/Jsoniq.g4 by ANTLR 4.9.3 // Java header package org.rumbledb.parser; diff --git a/src/main/java/org/rumbledb/parser/XQueryLexer.java b/src/main/java/org/rumbledb/parser/XQueryLexer.java index 8429e5ce88..7e03a5a610 100644 --- a/src/main/java/org/rumbledb/parser/XQueryLexer.java +++ b/src/main/java/org/rumbledb/parser/XQueryLexer.java @@ -1,4 +1,4 @@ -// Generated from ./src/main/java/org/rumbledb/parser/XQueryLexer.g4 by ANTLR 4.8 +// Generated from ./src/main/java/org/rumbledb/parser/XQueryLexer.g4 by ANTLR 4.9.3 // Java header package org.rumbledb.parser; @@ -18,7 +18,7 @@ @SuppressWarnings({"all", "warnings", "unchecked", "unused", "cast"}) public class XQueryLexer extends Lexer { - static { RuntimeMetaData.checkVersion("4.8", RuntimeMetaData.VERSION); } + static { RuntimeMetaData.checkVersion("4.9.3", RuntimeMetaData.VERSION); } protected static final DFA[] _decisionToDFA; protected static final PredictionContextCache _sharedContextCache = diff --git a/src/main/java/org/rumbledb/parser/XQueryParser.java b/src/main/java/org/rumbledb/parser/XQueryParser.java index af62b9768d..f737166a85 100644 --- a/src/main/java/org/rumbledb/parser/XQueryParser.java +++ b/src/main/java/org/rumbledb/parser/XQueryParser.java @@ -1,31 +1,20 @@ -// Generated from ./src/main/java/org/rumbledb/parser/XQueryParser.g4 by ANTLR 4.8 +// Generated from ./src/main/java/org/rumbledb/parser/XQueryParser.g4 by ANTLR 4.9.3 // Java header package org.rumbledb.parser; -import java.util.ArrayList; -import java.util.List; - -import org.antlr.v4.runtime.NoViableAltException; -import org.antlr.v4.runtime.Parser; -import org.antlr.v4.runtime.ParserRuleContext; -import org.antlr.v4.runtime.RecognitionException; -import org.antlr.v4.runtime.RuntimeMetaData; -import org.antlr.v4.runtime.Token; -import org.antlr.v4.runtime.TokenStream; -import org.antlr.v4.runtime.Vocabulary; -import org.antlr.v4.runtime.VocabularyImpl; -import org.antlr.v4.runtime.atn.ATN; -import org.antlr.v4.runtime.atn.ATNDeserializer; -import org.antlr.v4.runtime.atn.ParserATNSimulator; -import org.antlr.v4.runtime.atn.PredictionContextCache; +import org.antlr.v4.runtime.atn.*; import org.antlr.v4.runtime.dfa.DFA; -import org.antlr.v4.runtime.tree.ParseTreeVisitor; -import org.antlr.v4.runtime.tree.TerminalNode; +import org.antlr.v4.runtime.*; +import org.antlr.v4.runtime.misc.*; +import org.antlr.v4.runtime.tree.*; +import java.util.List; +import java.util.Iterator; +import java.util.ArrayList; @SuppressWarnings({"all", "warnings", "unchecked", "unused", "cast"}) public class XQueryParser extends Parser { - static { RuntimeMetaData.checkVersion("4.8", RuntimeMetaData.VERSION); } + static { RuntimeMetaData.checkVersion("4.9.3", RuntimeMetaData.VERSION); } protected static final DFA[] _decisionToDFA; protected static final PredictionContextCache _sharedContextCache = diff --git a/src/main/java/org/rumbledb/parser/XQueryParserBaseVisitor.java b/src/main/java/org/rumbledb/parser/XQueryParserBaseVisitor.java index bef7ecec8e..f41ea731a7 100644 --- a/src/main/java/org/rumbledb/parser/XQueryParserBaseVisitor.java +++ b/src/main/java/org/rumbledb/parser/XQueryParserBaseVisitor.java @@ -1,4 +1,4 @@ -// Generated from ./src/main/java/org/rumbledb/parser/XQueryParser.g4 by ANTLR 4.8 +// Generated from ./src/main/java/org/rumbledb/parser/XQueryParser.g4 by ANTLR 4.9.3 // Java header package org.rumbledb.parser; diff --git a/src/main/java/org/rumbledb/parser/XQueryParserVisitor.java b/src/main/java/org/rumbledb/parser/XQueryParserVisitor.java index c730984360..d8dafaa785 100644 --- a/src/main/java/org/rumbledb/parser/XQueryParserVisitor.java +++ b/src/main/java/org/rumbledb/parser/XQueryParserVisitor.java @@ -1,4 +1,4 @@ -// Generated from ./src/main/java/org/rumbledb/parser/XQueryParser.g4 by ANTLR 4.8 +// Generated from ./src/main/java/org/rumbledb/parser/XQueryParser.g4 by ANTLR 4.9.3 // Java header package org.rumbledb.parser; From 570b548b69f8d540c789a2e4271440d920350af9 Mon Sep 17 00:00:00 2001 From: Ghislain Fourny Date: Thu, 21 Dec 2023 11:18:56 +0100 Subject: [PATCH 07/20] Change jackson version to align with Spark. --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index aeb0a23d9a..536a9ba68f 100644 --- a/pom.xml +++ b/pom.xml @@ -285,7 +285,7 @@ com.fasterxml.jackson.dataformat jackson-dataformat-yaml - 2.13.4 + 2.15.2 From c9931b3b1f5409537bab14a2109cf4aad0700ae1 Mon Sep 17 00:00:00 2001 From: Ghislain Fourny Date: Thu, 21 Dec 2023 11:21:31 +0100 Subject: [PATCH 08/20] Fix test. --- .../org/rumbledb/parser/XQueryParser.java | 25 +++++++++++++------ .../DataFrames/GroupbyClause5.jq | 3 ++- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/rumbledb/parser/XQueryParser.java b/src/main/java/org/rumbledb/parser/XQueryParser.java index f737166a85..5af047f1f1 100644 --- a/src/main/java/org/rumbledb/parser/XQueryParser.java +++ b/src/main/java/org/rumbledb/parser/XQueryParser.java @@ -3,14 +3,25 @@ // Java header package org.rumbledb.parser; -import org.antlr.v4.runtime.atn.*; -import org.antlr.v4.runtime.dfa.DFA; -import org.antlr.v4.runtime.*; -import org.antlr.v4.runtime.misc.*; -import org.antlr.v4.runtime.tree.*; -import java.util.List; -import java.util.Iterator; import java.util.ArrayList; +import java.util.List; + +import org.antlr.v4.runtime.NoViableAltException; +import org.antlr.v4.runtime.Parser; +import org.antlr.v4.runtime.ParserRuleContext; +import org.antlr.v4.runtime.RecognitionException; +import org.antlr.v4.runtime.RuntimeMetaData; +import org.antlr.v4.runtime.Token; +import org.antlr.v4.runtime.TokenStream; +import org.antlr.v4.runtime.Vocabulary; +import org.antlr.v4.runtime.VocabularyImpl; +import org.antlr.v4.runtime.atn.ATN; +import org.antlr.v4.runtime.atn.ATNDeserializer; +import org.antlr.v4.runtime.atn.ParserATNSimulator; +import org.antlr.v4.runtime.atn.PredictionContextCache; +import org.antlr.v4.runtime.dfa.DFA; +import org.antlr.v4.runtime.tree.ParseTreeVisitor; +import org.antlr.v4.runtime.tree.TerminalNode; @SuppressWarnings({"all", "warnings", "unchecked", "unused", "cast"}) public class XQueryParser extends Parser { diff --git a/src/test/resources/test_files/runtime-spark/DataFrames/GroupbyClause5.jq b/src/test/resources/test_files/runtime-spark/DataFrames/GroupbyClause5.jq index 64f860404e..18a657d22b 100644 --- a/src/test/resources/test_files/runtime-spark/DataFrames/GroupbyClause5.jq +++ b/src/test/resources/test_files/runtime-spark/DataFrames/GroupbyClause5.jq @@ -1,10 +1,11 @@ -(:JIQS: ShouldRun; Output="([ 1, { "guess" : "Bulgarian", "choices" : [ "Albanian", "Bulgarian", "Russian", "Ukrainian" ] }, { "guess" : "Bulgarian", "choices" : [ "Arabic", "Bulgarian", "Hebrew", "Spanish" ] } ], [ 2, { "guess" : "Arabic", "choices" : [ "Arabic", "Cantonese", "Maltese", "Samoan" ] }, { "guess" : "Arabic", "choices" : [ "Albanian", "Arabic", "Czech", "Dutch" ] } ])" :) +(:JIQS: ShouldRun; Output="([ 1, { "guess" : "Arabic", "choices" : [ "Arabic", "Cantonese", "Maltese", "Samoan" ] }, { "guess" : "Arabic", "choices" : [ "Albanian", "Arabic", "Czech", "Dutch" ] } ], [ 2, { "guess" : "Bulgarian", "choices" : [ "Albanian", "Bulgarian", "Russian", "Ukrainian" ] }, { "guess" : "Bulgarian", "choices" : [ "Arabic", "Bulgarian", "Hebrew", "Spanish" ] } ])" :) for $i in parallelize(( {"guess": "Arabic", "choices": ["Arabic", "Cantonese", "Maltese", "Samoan"]}, {"guess": "Arabic", "choices": ["Albanian", "Arabic", "Czech", "Dutch"]}, {"guess": "Bulgarian", "choices": ["Albanian", "Bulgarian", "Russian", "Ukrainian"]}, {"guess": "Bulgarian", "choices": ["Arabic", "Bulgarian", "Hebrew", "Spanish"]})) group by $guess := $i.guess +order by $guess count $group_index return [$group_index, $i] From 1803a5c5bbb9b9006d6631353782208de7c9d750 Mon Sep 17 00:00:00 2001 From: Ghislain Fourny Date: Thu, 21 Dec 2023 11:28:33 +0100 Subject: [PATCH 09/20] Fix tests. --- src/test/resources/test_files/RumbleML/RumbleML/MLPipeline1.jq | 2 +- src/test/resources/test_files/RumbleML/RumbleML/MLPipeline2.jq | 2 +- src/test/resources/test_files/RumbleML/RumbleML/MLPipeline3.jq | 2 +- src/test/resources/test_files/RumbleML/RumbleML/MLPipeline4.jq | 2 +- src/test/resources/test_files/RumbleML/RumbleML/MLPipeline5.jq | 2 +- src/test/resources/test_files/RumbleML/RumbleML/MLPipeline6.jq | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline1.jq b/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline1.jq index 2979627829..d39162a15b 100644 --- a/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline1.jq +++ b/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline1.jq @@ -1,4 +1,4 @@ -(:JIQS: ShouldRun; Output="({ "id" : 0, "label" : 1, "col1" : -1, "col2" : 1.5, "col3" : 1.3, "features" : [ -1, 1.5, 1.3 ], "features2" : [ 0.6811443247067493, -1.6574062675838126 ], "rawPrediction" : [ -37.806767087841095, 37.806767087841095 ], "probability" : [ 3.80828704311553E-17, 1 ], "ocol1" : 1 }, { "id" : 1, "label" : 0, "col1" : 3, "col2" : 2, "col3" : -0.1, "features" : [ 3, 2, -0.1 ], "features2" : [ -2.9389857055027706, 0.48441694826197546 ], "rawPrediction" : [ 37.56770709258579, -37.56770709258579 ], "probability" : [ 1, 0 ], "ocol1" : 0 }, { "id" : 2, "label" : 1, "col1" : 0, "col2" : 2.2, "col3" : -1.5, "features" : [ 0, 2.2, -1.5 ], "features2" : [ 0.28609394877170796, 1.1822492465709793 ], "rawPrediction" : [ -19.168353441006694, 19.168353441006694 ], "probability" : [ 4.734671708106025E-9, 0.9999999952653283 ], "ocol1" : 1 })" :) +(:JIQS: ShouldRun; Output="({ "id" : 0, "label" : 1, "col1" : -1, "col2" : 1.5, "col3" : 1.3, "features" : [ -1, 1.5, 1.3 ], "features2" : [ 0.6811443247067493, -1.657406267583814 ], "rawPrediction" : [ -37.806767087841045, 37.806767087841045 ], "probability" : [ 3.808287043115719E-17, 1 ], "ocol1" : 1 }, { "id" : 1, "label" : 0, "col1" : 3, "col2" : 2, "col3" : -0.1, "features" : [ 3, 2, -0.1 ], "features2" : [ -2.9389857055027706, 0.48441694826197507 ], "rawPrediction" : [ 37.56770709258576, -37.56770709258576 ], "probability" : [ 1, 0 ], "ocol1" : 0 }, { "id" : 2, "label" : 1, "col1" : 0, "col2" : 2.2, "col3" : -1.5, "features" : [ 0, 2.2, -1.5 ], "features2" : [ 0.28609394877170796, 1.1822492465709795 ], "rawPrediction" : [ -19.168353441006655, 19.168353441006655 ], "probability" : [ 4.73467170810621E-9, 0.9999999952653283 ], "ocol1" : 1 })" :) let $vector-assembler := get-transformer("VectorAssembler") let $training-data := ( {"id": 0, "label": 1, "col1": 0.0, "col2": 1.1, "col3": 0.1}, diff --git a/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline2.jq b/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline2.jq index 57b2c3bc09..1c45b26953 100644 --- a/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline2.jq +++ b/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline2.jq @@ -1,4 +1,4 @@ -(:JIQS: ShouldRun; Output="({ "id" : 0, "label" : 1, "col1" : -1, "col2" : 1.5, "col3" : 1.3, "features" : [ -1, 1.5, 1.3 ], "features2" : [ 0.6811443247067493, -1.6574062675838126 ], "rawPrediction" : [ -37.806767087841095, 37.806767087841095 ], "probability" : [ 3.80828704311553E-17, 1 ], "ocol1" : 1 }, { "id" : 1, "label" : 0, "col1" : 3, "col2" : 2, "col3" : -0.1, "features" : [ 3, 2, -0.1 ], "features2" : [ -2.9389857055027706, 0.48441694826197546 ], "rawPrediction" : [ 37.56770709258579, -37.56770709258579 ], "probability" : [ 1, 0 ], "ocol1" : 0 }, { "id" : 2, "label" : 1, "col1" : 0, "col2" : 2.2, "col3" : -1.5, "features" : [ 0, 2.2, -1.5 ], "features2" : [ 0.28609394877170796, 1.1822492465709793 ], "rawPrediction" : [ -19.168353441006694, 19.168353441006694 ], "probability" : [ 4.734671708106025E-9, 0.9999999952653283 ], "ocol1" : 1 })" :) +(:JIQS: ShouldRun; Output="({ "id" : 0, "label" : 1, "col1" : -1, "col2" : 1.5, "col3" : 1.3, "features" : [ -1, 1.5, 1.3 ], "features2" : [ 0.6811443247067493, -1.6574062675838137 ], "rawPrediction" : [ -37.806767087840996, 37.806767087840996 ], "probability" : [ 3.808287043115909E-17, 1 ], "ocol1" : 1 }, { "id" : 1, "label" : 0, "col1" : 3, "col2" : 2, "col3" : -0.1, "features" : [ 3, 2, -0.1 ], "features2" : [ -2.9389857055027706, 0.48441694826197507 ], "rawPrediction" : [ 37.567707092585735, -37.567707092585735 ], "probability" : [ 1, 0 ], "ocol1" : 0 }, { "id" : 2, "label" : 1, "col1" : 0, "col2" : 2.2, "col3" : -1.5, "features" : [ 0, 2.2, -1.5 ], "features2" : [ 0.28609394877170796, 1.1822492465709793 ], "rawPrediction" : [ -19.168353441006627, 19.168353441006627 ], "probability" : [ 4.734671708106345E-9, 0.9999999952653283 ], "ocol1" : 1 })" :) let $vector-assembler := get-transformer("VectorAssembler")(?, { "inputCols" : [ "col1", "col2", "col3" ], "outputCol" : "features" }) let $training-data := ( {"id": 0, "label": 1, "col1": 0.0, "col2": 1.1, "col3": 0.1}, diff --git a/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline3.jq b/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline3.jq index 37e99a3949..98837bce5b 100644 --- a/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline3.jq +++ b/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline3.jq @@ -1,4 +1,4 @@ -(:JIQS: ShouldRun; Output="({ "id" : 0, "label" : 1, "col1" : -1, "col2" : 1.5, "col3" : 1.3, "features" : [ -1, 1.5, 1.3 ], "features2" : [ 0.6811443247067493, -1.6574062675838126 ], "rawPrediction" : [ -37.806767087841095, 37.806767087841095 ], "probability" : [ 3.80828704311553E-17, 1 ], "ocol1" : 1 }, { "id" : 1, "label" : 0, "col1" : 3, "col2" : 2, "col3" : -0.1, "features" : [ 3, 2, -0.1 ], "features2" : [ -2.9389857055027706, 0.48441694826197546 ], "rawPrediction" : [ 37.56770709258579, -37.56770709258579 ], "probability" : [ 1, 0 ], "ocol1" : 0 }, { "id" : 2, "label" : 1, "col1" : 0, "col2" : 2.2, "col3" : -1.5, "features" : [ 0, 2.2, -1.5 ], "features2" : [ 0.28609394877170796, 1.1822492465709793 ], "rawPrediction" : [ -19.168353441006694, 19.168353441006694 ], "probability" : [ 4.734671708106025E-9, 0.9999999952653283 ], "ocol1" : 1 })" :) +(:JIQS: ShouldRun; Output="({ "id" : 0, "label" : 1, "col1" : -1, "col2" : 1.5, "col3" : 1.3, "features" : [ -1, 1.5, 1.3 ], "features2" : [ 0.6811443247067493, -1.6574062675838128 ], "rawPrediction" : [ -37.806767087841024, 37.806767087841024 ], "probability" : [ 3.8082870431158006E-17, 1 ], "ocol1" : 1 }, { "id" : 1, "label" : 0, "col1" : 3, "col2" : 2, "col3" : -0.1, "features" : [ 3, 2, -0.1 ], "features2" : [ -2.9389857055027706, 0.48441694826197546 ], "rawPrediction" : [ 37.56770709258576, -37.56770709258576 ], "probability" : [ 1, 0 ], "ocol1" : 0 }, { "id" : 2, "label" : 1, "col1" : 0, "col2" : 2.2, "col3" : -1.5, "features" : [ 0, 2.2, -1.5 ], "features2" : [ 0.28609394877170796, 1.1822492465709793 ], "rawPrediction" : [ -19.168353441006637, 19.168353441006637 ], "probability" : [ 4.734671708106294E-9, 0.9999999952653283 ], "ocol1" : 1 })" :) declare type local:mytype as { "id": "integer", "label": "integer", "col1": "decimal", "col2": "decimal", "col3": "decimal" diff --git a/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline4.jq b/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline4.jq index ab9d09ab98..0d2e943f42 100644 --- a/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline4.jq +++ b/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline4.jq @@ -1,4 +1,4 @@ -(:JIQS: ShouldRun; Output="({ "id" : 0, "label" : 1, "col1" : -1, "col2" : 1.5, "col3" : 1.3, "features" : [ -1, 1.5, 1.3 ], "rawPrediction" : [ -38.071372676778395, 38.071372676778395 ], "probability" : [ 2.9228930727443083E-17, 1 ], "prediction" : 1 }, { "id" : 1, "label" : 0, "col1" : 3, "col2" : 2, "col3" : -0.1, "features" : [ 3, 2, -0.1 ], "rawPrediction" : [ 36.29582598188425, -36.29582598188425 ], "probability" : [ 0.9999999999999998, 2.220446049250313E-16 ], "prediction" : 0 }, { "id" : 2, "label" : 1, "col1" : 0, "col2" : 2.2, "col3" : -1.5, "features" : [ 0, 2.2, -1.5 ], "rawPrediction" : [ -21.188416917061183, 21.188416917061183 ], "probability" : [ 6.280402132397163E-10, 0.9999999993719598 ], "prediction" : 1 })" :) +(:JIQS: ShouldRun; Output="({ "id" : 0, "label" : 1, "col1" : -1, "col2" : 1.5, "col3" : 1.3, "features" : [ -1, 1.5, 1.3 ], "rawPrediction" : [ -38.071372676780044, 38.071372676780044 ], "probability" : [ 2.9228930727394895E-17, 1 ], "prediction" : 1 }, { "id" : 1, "label" : 0, "col1" : 3, "col2" : 2, "col3" : -0.1, "features" : [ 3, 2, -0.1 ], "rawPrediction" : [ 36.295825981882075, -36.295825981882075 ], "probability" : [ 0.9999999999999998, 2.220446049250313E-16 ], "prediction" : 0 }, { "id" : 2, "label" : 1, "col1" : 0, "col2" : 2.2, "col3" : -1.5, "features" : [ 0, 2.2, -1.5 ], "rawPrediction" : [ -21.18841691706346, 21.18841691706346 ], "probability" : [ 6.280402132382861E-10, 0.9999999993719598 ], "prediction" : 1 })" :) declare type local:mytype as {"id": "integer", "label": "integer", "col1": "decimal", "col2": "decimal", "col3": "decimal"}; diff --git a/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline5.jq b/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline5.jq index 679ff7ac19..b43c79b861 100644 --- a/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline5.jq +++ b/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline5.jq @@ -1,4 +1,4 @@ -(:JIQS: ShouldRun; Output="({ "id" : 0, "label" : 1, "col1" : -1, "col2" : 1.5, "col3" : 1.3, "features" : [ -1, 1.5, 1.3 ], "rawPrediction" : [ -38.071372676778395, 38.071372676778395 ], "probability" : [ 2.9228930727443083E-17, 1 ], "prediction" : 1 }, { "id" : 1, "label" : 0, "col1" : 3, "col2" : 2, "col3" : -0.1, "features" : [ 3, 2, -0.1 ], "rawPrediction" : [ 36.29582598188425, -36.29582598188425 ], "probability" : [ 0.9999999999999998, 2.220446049250313E-16 ], "prediction" : 0 }, { "id" : 2, "label" : 1, "col1" : 0, "col2" : 2.2, "col3" : -1.5, "features" : [ 0, 2.2, -1.5 ], "rawPrediction" : [ -21.188416917061183, 21.188416917061183 ], "probability" : [ 6.280402132397163E-10, 0.9999999993719598 ], "prediction" : 1 })" :) +(:JIQS: ShouldRun; Output="({ "id" : 0, "label" : 1, "col1" : -1, "col2" : 1.5, "col3" : 1.3, "features" : [ -1, 1.5, 1.3 ], "rawPrediction" : [ -38.07137267677621, 38.07137267677621 ], "probability" : [ 2.922893072750705E-17, 1 ], "prediction" : 1 }, { "id" : 1, "label" : 0, "col1" : 3, "col2" : 2, "col3" : -0.1, "features" : [ 3, 2, -0.1 ], "rawPrediction" : [ 36.29582598188709, -36.29582598188709 ], "probability" : [ 0.9999999999999998, 2.220446049250313E-16 ], "prediction" : 0 }, { "id" : 2, "label" : 1, "col1" : 0, "col2" : 2.2, "col3" : -1.5, "features" : [ 0, 2.2, -1.5 ], "rawPrediction" : [ -21.188416917058074, 21.188416917058074 ], "probability" : [ 6.280402132416686E-10, 0.9999999993719598 ], "prediction" : 1 })" :) declare type local:mytype as {"id": "integer", "label": "integer", "col1": "decimal", "col2": "decimal", "col3": "decimal"}; let $vector-assembler := get-transformer("VectorAssembler", {"inputCols" : [ "col1", "col2", "col3" ], "outputCol" : "features"}) diff --git a/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline6.jq b/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline6.jq index 69b981b468..b60117e615 100644 --- a/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline6.jq +++ b/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline6.jq @@ -1,4 +1,4 @@ -(:JIQS: ShouldRun; Output="({ "id" : 0, "label" : 1, "col1" : -1, "col2" : 1.5, "col3" : 1.3, "features" : [ -1, 1.5, 1.3 ], "rawPrediction" : [ -38.071372676778395, 38.071372676778395 ], "probability" : [ 2.9228930727443083E-17, 1 ], "prediction" : 1 }, { "id" : 1, "label" : 0, "col1" : 3, "col2" : 2, "col3" : -0.1, "features" : [ 3, 2, -0.1 ], "rawPrediction" : [ 36.29582598188425, -36.29582598188425 ], "probability" : [ 0.9999999999999998, 2.220446049250313E-16 ], "prediction" : 0 }, { "id" : 2, "label" : 1, "col1" : 0, "col2" : 2.2, "col3" : -1.5, "features" : [ 0, 2.2, -1.5 ], "rawPrediction" : [ -21.188416917061183, 21.188416917061183 ], "probability" : [ 6.280402132397163E-10, 0.9999999993719598 ], "prediction" : 1 })" :) +(:JIQS: ShouldRun; Output="({ "id" : 0, "label" : 1, "col1" : -1, "col2" : 1.5, "col3" : 1.3, "features" : [ -1, 1.5, 1.3 ], "rawPrediction" : [ -38.07137267677527, 38.07137267677527 ], "probability" : [ 2.922893072753446E-17, 1 ], "prediction" : 1 }, { "id" : 1, "label" : 0, "col1" : 3, "col2" : 2, "col3" : -0.1, "features" : [ 3, 2, -0.1 ], "rawPrediction" : [ 36.295825981888406, -36.295825981888406 ], "probability" : [ 0.9999999999999998, 2.220446049250313E-16 ], "prediction" : 0 }, { "id" : 2, "label" : 1, "col1" : 0, "col2" : 2.2, "col3" : -1.5, "features" : [ 0, 2.2, -1.5 ], "rawPrediction" : [ -21.188416917056745, 21.188416917056745 ], "probability" : [ 6.280402132425032E-10, 0.9999999993719598 ], "prediction" : 1 })" :) declare type local:mytype as {"id": "integer", "label": "integer", "col1": "decimal", "col2": "decimal", "col3": "decimal"}; let $pipeline := get-estimator("Pipeline", { From 1d1d9f667836b335917d37ef3478bec9f16a3cd5 Mon Sep 17 00:00:00 2001 From: Ghislain Fourny Date: Thu, 21 Dec 2023 12:04:11 +0100 Subject: [PATCH 10/20] Fix tests. --- src/test/resources/test_files/RumbleML/RumbleML/MLPipeline2.jq | 2 +- src/test/resources/test_files/RumbleML/RumbleML/MLPipeline5.jq | 2 +- src/test/resources/test_files/RumbleML/RumbleML/MLPipeline6.jq | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline2.jq b/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline2.jq index 1c45b26953..df2dc8f33d 100644 --- a/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline2.jq +++ b/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline2.jq @@ -1,4 +1,4 @@ -(:JIQS: ShouldRun; Output="({ "id" : 0, "label" : 1, "col1" : -1, "col2" : 1.5, "col3" : 1.3, "features" : [ -1, 1.5, 1.3 ], "features2" : [ 0.6811443247067493, -1.6574062675838137 ], "rawPrediction" : [ -37.806767087840996, 37.806767087840996 ], "probability" : [ 3.808287043115909E-17, 1 ], "ocol1" : 1 }, { "id" : 1, "label" : 0, "col1" : 3, "col2" : 2, "col3" : -0.1, "features" : [ 3, 2, -0.1 ], "features2" : [ -2.9389857055027706, 0.48441694826197507 ], "rawPrediction" : [ 37.567707092585735, -37.567707092585735 ], "probability" : [ 1, 0 ], "ocol1" : 0 }, { "id" : 2, "label" : 1, "col1" : 0, "col2" : 2.2, "col3" : -1.5, "features" : [ 0, 2.2, -1.5 ], "features2" : [ 0.28609394877170796, 1.1822492465709793 ], "rawPrediction" : [ -19.168353441006627, 19.168353441006627 ], "probability" : [ 4.734671708106345E-9, 0.9999999952653283 ], "ocol1" : 1 })" :) +\1;95;0c(:JIQS: ShouldRun; Output="({ "id" : 0, "label" : 1, "col1" : -1, "col2" : 1.5, "col3" : 1.3, "features" : [ -1, 1.5, 1.3 ], "features2" : [ 0.6811443247067493, -1.6574062675838137 ], "rawPrediction" : [ -37.806767087840996, 37.806767087840996 ], "probability" : [ 3.808287043115909E-17, 1 ], "ocol1" : 1 }, { "id" : 1, "label" : 0, "col1" : 3, "col2" : 2, "col3" : -0.1, "features" : [ 3, 2, -0.1 ], "features2" : [ -2.9389857055027706, 0.48441694826197507 ], "rawPrediction" : [ 37.567707092585735, -37.567707092585735 ], "probability" : [ 1, 0 ], "ocol1" : 0 }, { "id" : 2, "label" : 1, "col1" : 0, "col2" : 2.2, "col3" : -1.5, "features" : [ 0, 2.2, -1.5 ], "features2" : [ 0.28609394877170796, 1.1822492465709793 ], "rawPrediction" : [ -19.168353441006627, 19.168353441006627 ], "probability" : [ 4.734671708106345E-9, 0.9999999952653283 ], "ocol1" : 1 })" :) let $vector-assembler := get-transformer("VectorAssembler")(?, { "inputCols" : [ "col1", "col2", "col3" ], "outputCol" : "features" }) let $training-data := ( {"id": 0, "label": 1, "col1": 0.0, "col2": 1.1, "col3": 0.1}, diff --git a/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline5.jq b/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline5.jq index b43c79b861..7532e29662 100644 --- a/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline5.jq +++ b/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline5.jq @@ -1,4 +1,4 @@ -(:JIQS: ShouldRun; Output="({ "id" : 0, "label" : 1, "col1" : -1, "col2" : 1.5, "col3" : 1.3, "features" : [ -1, 1.5, 1.3 ], "rawPrediction" : [ -38.07137267677621, 38.07137267677621 ], "probability" : [ 2.922893072750705E-17, 1 ], "prediction" : 1 }, { "id" : 1, "label" : 0, "col1" : 3, "col2" : 2, "col3" : -0.1, "features" : [ 3, 2, -0.1 ], "rawPrediction" : [ 36.29582598188709, -36.29582598188709 ], "probability" : [ 0.9999999999999998, 2.220446049250313E-16 ], "prediction" : 0 }, { "id" : 2, "label" : 1, "col1" : 0, "col2" : 2.2, "col3" : -1.5, "features" : [ 0, 2.2, -1.5 ], "rawPrediction" : [ -21.188416917058074, 21.188416917058074 ], "probability" : [ 6.280402132416686E-10, 0.9999999993719598 ], "prediction" : 1 })" :) +(:JIQS: ShouldRun; Output="({ "id" : 0, "label" : 1, "col1" : -1, "col2" : 1.5, "col3" : 1.3, "features" : [ -1, 1.5, 1.3 ], "rawPrediction" : [ -38.07137267677909, 38.07137267677909 ], "probability" : [ 2.922893072742273E-17, 1 ], "prediction" : 1 }, { "id" : 1, "label" : 0, "col1" : 3, "col2" : 2, "col3" : -0.1, "features" : [ 3, 2, -0.1 ], "rawPrediction" : [ 36.29582598188342, -36.29582598188342 ], "probability" : [ 0.9999999999999998, 2.220446049250313E-16 ], "prediction" : 0 }, { "id" : 2, "label" : 1, "col1" : 0, "col2" : 2.2, "col3" : -1.5, "features" : [ 0, 2.2, -1.5 ], "rawPrediction" : [ -21.188416917062117, 21.188416917062117 ], "probability" : [ 6.280402132391294E-10, 0.9999999993719598 ], "prediction" : 1 })" :) declare type local:mytype as {"id": "integer", "label": "integer", "col1": "decimal", "col2": "decimal", "col3": "decimal"}; let $vector-assembler := get-transformer("VectorAssembler", {"inputCols" : [ "col1", "col2", "col3" ], "outputCol" : "features"}) diff --git a/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline6.jq b/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline6.jq index b60117e615..8e81c5d0b3 100644 --- a/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline6.jq +++ b/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline6.jq @@ -1,4 +1,4 @@ -(:JIQS: ShouldRun; Output="({ "id" : 0, "label" : 1, "col1" : -1, "col2" : 1.5, "col3" : 1.3, "features" : [ -1, 1.5, 1.3 ], "rawPrediction" : [ -38.07137267677527, 38.07137267677527 ], "probability" : [ 2.922893072753446E-17, 1 ], "prediction" : 1 }, { "id" : 1, "label" : 0, "col1" : 3, "col2" : 2, "col3" : -0.1, "features" : [ 3, 2, -0.1 ], "rawPrediction" : [ 36.295825981888406, -36.295825981888406 ], "probability" : [ 0.9999999999999998, 2.220446049250313E-16 ], "prediction" : 0 }, { "id" : 2, "label" : 1, "col1" : 0, "col2" : 2.2, "col3" : -1.5, "features" : [ 0, 2.2, -1.5 ], "rawPrediction" : [ -21.188416917056745, 21.188416917056745 ], "probability" : [ 6.280402132425032E-10, 0.9999999993719598 ], "prediction" : 1 })" :) +(:JIQS: ShouldRun; Output="({ "id" : 0, "label" : 1, "col1" : -1, "col2" : 1.5, "col3" : 1.3, "features" : [ -1, 1.5, 1.3 ], "rawPrediction" : [ -38.071372676775525, 38.071372676775525 ], "probability" : [ 2.9228930727526986E-17, 1 ], "prediction" : 1 }, { "id" : 1, "label" : 0, "col1" : 3, "col2" : 2, "col3" : -0.1, "features" : [ 3, 2, -0.1 ], "rawPrediction" : [ 36.295825981888, -36.295825981888 ], "probability" : [ 0.9999999999999998, 2.220446049250313E-16 ], "prediction" : 0 }, { "id" : 2, "label" : 1, "col1" : 0, "col2" : 2.2, "col3" : -1.5, "features" : [ 0, 2.2, -1.5 ], "rawPrediction" : [ -21.18841691705713, 21.18841691705713 ], "probability" : [ 6.280402132422621E-10, 0.9999999993719598 ], "prediction" : 1 })" :) declare type local:mytype as {"id": "integer", "label": "integer", "col1": "decimal", "col2": "decimal", "col3": "decimal"}; let $pipeline := get-estimator("Pipeline", { From 524667d5fc2ca9e85bb31eaac6b34beb2528fb1a Mon Sep 17 00:00:00 2001 From: Ghislain Fourny Date: Thu, 21 Dec 2023 12:05:08 +0100 Subject: [PATCH 11/20] Fix tests. --- src/test/resources/test_files/RumbleML/RumbleML/MLPipeline1.jq | 2 +- src/test/resources/test_files/RumbleML/RumbleML/MLPipeline4.jq | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline1.jq b/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline1.jq index d39162a15b..ecc669443a 100644 --- a/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline1.jq +++ b/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline1.jq @@ -1,4 +1,4 @@ -(:JIQS: ShouldRun; Output="({ "id" : 0, "label" : 1, "col1" : -1, "col2" : 1.5, "col3" : 1.3, "features" : [ -1, 1.5, 1.3 ], "features2" : [ 0.6811443247067493, -1.657406267583814 ], "rawPrediction" : [ -37.806767087841045, 37.806767087841045 ], "probability" : [ 3.808287043115719E-17, 1 ], "ocol1" : 1 }, { "id" : 1, "label" : 0, "col1" : 3, "col2" : 2, "col3" : -0.1, "features" : [ 3, 2, -0.1 ], "features2" : [ -2.9389857055027706, 0.48441694826197507 ], "rawPrediction" : [ 37.56770709258576, -37.56770709258576 ], "probability" : [ 1, 0 ], "ocol1" : 0 }, { "id" : 2, "label" : 1, "col1" : 0, "col2" : 2.2, "col3" : -1.5, "features" : [ 0, 2.2, -1.5 ], "features2" : [ 0.28609394877170796, 1.1822492465709795 ], "rawPrediction" : [ -19.168353441006655, 19.168353441006655 ], "probability" : [ 4.73467170810621E-9, 0.9999999952653283 ], "ocol1" : 1 })" :) +(:JIQS: ShouldRun; Output="({ "id" : 0, "label" : 1, "col1" : -1, "col2" : 1.5, "col3" : 1.3, "features" : [ -1, 1.5, 1.3 ], "features2" : [ 0.6811443247067493, -1.6574062675838137 ], "rawPrediction" : [ -37.80676708784381, 37.80676708784381 ], "probability" : [ 3.8082870431051935E-17, 1 ], "ocol1" : 1 }, { "id" : 1, "label" : 0, "col1" : 3, "col2" : 2, "col3" : -0.1, "features" : [ 3, 2, -0.1 ], "features2" : [ -2.9389857055027706, 0.48441694826197507 ], "rawPrediction" : [ 37.567707092586666, -37.567707092586666 ], "probability" : [ 1, 0 ], "ocol1" : 0 }, { "id" : 2, "label" : 1, "col1" : 0, "col2" : 2.2, "col3" : -1.5, "features" : [ 0, 2.2, -1.5 ], "features2" : [ 0.28609394877170796, 1.1822492465709793 ], "rawPrediction" : [ -19.168353441008513, 19.168353441008513 ], "probability" : [ 4.734671708097413E-9, 0.9999999952653283 ], "ocol1" : 1 })" :) let $vector-assembler := get-transformer("VectorAssembler") let $training-data := ( {"id": 0, "label": 1, "col1": 0.0, "col2": 1.1, "col3": 0.1}, diff --git a/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline4.jq b/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline4.jq index 0d2e943f42..9e2037bf60 100644 --- a/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline4.jq +++ b/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline4.jq @@ -1,4 +1,4 @@ -(:JIQS: ShouldRun; Output="({ "id" : 0, "label" : 1, "col1" : -1, "col2" : 1.5, "col3" : 1.3, "features" : [ -1, 1.5, 1.3 ], "rawPrediction" : [ -38.071372676780044, 38.071372676780044 ], "probability" : [ 2.9228930727394895E-17, 1 ], "prediction" : 1 }, { "id" : 1, "label" : 0, "col1" : 3, "col2" : 2, "col3" : -0.1, "features" : [ 3, 2, -0.1 ], "rawPrediction" : [ 36.295825981882075, -36.295825981882075 ], "probability" : [ 0.9999999999999998, 2.220446049250313E-16 ], "prediction" : 0 }, { "id" : 2, "label" : 1, "col1" : 0, "col2" : 2.2, "col3" : -1.5, "features" : [ 0, 2.2, -1.5 ], "rawPrediction" : [ -21.18841691706346, 21.18841691706346 ], "probability" : [ 6.280402132382861E-10, 0.9999999993719598 ], "prediction" : 1 })" :) +(:JIQS: ShouldRun; Output="({ "id" : 0, "label" : 1, "col1" : -1, "col2" : 1.5, "col3" : 1.3, "features" : [ -1, 1.5, 1.3 ], "rawPrediction" : [ -38.071372676777095, 38.071372676777095 ], "probability" : [ 2.9228930727481084E-17, 1 ], "prediction" : 1 }, { "id" : 1, "label" : 0, "col1" : 3, "col2" : 2, "col3" : -0.1, "features" : [ 3, 2, -0.1 ], "rawPrediction" : [ 36.2958259818859, -36.2958259818859 ], "probability" : [ 0.9999999999999998, 2.220446049250313E-16 ], "prediction" : 0 }, { "id" : 2, "label" : 1, "col1" : 0, "col2" : 2.2, "col3" : -1.5, "features" : [ 0, 2.2, -1.5 ], "rawPrediction" : [ -21.18841691705933, 21.18841691705933 ], "probability" : [ 6.280402132408788E-10, 0.9999999993719598 ], "prediction" : 1 })" :) declare type local:mytype as {"id": "integer", "label": "integer", "col1": "decimal", "col2": "decimal", "col3": "decimal"}; From c52ff1f1ede0d015773e20042ffaf05b52a4107b Mon Sep 17 00:00:00 2001 From: Ghislain Fourny Date: Tue, 27 Feb 2024 17:07:07 +0100 Subject: [PATCH 12/20] Fix test. --- .../resources/test_files/RumbleML/RumbleML/MLPipeline3.jq | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline3.jq b/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline3.jq index 047744ce8a..43aefcb264 100644 --- a/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline3.jq +++ b/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline3.jq @@ -1,4 +1,4 @@ -(:JIQS: ShouldRun; Output="({ "id" : 0, "label" : 1, "col1" : -1, "col2" : 1.5, "col3" : 1.3, "features" : [ -1, 1.5, 1.3 ], "features2" : [ 0.6811443247067493, -1.6574062675838128 ], "rawPrediction" : [ -37.806767087841024, 37.806767087841024 ], "probability" : [ 3.8082870431158006E-17, 1 ], "ocol1" : 1 }, { "id" : 1, "label" : 0, "col1" : 3, "col2" : 2, "col3" : -0.1, "features" : [ 3, 2, -0.1 ], "features2" : [ -2.9389857055027706, 0.48441694826197546 ], "rawPrediction" : [ 37.56770709258576, -37.56770709258576 ], "probability" : [ 1, 0 ], "ocol1" : 0 }, { "id" : 2, "label" : 1, "col1" : 0, "col2" : 2.2, "col3" : -1.5, "features" : [ 0, 2.2, -1.5 ], "features2" : [ 0.28609394877170796, 1.1822492465709793 ], "rawPrediction" : [ -19.168353441006637, 19.168353441006637 ], "probability" : [ 4.734671708106294E-9, 0.9999999952653283 ], "ocol1" : 1 })" :) +(:JIQS: ShouldRun; Output="({ "id" : 0, "label" : 1, "col1" : -1, "col2" : 1.5, "col3" : 1.3, "features" : [ -1, 1.5, 1.3 ], "features2" : [ 0.6811443247067492, -1.6574062675838126 ], "rawPrediction" : [ -37.806767087840996, 37.806767087840996 ], "probability" : [ 3.808287043115909E-17, 1 ], "ocol1" : 1 }, { "id" : 1, "label" : 0, "col1" : 3, "col2" : 2, "col3" : -0.1, "features" : [ 3, 2, -0.1 ], "features2" : [ -2.93898570550277, 0.48441694826197534 ], "rawPrediction" : [ 37.56770709258572, -37.56770709258572 ], "probability" : [ 1, 0 ], "ocol1" : 0 }, { "id" : 2, "label" : 1, "col1" : 0, "col2" : 2.2, "col3" : -1.5, "features" : [ 0, 2.2, -1.5 ], "features2" : [ 0.28609394877170796, 1.1822492465709793 ], "rawPrediction" : [ -19.16835344100661, 19.16835344100661 ], "probability" : [ 4.734671708106428E-9, 0.9999999952653283 ], "ocol1" : 1 })" :) declare type local:mytype as { "id": "integer", "label": "integer", "col1": "decimal", "col2": "decimal", "col3": "decimal" @@ -33,4 +33,4 @@ for $i in $trained_est2( $my-new-test-data, {"featuresCol": "features2", "predictionCol": "ocol1"} ) -return local:round($i) \ No newline at end of file +return $i \ No newline at end of file From 4aef76912649b3147652e5babe048beaef456a77 Mon Sep 17 00:00:00 2001 From: Ghislain Fourny Date: Tue, 27 Feb 2024 17:08:06 +0100 Subject: [PATCH 13/20] Fix test. --- .../test_files/RumbleML/RumbleML/MLPipeline3.jq | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline3.jq b/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline3.jq index 43aefcb264..2b1ac52fdf 100644 --- a/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline3.jq +++ b/src/test/resources/test_files/RumbleML/RumbleML/MLPipeline3.jq @@ -1,4 +1,16 @@ -(:JIQS: ShouldRun; Output="({ "id" : 0, "label" : 1, "col1" : -1, "col2" : 1.5, "col3" : 1.3, "features" : [ -1, 1.5, 1.3 ], "features2" : [ 0.6811443247067492, -1.6574062675838126 ], "rawPrediction" : [ -37.806767087840996, 37.806767087840996 ], "probability" : [ 3.808287043115909E-17, 1 ], "ocol1" : 1 }, { "id" : 1, "label" : 0, "col1" : 3, "col2" : 2, "col3" : -0.1, "features" : [ 3, 2, -0.1 ], "features2" : [ -2.93898570550277, 0.48441694826197534 ], "rawPrediction" : [ 37.56770709258572, -37.56770709258572 ], "probability" : [ 1, 0 ], "ocol1" : 0 }, { "id" : 2, "label" : 1, "col1" : 0, "col2" : 2.2, "col3" : -1.5, "features" : [ 0, 2.2, -1.5 ], "features2" : [ 0.28609394877170796, 1.1822492465709793 ], "rawPrediction" : [ -19.16835344100661, 19.16835344100661 ], "probability" : [ 4.734671708106428E-9, 0.9999999952653283 ], "ocol1" : 1 })" :) +(:JIQS: ShouldRun; Output="({ "id" : 0, "label" : 1, "col1" : -1, "col2" : 1.5, "col3" : 1.3, "features" : [ -1, 1.5, 1.3 ], "ocol1" : 1, "features2" : [ 0.6811443, -1.6574062 ], "rawPrediction" : [ -37.806767, 37.806767 ], "probability" : [ 3.808287E-17, 1 ] }, { "id" : 1, "label" : 0, "col1" : 3, "col2" : 2, "col3" : -0.1, "features" : [ 3, 2, -0.1 ], "ocol1" : 0, "features2" : [ -2.9389858, 0.48441696 ], "rawPrediction" : [ 37.567707, -37.567707 ], "probability" : [ 1, 0 ] }, { "id" : 2, "label" : 1, "col1" : 0, "col2" : 2.2, "col3" : -1.5, "features" : [ 0, 2.2, -1.5 ], "ocol1" : 1, "features2" : [ 0.28609395, 1.1822492 ], "rawPrediction" : [ -19.168354, 19.168354 ], "probability" : [ 4.7346718E-9, 1 ] })" :) + +declare function local:round($i as object) as object { + {| + remove-keys($i, ("features2", "rawPrediction", "probability")), + { + "features2" : [ for $v in $i.features2[] return float($v) ], + "rawPrediction" : [ for $v in $i.rawPrediction[] return float($v) ], + "probability" : [ for $v in $i.probability[] return float($v) ] + } + |} +}; + declare type local:mytype as { "id": "integer", "label": "integer", "col1": "decimal", "col2": "decimal", "col3": "decimal" @@ -33,4 +45,4 @@ for $i in $trained_est2( $my-new-test-data, {"featuresCol": "features2", "predictionCol": "ocol1"} ) -return $i \ No newline at end of file +return local:round($i) \ No newline at end of file From c4a0e34c743f3c504fcaf2a8e3111d793c6cc195 Mon Sep 17 00:00:00 2001 From: Ghislain Fourny Date: Wed, 10 Jul 2024 10:42:45 +0200 Subject: [PATCH 14/20] Minor update. --- pom.xml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pom.xml b/pom.xml index 536a9ba68f..53ad33031e 100644 --- a/pom.xml +++ b/pom.xml @@ -200,19 +200,19 @@ org.apache.spark spark-core_2.12 - 3.5.0 + 3.5.1 provided org.apache.spark spark-sql_2.12 - 3.5.0 + 3.5.1 provided org.apache.spark spark-mllib_2.12 - 3.5.0 + 3.5.1 provided @@ -224,7 +224,7 @@ org.apache.spark spark-avro_2.12 - 3.4.0 + 3.5.1 org.antlr From 3ce80c1a69170b784496d9c3d858a081d347ae38 Mon Sep 17 00:00:00 2001 From: Ghislain Fourny Date: Thu, 17 Oct 2024 14:43:11 +0200 Subject: [PATCH 15/20] Update pom.xml --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index d66785f1ea..43b5e3fec3 100644 --- a/pom.xml +++ b/pom.xml @@ -304,8 +304,8 @@ io.delta - delta-core_2.12 - 2.4.0 + delta-spark_2.12 + 3.2.1 From bb5270dcfc60d5f71fed3962245921c911cd2a53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marco=20Sch=C3=B6b?= Date: Fri, 25 Oct 2024 09:50:16 +0200 Subject: [PATCH 16/20] upgrade to scala 2.13 to fix tests --- pom.xml | 72 +++++++++---------- .../rumbledb/items/parsing/ItemParser.java | 16 +++-- .../runtime/flwor/FlworDataFrameUtils.java | 14 ++-- ...upClauseArrayMergeAggregateResultsUDF.java | 25 ++++--- ...oupClauseSerializeAggregateResultsUDF.java | 6 +- src/test/java/iq/Bugs.java | 8 ++- src/test/java/iq/DeltaUpdateRuntimeTests.java | 8 ++- src/test/java/iq/RuntimeTests.java | 11 ++- src/test/java/iq/StaticTypeTests.java | 9 ++- .../java/iq/UpdatesForRumbleBenchmark.java | 9 ++- .../Exponential/FunctionExp.jq | 2 +- 11 files changed, 114 insertions(+), 66 deletions(-) diff --git a/pom.xml b/pom.xml index 43b5e3fec3..ef97610dff 100644 --- a/pom.xml +++ b/pom.xml @@ -209,31 +209,31 @@ org.apache.spark - spark-core_2.12 + spark-core_2.13 3.5.1 provided org.apache.spark - spark-sql_2.12 + spark-sql_2.13 3.5.1 provided org.apache.spark - spark-mllib_2.12 + spark-mllib_2.13 3.5.1 provided org.apache.hadoop - hadoop-aws + hadoop-aws 3.3.6 provided org.apache.spark - spark-avro_2.12 + spark-avro_2.13 3.5.1 @@ -277,37 +277,37 @@ commons-io 2.11.0 - - org.apache.httpcomponents - httpclient - 4.5.13 - - - - org.jgrapht - jgrapht-core - 1.4.0 - - - joda-time - joda-time - 2.10.6 - - - com.fasterxml.jackson.dataformat - jackson-dataformat-yaml - 2.15.2 - - - io.delta - delta-spark_2.12 - 3.2.1 - - + + org.apache.httpcomponents + httpclient + 4.5.13 + + + + org.jgrapht + jgrapht-core + 1.4.0 + + + joda-time + joda-time + 2.10.6 + + + com.fasterxml.jackson.dataformat + jackson-dataformat-yaml + 2.15.2 + + + io.delta + delta-spark_2.13 + 3.2.1 + + diff --git a/src/main/java/org/rumbledb/items/parsing/ItemParser.java b/src/main/java/org/rumbledb/items/parsing/ItemParser.java index 434e9a43b4..be4b8fd93f 100644 --- a/src/main/java/org/rumbledb/items/parsing/ItemParser.java +++ b/src/main/java/org/rumbledb/items/parsing/ItemParser.java @@ -46,7 +46,9 @@ import org.rumbledb.types.BuiltinTypesCatalogue; import org.rumbledb.types.FieldDescriptor; import org.rumbledb.types.ItemType; -import scala.collection.mutable.WrappedArray; +import scala.collection.immutable.ArraySeq; +import scala.collection.Iterator; + import sparksoniq.spark.SparkSessionManager; import java.io.IOException; @@ -589,9 +591,15 @@ private static Item convertValueToItem( } } else { @SuppressWarnings("unchecked") - Object arrayObject = ((WrappedArray) o).array(); - for (int index = 0; index < Array.getLength(arrayObject); index++) { - Object value = Array.get(arrayObject, index); + Iterator iterator = null; + if (o instanceof scala.collection.mutable.ArraySeq) { + iterator = ((scala.collection.mutable.ArraySeq) o).iterator(); + } else { + iterator = ((ArraySeq) o).iterator(); + } + while (iterator.hasNext()) { + Object value = iterator.next(); + members.add(convertValueToItem(value, dataType, metadata, memberType)); } } diff --git a/src/main/java/org/rumbledb/runtime/flwor/FlworDataFrameUtils.java b/src/main/java/org/rumbledb/runtime/flwor/FlworDataFrameUtils.java index 6144043968..b8cfb797ba 100644 --- a/src/main/java/org/rumbledb/runtime/flwor/FlworDataFrameUtils.java +++ b/src/main/java/org/rumbledb/runtime/flwor/FlworDataFrameUtils.java @@ -86,7 +86,8 @@ import com.esotericsoftware.kryo.io.Input; import com.esotericsoftware.kryo.io.Output; -import scala.collection.mutable.WrappedArray; +import scala.collection.immutable.ArraySeq; +import scala.collection.Iterator; import sparksoniq.spark.SparkSessionManager; public class FlworDataFrameUtils { @@ -807,19 +808,20 @@ private static Object deserializeByteArray(byte[] toDeserialize, Kryo kryo, Inpu } public static void deserializeWrappedParameters( - WrappedArray wrappedParameters, + ArraySeq wrappedParameters, List> deserializedParams, Kryo kryo, Input input ) { - Object[] serializedParams = (Object[]) wrappedParameters.array(); - for (Object serializedParam : serializedParams) { - if (serializedParam == null) { + Iterator iterator = wrappedParameters.iterator(); + while (iterator.hasNext()) { + byte[] bytes = iterator.next(); + if (bytes == null) { deserializedParams.add(Collections.emptyList()); continue; } @SuppressWarnings("unchecked") - List deserializedParam = (List) deserializeByteArray((byte[]) serializedParam, kryo, input); + List deserializedParam = (List) deserializeByteArray((byte[]) bytes, kryo, input); deserializedParams.add(deserializedParam); } } diff --git a/src/main/java/org/rumbledb/runtime/flwor/udfs/GroupClauseArrayMergeAggregateResultsUDF.java b/src/main/java/org/rumbledb/runtime/flwor/udfs/GroupClauseArrayMergeAggregateResultsUDF.java index 2054b1519e..e1c3e189fc 100644 --- a/src/main/java/org/rumbledb/runtime/flwor/udfs/GroupClauseArrayMergeAggregateResultsUDF.java +++ b/src/main/java/org/rumbledb/runtime/flwor/udfs/GroupClauseArrayMergeAggregateResultsUDF.java @@ -25,12 +25,14 @@ import org.rumbledb.api.Item; import org.rumbledb.exceptions.OurBadException; -import scala.collection.mutable.WrappedArray; +import scala.collection.immutable.ArraySeq; +import scala.collection.Iterator; + import java.util.ArrayList; import java.util.List; -public class GroupClauseArrayMergeAggregateResultsUDF implements UDF1, Object[]> { +public class GroupClauseArrayMergeAggregateResultsUDF implements UDF1, Object[]> { private static final long serialVersionUID = 1L; @@ -43,22 +45,23 @@ public GroupClauseArrayMergeAggregateResultsUDF() { } @Override - public Object[] call(WrappedArray wrappedParameters) { + public Object[] call(ArraySeq wrappedParameters) { this.nextResult.clear(); this.deserializedParams.clear(); List result = new ArrayList(); - Object[] insideArrays = (Object[]) wrappedParameters.array(); - for (Object o : insideArrays) { + Iterator iterator = wrappedParameters.iterator(); + while (iterator.hasNext()) { + Object o = iterator.next(); if (o instanceof Row) { Row row = (Row) o; result.add(row); } - if (o instanceof WrappedArray) { - @SuppressWarnings("rawtypes") - WrappedArray wrappedArray = (WrappedArray) o; - Object[] insideArrays2 = (Object[]) wrappedArray.array(); - for (Object p : insideArrays2) - result.add(p); + if (o instanceof ArraySeq) { + @SuppressWarnings("unchecked") + ArraySeq arraySeq = (ArraySeq) o; + Iterator iterator2 = arraySeq.iterator(); + while (iterator2.hasNext()) + result.add(iterator2.next()); } else { throw new OurBadException("We cannot process " + o.getClass().getCanonicalName()); } diff --git a/src/main/java/org/rumbledb/runtime/flwor/udfs/GroupClauseSerializeAggregateResultsUDF.java b/src/main/java/org/rumbledb/runtime/flwor/udfs/GroupClauseSerializeAggregateResultsUDF.java index 4bd53e2325..bcbcb0c0c7 100644 --- a/src/main/java/org/rumbledb/runtime/flwor/udfs/GroupClauseSerializeAggregateResultsUDF.java +++ b/src/main/java/org/rumbledb/runtime/flwor/udfs/GroupClauseSerializeAggregateResultsUDF.java @@ -23,12 +23,12 @@ import org.apache.spark.sql.api.java.UDF1; import org.rumbledb.api.Item; import org.rumbledb.runtime.flwor.FlworDataFrameUtils; -import scala.collection.mutable.WrappedArray; +import scala.collection.immutable.ArraySeq; import java.util.ArrayList; import java.util.List; -public class GroupClauseSerializeAggregateResultsUDF implements UDF1, byte[]> { +public class GroupClauseSerializeAggregateResultsUDF implements UDF1, byte[]> { private static final long serialVersionUID = 1L; @@ -43,7 +43,7 @@ public GroupClauseSerializeAggregateResultsUDF() { } @Override - public byte[] call(WrappedArray wrappedParameters) { + public byte[] call(ArraySeq wrappedParameters) { this.nextResult.clear(); this.deserializedParams.clear(); FlworDataFrameUtils.deserializeWrappedParameters( diff --git a/src/test/java/iq/Bugs.java b/src/test/java/iq/Bugs.java index 7049163312..ee2b0f2160 100644 --- a/src/test/java/iq/Bugs.java +++ b/src/test/java/iq/Bugs.java @@ -21,6 +21,7 @@ package iq; import iq.base.AnnotationsTestsBase; +import scala.Function0; import scala.util.Properties; import org.apache.spark.SparkConf; @@ -51,7 +52,12 @@ public class Bugs extends AnnotationsTestsBase { public static final String javaVersion = System.getProperty("java.version"); public static final String scalaVersion = - Properties.scalaPropOrElse("version.number", "unknown"); + Properties.scalaPropOrElse("version.number", new Function0() { + @Override + public String apply() { + return "unknown"; + } + }); protected static List _testFiles = new ArrayList<>(); protected final File testFile; diff --git a/src/test/java/iq/DeltaUpdateRuntimeTests.java b/src/test/java/iq/DeltaUpdateRuntimeTests.java index 7341472bd4..5587ccab75 100644 --- a/src/test/java/iq/DeltaUpdateRuntimeTests.java +++ b/src/test/java/iq/DeltaUpdateRuntimeTests.java @@ -38,6 +38,7 @@ import org.rumbledb.api.Item; import org.rumbledb.api.SequenceOfItems; import scala.util.Properties; +import scala.Function0; import sparksoniq.spark.SparkSessionManager; import utils.FileManager; @@ -58,7 +59,12 @@ public class DeltaUpdateRuntimeTests extends AnnotationsTestsBase { public static final String javaVersion = System.getProperty("java.version"); public static final String scalaVersion = - Properties.scalaPropOrElse("version.number", "unknown"); + Properties.scalaPropOrElse("version.number", new Function0() { + @Override + public String apply() { + return "unknown"; + } + }); public RumbleRuntimeConfiguration getConfiguration() { return new RumbleRuntimeConfiguration( diff --git a/src/test/java/iq/RuntimeTests.java b/src/test/java/iq/RuntimeTests.java index 7dbbffed82..1a3eb1c735 100644 --- a/src/test/java/iq/RuntimeTests.java +++ b/src/test/java/iq/RuntimeTests.java @@ -36,6 +36,9 @@ import scala.util.Properties; import sparksoniq.spark.SparkSessionManager; import utils.FileManager; +import scala.Function0; +import scala.util.Properties; + import java.io.File; import java.util.*; @@ -51,7 +54,13 @@ public class RuntimeTests extends AnnotationsTestsBase { public static final String javaVersion = System.getProperty("java.version"); public static final String scalaVersion = - Properties.scalaPropOrElse("version.number", "unknown"); + Properties.scalaPropOrElse("version.number", new Function0() { + @Override + public String apply() { + return "unknown"; + } + }); + protected static List _testFiles = new ArrayList<>(); protected final File testFile; diff --git a/src/test/java/iq/StaticTypeTests.java b/src/test/java/iq/StaticTypeTests.java index edec1671cb..c0827526e9 100644 --- a/src/test/java/iq/StaticTypeTests.java +++ b/src/test/java/iq/StaticTypeTests.java @@ -3,6 +3,7 @@ import org.rumbledb.config.RumbleRuntimeConfiguration; import iq.base.AnnotationsTestsBase; +import scala.Function0; import scala.util.Properties; import org.apache.spark.SparkConf; @@ -37,7 +38,13 @@ public class StaticTypeTests extends AnnotationsTestsBase { public static final String javaVersion = System.getProperty("java.version"); public static final String scalaVersion = - Properties.scalaPropOrElse("version.number", "unknown"); + Properties.scalaPropOrElse("version.number", new Function0() { + @Override + public String apply() { + return "unknown"; + } + }); + protected static List _testFiles = new ArrayList<>(); protected final File testFile; diff --git a/src/test/java/iq/UpdatesForRumbleBenchmark.java b/src/test/java/iq/UpdatesForRumbleBenchmark.java index da2a41317f..3e6cf4f92c 100644 --- a/src/test/java/iq/UpdatesForRumbleBenchmark.java +++ b/src/test/java/iq/UpdatesForRumbleBenchmark.java @@ -12,6 +12,7 @@ import org.rumbledb.exceptions.ExceptionMetadata; import org.rumbledb.runtime.functions.input.FileSystemUtil; import scala.util.Properties; +import scala.Function0; import sparksoniq.spark.SparkSessionManager; import java.io.BufferedWriter; @@ -28,7 +29,13 @@ public class UpdatesForRumbleBenchmark { public static final String javaVersion = System.getProperty("java.version"); public static final String scalaVersion = - Properties.scalaPropOrElse("version.number", "unknown"); + Properties.scalaPropOrElse("version.number", new Function0() { + @Override + public String apply() { + return "unknown"; + } + }); + public List benchmarkFiles; diff --git a/src/test/resources/test_files/runtime/FunctionNumerics/Exponential/FunctionExp.jq b/src/test/resources/test_files/runtime/FunctionNumerics/Exponential/FunctionExp.jq index 95a6eb6663..c93d2ef15b 100644 --- a/src/test/resources/test_files/runtime/FunctionNumerics/Exponential/FunctionExp.jq +++ b/src/test/resources/test_files/runtime/FunctionNumerics/Exponential/FunctionExp.jq @@ -1,4 +1,4 @@ -(:JIQS: ShouldRun; Output="(1, 2.718281828459045)" :) +(:JIQS: ShouldRun; Output="(1, 2.7182818284590455)" :) exp(0), exp(1), exp(()) From c03ee2ec2d58677fbea4b7b4fba0fc22baaf536b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marco=20Sch=C3=B6b?= Date: Fri, 25 Oct 2024 09:54:18 +0200 Subject: [PATCH 17/20] maven spotless --- .../java/org/rumbledb/items/parsing/ItemParser.java | 1 - src/test/java/iq/Bugs.java | 12 ++++++------ src/test/java/iq/DeltaUpdateRuntimeTests.java | 12 ++++++------ src/test/java/iq/RuntimeTests.java | 12 ++++++------ src/test/java/iq/StaticTypeTests.java | 12 ++++++------ src/test/java/iq/UpdatesForRumbleBenchmark.java | 12 ++++++------ 6 files changed, 30 insertions(+), 31 deletions(-) diff --git a/src/main/java/org/rumbledb/items/parsing/ItemParser.java b/src/main/java/org/rumbledb/items/parsing/ItemParser.java index be4b8fd93f..4c83d2047b 100644 --- a/src/main/java/org/rumbledb/items/parsing/ItemParser.java +++ b/src/main/java/org/rumbledb/items/parsing/ItemParser.java @@ -54,7 +54,6 @@ import java.io.IOException; import java.io.Serializable; import java.io.StringReader; -import java.lang.reflect.Array; import java.math.BigDecimal; import java.math.BigInteger; import java.sql.Date; diff --git a/src/test/java/iq/Bugs.java b/src/test/java/iq/Bugs.java index ee2b0f2160..21ec03f0a6 100644 --- a/src/test/java/iq/Bugs.java +++ b/src/test/java/iq/Bugs.java @@ -52,12 +52,12 @@ public class Bugs extends AnnotationsTestsBase { public static final String javaVersion = System.getProperty("java.version"); public static final String scalaVersion = - Properties.scalaPropOrElse("version.number", new Function0() { - @Override - public String apply() { - return "unknown"; - } - }); + Properties.scalaPropOrElse("version.number", new Function0() { + @Override + public String apply() { + return "unknown"; + } + }); protected static List _testFiles = new ArrayList<>(); protected final File testFile; diff --git a/src/test/java/iq/DeltaUpdateRuntimeTests.java b/src/test/java/iq/DeltaUpdateRuntimeTests.java index 5587ccab75..990151dc6e 100644 --- a/src/test/java/iq/DeltaUpdateRuntimeTests.java +++ b/src/test/java/iq/DeltaUpdateRuntimeTests.java @@ -59,12 +59,12 @@ public class DeltaUpdateRuntimeTests extends AnnotationsTestsBase { public static final String javaVersion = System.getProperty("java.version"); public static final String scalaVersion = - Properties.scalaPropOrElse("version.number", new Function0() { - @Override - public String apply() { - return "unknown"; - } - }); + Properties.scalaPropOrElse("version.number", new Function0() { + @Override + public String apply() { + return "unknown"; + } + }); public RumbleRuntimeConfiguration getConfiguration() { return new RumbleRuntimeConfiguration( diff --git a/src/test/java/iq/RuntimeTests.java b/src/test/java/iq/RuntimeTests.java index 1a3eb1c735..71674f4ff0 100644 --- a/src/test/java/iq/RuntimeTests.java +++ b/src/test/java/iq/RuntimeTests.java @@ -54,12 +54,12 @@ public class RuntimeTests extends AnnotationsTestsBase { public static final String javaVersion = System.getProperty("java.version"); public static final String scalaVersion = - Properties.scalaPropOrElse("version.number", new Function0() { - @Override - public String apply() { - return "unknown"; - } - }); + Properties.scalaPropOrElse("version.number", new Function0() { + @Override + public String apply() { + return "unknown"; + } + }); protected static List _testFiles = new ArrayList<>(); protected final File testFile; diff --git a/src/test/java/iq/StaticTypeTests.java b/src/test/java/iq/StaticTypeTests.java index c0827526e9..7ce351bff0 100644 --- a/src/test/java/iq/StaticTypeTests.java +++ b/src/test/java/iq/StaticTypeTests.java @@ -38,12 +38,12 @@ public class StaticTypeTests extends AnnotationsTestsBase { public static final String javaVersion = System.getProperty("java.version"); public static final String scalaVersion = - Properties.scalaPropOrElse("version.number", new Function0() { - @Override - public String apply() { - return "unknown"; - } - }); + Properties.scalaPropOrElse("version.number", new Function0() { + @Override + public String apply() { + return "unknown"; + } + }); protected static List _testFiles = new ArrayList<>(); protected final File testFile; diff --git a/src/test/java/iq/UpdatesForRumbleBenchmark.java b/src/test/java/iq/UpdatesForRumbleBenchmark.java index 3e6cf4f92c..5697fd0081 100644 --- a/src/test/java/iq/UpdatesForRumbleBenchmark.java +++ b/src/test/java/iq/UpdatesForRumbleBenchmark.java @@ -29,12 +29,12 @@ public class UpdatesForRumbleBenchmark { public static final String javaVersion = System.getProperty("java.version"); public static final String scalaVersion = - Properties.scalaPropOrElse("version.number", new Function0() { - @Override - public String apply() { - return "unknown"; - } - }); + Properties.scalaPropOrElse("version.number", new Function0() { + @Override + public String apply() { + return "unknown"; + } + }); public List benchmarkFiles; From dbcd1d70eb0bdc18a13131587494f33cb2501c14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marco=20Sch=C3=B6b?= Date: Fri, 25 Oct 2024 10:08:29 +0200 Subject: [PATCH 18/20] update tests again --- .../RumbleML/RumbleML/EstimatorTests/MLEstimator-FPGrowth.jq | 2 +- .../runtime/FunctionNumerics/Exponential/FunctionExp.jq | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/resources/test_files/RumbleML/RumbleML/EstimatorTests/MLEstimator-FPGrowth.jq b/src/test/resources/test_files/RumbleML/RumbleML/EstimatorTests/MLEstimator-FPGrowth.jq index 6302bdcf4b..d6c435cf48 100644 --- a/src/test/resources/test_files/RumbleML/RumbleML/EstimatorTests/MLEstimator-FPGrowth.jq +++ b/src/test/resources/test_files/RumbleML/RumbleML/EstimatorTests/MLEstimator-FPGrowth.jq @@ -1,4 +1,4 @@ -(:JIQS: ShouldRun; Output="({ "label" : 0, "name" : "a", "prediction" : [ "3", "4", "2", "6", "5" ] }, { "label" : 1, "name" : "b", "prediction" : [ "3", "4", "6", "5", "1" ] }, { "label" : 2, "name" : "c", "prediction" : [ "4", "2", "6", "5", "1" ] }, { "label" : 3, "name" : "d", "prediction" : [ "3", "2", "6", "5", "1" ] }, { "label" : 4, "name" : "e", "prediction" : [ "3", "4", "2", "6", "1" ] }, { "label" : 5, "name" : "f", "prediction" : [ "3", "4", "2", "5", "1" ] })" :) +(:JIQS: ShouldRun; Output="({ "label" : 0, "name" : "a", "prediction" : [ "4", "2", "3", "6", "5" ] }, { "label" : 1, "name" : "b", "prediction" : [ "4", "3", "6", "5", "1" ] }, { "label" : 2, "name" : "c", "prediction" : [ "4", "2", "6", "5", "1" ] }, { "label" : 3, "name" : "d", "prediction" : [ "2", "3", "6", "5", "1" ] }, { "label" : 4, "name" : "e", "prediction" : [ "4", "2", "3", "6", "1" ] }, { "label" : 5, "name" : "f", "prediction" : [ "4", "2", "3", "5", "1" ] })" :) let $data := annotate( json-file("../../../../queries/rumbleML/sample-ml-data-flat.json"), { "label": "integer", "binaryLabel": "integer", "name": "string", "age": "double", "weight": "double", "booleanCol": "boolean", "nullCol": "null", "stringCol": "string", "stringArrayCol": ["string"], "intArrayCol": ["integer"], "doubleArrayCol": ["double"], "doubleArrayArrayCol": [["double"]] } diff --git a/src/test/resources/test_files/runtime/FunctionNumerics/Exponential/FunctionExp.jq b/src/test/resources/test_files/runtime/FunctionNumerics/Exponential/FunctionExp.jq index c93d2ef15b..95a6eb6663 100644 --- a/src/test/resources/test_files/runtime/FunctionNumerics/Exponential/FunctionExp.jq +++ b/src/test/resources/test_files/runtime/FunctionNumerics/Exponential/FunctionExp.jq @@ -1,4 +1,4 @@ -(:JIQS: ShouldRun; Output="(1, 2.7182818284590455)" :) +(:JIQS: ShouldRun; Output="(1, 2.718281828459045)" :) exp(0), exp(1), exp(()) From f9fb6d3b45a1161d51527ca8fa0769a8edb5ce83 Mon Sep 17 00:00:00 2001 From: Ghislain Fourny Date: Mon, 28 Oct 2024 10:58:26 +0100 Subject: [PATCH 19/20] Execution mode for program. --- .../compiler/ExecutionModeVisitor.java | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/src/main/java/org/rumbledb/compiler/ExecutionModeVisitor.java b/src/main/java/org/rumbledb/compiler/ExecutionModeVisitor.java index 5e8a7ccc2c..8a5380858a 100644 --- a/src/main/java/org/rumbledb/compiler/ExecutionModeVisitor.java +++ b/src/main/java/org/rumbledb/compiler/ExecutionModeVisitor.java @@ -656,7 +656,11 @@ public StaticContext visitProlog(Prolog prolog, StaticContext argument) { @Override public StaticContext visitProgram(Program program, StaticContext argument) { visitDescendants(program, argument); - ExecutionMode mergedExecutionMode = getHighestExecutionModeFromStatements(exitStatementChildren); + ExecutionMode mergedExecutionMode = program.getStatementsAndOptionalExpr().getHighestExecutionMode(); + for (Statement statement : exitStatementChildren) { + ExecutionMode statementExecMode = statement.getHighestExecutionMode(this.visitorConfig); + mergedExecutionMode = getHighestExecutionMode(mergedExecutionMode, statementExecMode); + } program.setHighestExecutionMode(mergedExecutionMode); return argument; } @@ -1044,7 +1048,7 @@ public StaticContext visitExitStatement(ExitStatement exitStatement, StaticConte return argument; } - private ExecutionMode getHighestExecutionMode(ExecutionMode firstExecMode, ExecutionMode secondExecMode) { + private static ExecutionMode getHighestExecutionMode(ExecutionMode firstExecMode, ExecutionMode secondExecMode) { if (firstExecMode == ExecutionMode.UNSET || secondExecMode == ExecutionMode.UNSET) { return ExecutionMode.UNSET; } @@ -1061,16 +1065,7 @@ private ExecutionMode getHighestExecutionModeFromStatements(List stat ExecutionMode result = ExecutionMode.UNSET; for (Statement statement : statements) { ExecutionMode statementExecMode = statement.getHighestExecutionMode(this.visitorConfig); - if (statementExecMode.isUnset()) { - return ExecutionMode.UNSET; - } - if (result.isUnset()) { - result = statementExecMode; - } else if (result.isRDD() && statementExecMode.isLocal()) { - return ExecutionMode.LOCAL; - } else if (result.isDataFrame() && !statementExecMode.isDataFrame()) { - result = statementExecMode; - } + result = getHighestExecutionMode(result, statementExecMode); } return result; } From 5c4be945bf1e4de8c9b6c0a809a43f9f45c5f307 Mon Sep 17 00:00:00 2001 From: Ghislain Fourny Date: Mon, 28 Oct 2024 11:12:50 +0100 Subject: [PATCH 20/20] Fix tests. --- .../org/rumbledb/compiler/ExecutionModeVisitor.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/rumbledb/compiler/ExecutionModeVisitor.java b/src/main/java/org/rumbledb/compiler/ExecutionModeVisitor.java index 8a5380858a..61693f431a 100644 --- a/src/main/java/org/rumbledb/compiler/ExecutionModeVisitor.java +++ b/src/main/java/org/rumbledb/compiler/ExecutionModeVisitor.java @@ -1065,7 +1065,16 @@ private ExecutionMode getHighestExecutionModeFromStatements(List stat ExecutionMode result = ExecutionMode.UNSET; for (Statement statement : statements) { ExecutionMode statementExecMode = statement.getHighestExecutionMode(this.visitorConfig); - result = getHighestExecutionMode(result, statementExecMode); + if (statementExecMode.isUnset()) { + return ExecutionMode.UNSET; + } + if (result.isUnset()) { + result = statementExecMode; + } else if (result.isRDD() && statementExecMode.isLocal()) { + return ExecutionMode.LOCAL; + } else if (result.isDataFrame() && !statementExecMode.isDataFrame()) { + result = statementExecMode; + } } return result; }