From 86e3c67b966b3c7c60c3ffe3389e37f72e1eccad Mon Sep 17 00:00:00 2001 From: David Rabinowitz Date: Fri, 13 Jan 2023 07:55:44 -0800 Subject: [PATCH] Adding Spark 3.2 Connector (#860) --- cloudbuild/cloudbuild.yaml | 19 +++++++-- cloudbuild/nightly.sh | 4 +- cloudbuild/nightly.yaml | 3 ++ cloudbuild/presubmit.sh | 9 +++- .../Spark31DirectWriteIntegrationTest.java | 0 .../Spark31IndirectWriteIntegrationTest.java | 0 .../Spark31QueryPushdownIntegrationTest.java | 0 .../Spark31ReadByFormatIntegrationTest.java | 0 .../Spark31ReadFromQueryIntegrationTest.java | 0 .../Spark31ReadIntegrationTest.java | 0 .../v2/Spark32BigQueryTableProvider.java | 9 +--- ...mericDataprocServerlessAcceptanceTest.java | 27 ++++++++++++ .../Spark32DataprocImage21AcceptanceTest.java | 41 +++++++++++++++++++ ...Image21DisableConscryptAcceptanceTest.java | 41 +++++++++++++++++++ ...peareDataprocServerlessAcceptanceTest.java | 27 ++++++++++++ ...treamDataprocServerlessAcceptanceTest.java | 30 ++++++++++++++ .../Spark32DirectWriteIntegrationTest.java | 27 ++++++++++++ .../Spark32IndirectWriteIntegrationTest.java | 35 ++++++++++++++++ .../Spark32QueryPushdownIntegrationTest.java | 27 ++++++++++++ .../Spark32ReadByFormatIntegrationTest.java | 32 +++++++++++++++ .../Spark32ReadFromQueryIntegrationTest.java | 22 ++++++++++ .../Spark32ReadIntegrationTest.java | 25 +++++++++++ 22 files changed, 363 insertions(+), 15 deletions(-) rename spark-bigquery-dsv2/{spark-3.1-bigquery-lib => spark-3.1-bigquery}/src/test/java/com/google/cloud/spark/bigquery/integration/Spark31DirectWriteIntegrationTest.java (100%) rename spark-bigquery-dsv2/{spark-3.1-bigquery-lib => spark-3.1-bigquery}/src/test/java/com/google/cloud/spark/bigquery/integration/Spark31IndirectWriteIntegrationTest.java (100%) rename spark-bigquery-dsv2/{spark-3.1-bigquery-lib => spark-3.1-bigquery}/src/test/java/com/google/cloud/spark/bigquery/integration/Spark31QueryPushdownIntegrationTest.java (100%) rename spark-bigquery-dsv2/{spark-3.1-bigquery-lib => spark-3.1-bigquery}/src/test/java/com/google/cloud/spark/bigquery/integration/Spark31ReadByFormatIntegrationTest.java (100%) rename spark-bigquery-dsv2/{spark-3.1-bigquery-lib => spark-3.1-bigquery}/src/test/java/com/google/cloud/spark/bigquery/integration/Spark31ReadFromQueryIntegrationTest.java (100%) rename spark-bigquery-dsv2/{spark-3.1-bigquery-lib => spark-3.1-bigquery}/src/test/java/com/google/cloud/spark/bigquery/integration/Spark31ReadIntegrationTest.java (100%) create mode 100644 spark-bigquery-dsv2/spark-3.2-bigquery/src/test/java/com/google/cloud/spark/bigquery/acceptance/Spark32BigNumericDataprocServerlessAcceptanceTest.java create mode 100644 spark-bigquery-dsv2/spark-3.2-bigquery/src/test/java/com/google/cloud/spark/bigquery/acceptance/Spark32DataprocImage21AcceptanceTest.java create mode 100644 spark-bigquery-dsv2/spark-3.2-bigquery/src/test/java/com/google/cloud/spark/bigquery/acceptance/Spark32DataprocImage21DisableConscryptAcceptanceTest.java create mode 100644 spark-bigquery-dsv2/spark-3.2-bigquery/src/test/java/com/google/cloud/spark/bigquery/acceptance/Spark32ReadSheakspeareDataprocServerlessAcceptanceTest.java create mode 100644 spark-bigquery-dsv2/spark-3.2-bigquery/src/test/java/com/google/cloud/spark/bigquery/acceptance/Spark32WriteStreamDataprocServerlessAcceptanceTest.java create mode 100644 spark-bigquery-dsv2/spark-3.2-bigquery/src/test/java/com/google/cloud/spark/bigquery/integration/Spark32DirectWriteIntegrationTest.java create mode 100644 spark-bigquery-dsv2/spark-3.2-bigquery/src/test/java/com/google/cloud/spark/bigquery/integration/Spark32IndirectWriteIntegrationTest.java create mode 100644 spark-bigquery-dsv2/spark-3.2-bigquery/src/test/java/com/google/cloud/spark/bigquery/integration/Spark32QueryPushdownIntegrationTest.java create mode 100644 spark-bigquery-dsv2/spark-3.2-bigquery/src/test/java/com/google/cloud/spark/bigquery/integration/Spark32ReadByFormatIntegrationTest.java create mode 100644 spark-bigquery-dsv2/spark-3.2-bigquery/src/test/java/com/google/cloud/spark/bigquery/integration/Spark32ReadFromQueryIntegrationTest.java create mode 100644 spark-bigquery-dsv2/spark-3.2-bigquery/src/test/java/com/google/cloud/spark/bigquery/integration/Spark32ReadIntegrationTest.java diff --git a/cloudbuild/cloudbuild.yaml b/cloudbuild/cloudbuild.yaml index e6588af3e..c120edbf6 100644 --- a/cloudbuild/cloudbuild.yaml +++ b/cloudbuild/cloudbuild.yaml @@ -36,7 +36,7 @@ steps: # 4b. Run integration tests concurrently with unit tests (DSv1, Scala 2.13) - name: 'gcr.io/$PROJECT_ID/dataproc-spark-bigquery-connector-presubmit' id: 'integration-tests-2.13' - waitFor: ['integration-tests-2.12'] + waitFor: ['unit-tests'] entrypoint: 'bash' args: ['/workspace/cloudbuild/presubmit.sh', 'integrationtest-2.13'] env: @@ -47,7 +47,7 @@ steps: # 4c. Run integration tests concurrently with unit tests (DSv2, Spark 2.4) - name: 'gcr.io/$PROJECT_ID/dataproc-spark-bigquery-connector-presubmit' id: 'integration-tests-2.4' - waitFor: ['unit-tests'] + waitFor: ['integration-tests-2.12'] entrypoint: 'bash' args: ['/workspace/cloudbuild/presubmit.sh', 'integrationtest-2.4'] env: @@ -58,7 +58,7 @@ steps: # 4d. Run integration tests concurrently with unit tests (DSv2, Spark 3.1) - name: 'gcr.io/$PROJECT_ID/dataproc-spark-bigquery-connector-presubmit' id: 'integration-tests-3.1' - waitFor: ['integration-tests-2.4'] + waitFor: ['integration-tests-2.13'] entrypoint: 'bash' args: ['/workspace/cloudbuild/presubmit.sh', 'integrationtest-3.1'] env: @@ -66,10 +66,21 @@ steps: - 'TEMPORARY_GCS_BUCKET=${_TEMPORARY_GCS_BUCKET}' - 'BIGLAKE_CONNECTION_ID=${_BIGLAKE_CONNECTION_ID}' +# 4c. Run integration tests concurrently with unit tests (DSv2, Spark 3.2) + - name: 'gcr.io/$PROJECT_ID/dataproc-spark-bigquery-connector-presubmit' + id: 'integration-tests-3.2' + waitFor: ['integration-tests-2.4'] + entrypoint: 'bash' + args: ['/workspace/cloudbuild/presubmit.sh', 'integrationtest-3.2'] + env: + - 'GOOGLE_CLOUD_PROJECT=${_GOOGLE_CLOUD_PROJECT}' + - 'TEMPORARY_GCS_BUCKET=${_TEMPORARY_GCS_BUCKET}' + - 'BIGLAKE_CONNECTION_ID=${_BIGLAKE_CONNECTION_ID}' + # 5. Upload coverage to CodeCov - name: 'gcr.io/$PROJECT_ID/dataproc-spark-bigquery-connector-presubmit' id: 'upload-it-to-codecov' - waitFor: ['integration-tests-2.12','integration-tests-2.13','integration-tests-2.4','integration-tests-3.1'] + waitFor: ['integration-tests-2.12','integration-tests-2.13','integration-tests-2.4','integration-tests-3.1','integration-tests-3.2'] entrypoint: 'bash' args: ['/workspace/cloudbuild/presubmit.sh', 'upload-it-to-codecov'] env: diff --git a/cloudbuild/nightly.sh b/cloudbuild/nightly.sh index 2f255cebb..bfc496576 100644 --- a/cloudbuild/nightly.sh +++ b/cloudbuild/nightly.sh @@ -34,9 +34,9 @@ $MVN install -DskipTests -Pdsv1,dsv2 #coverage report $MVN test jacoco:report jacoco:report-aggregate -Pcoverage,dsv1,dsv2 # Run integration tests -$MVN failsafe:integration-test failsafe:verify jacoco:report jacoco:report-aggregate -Pcoverage,integration,dsv1,dsv2_2.4,dsv2_3.1 +$MVN failsafe:integration-test failsafe:verify jacoco:report jacoco:report-aggregate -Pcoverage,integration,dsv1,dsv2_2.4,dsv2_3.1,dsv2_3.2 # Run acceptance tests -$MVN failsafe:integration-test failsafe:verify jacoco:report jacoco:report-aggregate -Pcoverage,acceptance,dsv1,dsv2_2.4,dsv2_3.1 +$MVN failsafe:integration-test failsafe:verify jacoco:report jacoco:report-aggregate -Pcoverage,acceptance,dsv1,dsv2_2.4,dsv2_3.1,dsv2_3.2 # Upload test coverage report to Codecov bash <(curl -s https://codecov.io/bash) -K -F "nightly" diff --git a/cloudbuild/nightly.yaml b/cloudbuild/nightly.yaml index 409204bca..486c762e5 100644 --- a/cloudbuild/nightly.yaml +++ b/cloudbuild/nightly.yaml @@ -41,6 +41,9 @@ steps: gsutil cp "/workspace/.repository/com/google/cloud/spark/spark-3.1-bigquery/$revision-preview/spark-3.1-bigquery-$revision-preview.jar" "gs://$bucket" gsutil cp "gs://$bucket/spark-3.1-bigquery-$revision-preview.jar" "gs://$bucket/spark-3.1-bigquery-nightly-snapshot-preview.jar" + + gsutil cp "/workspace/.repository/com/google/cloud/spark/spark-3.2-bigquery/$revision-preview/spark-3.2-bigquery-$revision-preview.jar" "gs://$bucket" + gsutil cp "gs://$bucket/spark-3.2-bigquery-$revision-preview.jar" "gs://$bucket/spark-3.2-bigquery-nightly-snapshot-preview.jar" - name: 'gcr.io/cloud-builders/docker' id: 'docker-push' args: ['push', 'gcr.io/$PROJECT_ID/dataproc-spark-bigquery-connector-nightly:latest'] diff --git a/cloudbuild/presubmit.sh b/cloudbuild/presubmit.sh index aff2e8177..611fbfc19 100644 --- a/cloudbuild/presubmit.sh +++ b/cloudbuild/presubmit.sh @@ -33,13 +33,13 @@ case $STEP in # Download maven and all the dependencies init) checkenv - $MVN install -DskipTests -Pdsv1_2.12,dsv1_2.13,dsv2_2.4,dsv2_3.1 + $MVN install -DskipTests -Pdsv1_2.12,dsv1_2.13,dsv2_2.4,dsv2_3.1,dsv2_3.2 exit ;; # Run unit tests unittest) - $MVN test jacoco:report jacoco:report-aggregate -Pcoverage,dsv1_2.12,dsv1_2.13,dsv2_2.4,dsv2_3.1 + $MVN test jacoco:report jacoco:report-aggregate -Pcoverage,dsv1_2.12,dsv1_2.13,dsv2_2.4,dsv2_3.1,dsv2_3.2 # Upload test coverage report to Codecov bash <(curl -s https://codecov.io/bash) -K -F "${STEP}" ;; @@ -64,6 +64,11 @@ case $STEP in $MVN failsafe:integration-test failsafe:verify jacoco:report jacoco:report-aggregate -Pcoverage,integration,dsv2_3.1 ;; + # Run integration tests + integrationtest-3.2) + $MVN failsafe:integration-test failsafe:verify jacoco:report jacoco:report-aggregate -Pcoverage,integration,dsv2_3.2 + ;; + upload-it-to-codecov) checkenv # Upload test coverage report to Codecov diff --git a/spark-bigquery-dsv2/spark-3.1-bigquery-lib/src/test/java/com/google/cloud/spark/bigquery/integration/Spark31DirectWriteIntegrationTest.java b/spark-bigquery-dsv2/spark-3.1-bigquery/src/test/java/com/google/cloud/spark/bigquery/integration/Spark31DirectWriteIntegrationTest.java similarity index 100% rename from spark-bigquery-dsv2/spark-3.1-bigquery-lib/src/test/java/com/google/cloud/spark/bigquery/integration/Spark31DirectWriteIntegrationTest.java rename to spark-bigquery-dsv2/spark-3.1-bigquery/src/test/java/com/google/cloud/spark/bigquery/integration/Spark31DirectWriteIntegrationTest.java diff --git a/spark-bigquery-dsv2/spark-3.1-bigquery-lib/src/test/java/com/google/cloud/spark/bigquery/integration/Spark31IndirectWriteIntegrationTest.java b/spark-bigquery-dsv2/spark-3.1-bigquery/src/test/java/com/google/cloud/spark/bigquery/integration/Spark31IndirectWriteIntegrationTest.java similarity index 100% rename from spark-bigquery-dsv2/spark-3.1-bigquery-lib/src/test/java/com/google/cloud/spark/bigquery/integration/Spark31IndirectWriteIntegrationTest.java rename to spark-bigquery-dsv2/spark-3.1-bigquery/src/test/java/com/google/cloud/spark/bigquery/integration/Spark31IndirectWriteIntegrationTest.java diff --git a/spark-bigquery-dsv2/spark-3.1-bigquery-lib/src/test/java/com/google/cloud/spark/bigquery/integration/Spark31QueryPushdownIntegrationTest.java b/spark-bigquery-dsv2/spark-3.1-bigquery/src/test/java/com/google/cloud/spark/bigquery/integration/Spark31QueryPushdownIntegrationTest.java similarity index 100% rename from spark-bigquery-dsv2/spark-3.1-bigquery-lib/src/test/java/com/google/cloud/spark/bigquery/integration/Spark31QueryPushdownIntegrationTest.java rename to spark-bigquery-dsv2/spark-3.1-bigquery/src/test/java/com/google/cloud/spark/bigquery/integration/Spark31QueryPushdownIntegrationTest.java diff --git a/spark-bigquery-dsv2/spark-3.1-bigquery-lib/src/test/java/com/google/cloud/spark/bigquery/integration/Spark31ReadByFormatIntegrationTest.java b/spark-bigquery-dsv2/spark-3.1-bigquery/src/test/java/com/google/cloud/spark/bigquery/integration/Spark31ReadByFormatIntegrationTest.java similarity index 100% rename from spark-bigquery-dsv2/spark-3.1-bigquery-lib/src/test/java/com/google/cloud/spark/bigquery/integration/Spark31ReadByFormatIntegrationTest.java rename to spark-bigquery-dsv2/spark-3.1-bigquery/src/test/java/com/google/cloud/spark/bigquery/integration/Spark31ReadByFormatIntegrationTest.java diff --git a/spark-bigquery-dsv2/spark-3.1-bigquery-lib/src/test/java/com/google/cloud/spark/bigquery/integration/Spark31ReadFromQueryIntegrationTest.java b/spark-bigquery-dsv2/spark-3.1-bigquery/src/test/java/com/google/cloud/spark/bigquery/integration/Spark31ReadFromQueryIntegrationTest.java similarity index 100% rename from spark-bigquery-dsv2/spark-3.1-bigquery-lib/src/test/java/com/google/cloud/spark/bigquery/integration/Spark31ReadFromQueryIntegrationTest.java rename to spark-bigquery-dsv2/spark-3.1-bigquery/src/test/java/com/google/cloud/spark/bigquery/integration/Spark31ReadFromQueryIntegrationTest.java diff --git a/spark-bigquery-dsv2/spark-3.1-bigquery-lib/src/test/java/com/google/cloud/spark/bigquery/integration/Spark31ReadIntegrationTest.java b/spark-bigquery-dsv2/spark-3.1-bigquery/src/test/java/com/google/cloud/spark/bigquery/integration/Spark31ReadIntegrationTest.java similarity index 100% rename from spark-bigquery-dsv2/spark-3.1-bigquery-lib/src/test/java/com/google/cloud/spark/bigquery/integration/Spark31ReadIntegrationTest.java rename to spark-bigquery-dsv2/spark-3.1-bigquery/src/test/java/com/google/cloud/spark/bigquery/integration/Spark31ReadIntegrationTest.java diff --git a/spark-bigquery-dsv2/spark-3.2-bigquery-lib/src/main/java/com/google/cloud/spark/bigquery/v2/Spark32BigQueryTableProvider.java b/spark-bigquery-dsv2/spark-3.2-bigquery-lib/src/main/java/com/google/cloud/spark/bigquery/v2/Spark32BigQueryTableProvider.java index f78f4cb26..fb433acc2 100644 --- a/spark-bigquery-dsv2/spark-3.2-bigquery-lib/src/main/java/com/google/cloud/spark/bigquery/v2/Spark32BigQueryTableProvider.java +++ b/spark-bigquery-dsv2/spark-3.2-bigquery-lib/src/main/java/com/google/cloud/spark/bigquery/v2/Spark32BigQueryTableProvider.java @@ -1,4 +1,4 @@ -/* +/* * Copyright 2023 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,11 +15,6 @@ */ package com.google.cloud.spark.bigquery.v2; -public class Spark32BigQueryTableProvider extends Spark31BigQueryTableProvider { +public class Spark32BigQueryTableProvider extends Spark31BigQueryTableProvider { // empty } - - - - - diff --git a/spark-bigquery-dsv2/spark-3.2-bigquery/src/test/java/com/google/cloud/spark/bigquery/acceptance/Spark32BigNumericDataprocServerlessAcceptanceTest.java b/spark-bigquery-dsv2/spark-3.2-bigquery/src/test/java/com/google/cloud/spark/bigquery/acceptance/Spark32BigNumericDataprocServerlessAcceptanceTest.java new file mode 100644 index 000000000..c2328a0a7 --- /dev/null +++ b/spark-bigquery-dsv2/spark-3.2-bigquery/src/test/java/com/google/cloud/spark/bigquery/acceptance/Spark32BigNumericDataprocServerlessAcceptanceTest.java @@ -0,0 +1,27 @@ +/* + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.spark.bigquery.acceptance; + +public class Spark32BigNumericDataprocServerlessAcceptanceTest + extends BigNumericDataprocServerlessAcceptanceTestBase { + + public Spark32BigNumericDataprocServerlessAcceptanceTest() { + super("spark-3.2-bigquery", "1.0"); + } + + // tests from superclass + +} diff --git a/spark-bigquery-dsv2/spark-3.2-bigquery/src/test/java/com/google/cloud/spark/bigquery/acceptance/Spark32DataprocImage21AcceptanceTest.java b/spark-bigquery-dsv2/spark-3.2-bigquery/src/test/java/com/google/cloud/spark/bigquery/acceptance/Spark32DataprocImage21AcceptanceTest.java new file mode 100644 index 000000000..4010f4569 --- /dev/null +++ b/spark-bigquery-dsv2/spark-3.2-bigquery/src/test/java/com/google/cloud/spark/bigquery/acceptance/Spark32DataprocImage21AcceptanceTest.java @@ -0,0 +1,41 @@ +/* + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.spark.bigquery.acceptance; + +import java.util.Collections; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +public class Spark32DataprocImage21AcceptanceTest extends DataprocAcceptanceTestBase { + + private static AcceptanceTestContext context; + + public Spark32DataprocImage21AcceptanceTest() { + super(context, false); + } + + @BeforeClass + public static void setup() throws Exception { + context = + DataprocAcceptanceTestBase.setup( + "2.1-debian11", "spark-3.2-bigquery", Collections.emptyList()); + } + + @AfterClass + public static void tearDown() throws Exception { + DataprocAcceptanceTestBase.tearDown(context); + } +} diff --git a/spark-bigquery-dsv2/spark-3.2-bigquery/src/test/java/com/google/cloud/spark/bigquery/acceptance/Spark32DataprocImage21DisableConscryptAcceptanceTest.java b/spark-bigquery-dsv2/spark-3.2-bigquery/src/test/java/com/google/cloud/spark/bigquery/acceptance/Spark32DataprocImage21DisableConscryptAcceptanceTest.java new file mode 100644 index 000000000..6435a3769 --- /dev/null +++ b/spark-bigquery-dsv2/spark-3.2-bigquery/src/test/java/com/google/cloud/spark/bigquery/acceptance/Spark32DataprocImage21DisableConscryptAcceptanceTest.java @@ -0,0 +1,41 @@ +/* + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.spark.bigquery.acceptance; + +import org.junit.AfterClass; +import org.junit.BeforeClass; + +public class Spark32DataprocImage21DisableConscryptAcceptanceTest + extends DataprocAcceptanceTestBase { + + private static AcceptanceTestContext context; + + public Spark32DataprocImage21DisableConscryptAcceptanceTest() { + super(context, false); + } + + @BeforeClass + public static void setup() throws Exception { + context = + DataprocAcceptanceTestBase.setup( + "2.1-debian11", "spark-3.2-bigquery", DISABLE_CONSCRYPT_LIST); + } + + @AfterClass + public static void tearDown() throws Exception { + DataprocAcceptanceTestBase.tearDown(context); + } +} diff --git a/spark-bigquery-dsv2/spark-3.2-bigquery/src/test/java/com/google/cloud/spark/bigquery/acceptance/Spark32ReadSheakspeareDataprocServerlessAcceptanceTest.java b/spark-bigquery-dsv2/spark-3.2-bigquery/src/test/java/com/google/cloud/spark/bigquery/acceptance/Spark32ReadSheakspeareDataprocServerlessAcceptanceTest.java new file mode 100644 index 000000000..3232ba895 --- /dev/null +++ b/spark-bigquery-dsv2/spark-3.2-bigquery/src/test/java/com/google/cloud/spark/bigquery/acceptance/Spark32ReadSheakspeareDataprocServerlessAcceptanceTest.java @@ -0,0 +1,27 @@ +/* + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.spark.bigquery.acceptance; + +public class Spark32ReadSheakspeareDataprocServerlessAcceptanceTest + extends ReadSheakspeareDataprocServerlessAcceptanceTestBase { + + public Spark32ReadSheakspeareDataprocServerlessAcceptanceTest() { + super("spark-3.2-bigquery", "1.0"); + } + + // tests from superclass + +} diff --git a/spark-bigquery-dsv2/spark-3.2-bigquery/src/test/java/com/google/cloud/spark/bigquery/acceptance/Spark32WriteStreamDataprocServerlessAcceptanceTest.java b/spark-bigquery-dsv2/spark-3.2-bigquery/src/test/java/com/google/cloud/spark/bigquery/acceptance/Spark32WriteStreamDataprocServerlessAcceptanceTest.java new file mode 100644 index 000000000..7f9eed4d2 --- /dev/null +++ b/spark-bigquery-dsv2/spark-3.2-bigquery/src/test/java/com/google/cloud/spark/bigquery/acceptance/Spark32WriteStreamDataprocServerlessAcceptanceTest.java @@ -0,0 +1,30 @@ +/* + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.spark.bigquery.acceptance; + +import org.junit.Ignore; + +@Ignore // spark-3.2-bigquery does not support streaming yet +public class Spark32WriteStreamDataprocServerlessAcceptanceTest + extends WriteStreamDataprocServerlessAcceptanceTestBase { + + public Spark32WriteStreamDataprocServerlessAcceptanceTest() { + super("spark-3.2-bigquery", "1.0"); + } + + // tests from superclass + +} diff --git a/spark-bigquery-dsv2/spark-3.2-bigquery/src/test/java/com/google/cloud/spark/bigquery/integration/Spark32DirectWriteIntegrationTest.java b/spark-bigquery-dsv2/spark-3.2-bigquery/src/test/java/com/google/cloud/spark/bigquery/integration/Spark32DirectWriteIntegrationTest.java new file mode 100644 index 000000000..5f0e70f10 --- /dev/null +++ b/spark-bigquery-dsv2/spark-3.2-bigquery/src/test/java/com/google/cloud/spark/bigquery/integration/Spark32DirectWriteIntegrationTest.java @@ -0,0 +1,27 @@ +/* + * Copyright 2018 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.spark.bigquery.integration; + +import com.google.cloud.spark.bigquery.SparkBigQueryConfig; + +public class Spark32DirectWriteIntegrationTest extends WriteIntegrationTestBase { + + public Spark32DirectWriteIntegrationTest() { + super(SparkBigQueryConfig.WriteMethod.DIRECT); + } + + // tests from superclass +} diff --git a/spark-bigquery-dsv2/spark-3.2-bigquery/src/test/java/com/google/cloud/spark/bigquery/integration/Spark32IndirectWriteIntegrationTest.java b/spark-bigquery-dsv2/spark-3.2-bigquery/src/test/java/com/google/cloud/spark/bigquery/integration/Spark32IndirectWriteIntegrationTest.java new file mode 100644 index 000000000..ee3dca530 --- /dev/null +++ b/spark-bigquery-dsv2/spark-3.2-bigquery/src/test/java/com/google/cloud/spark/bigquery/integration/Spark32IndirectWriteIntegrationTest.java @@ -0,0 +1,35 @@ +/* + * Copyright 2018 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.spark.bigquery.integration; + +import com.google.cloud.spark.bigquery.SparkBigQueryConfig; +import org.junit.Before; + +public class Spark32IndirectWriteIntegrationTest extends WriteIntegrationTestBase { + + public Spark32IndirectWriteIntegrationTest() { + super(SparkBigQueryConfig.WriteMethod.INDIRECT); + } + + @Before + public void setParquetLoadBehaviour() { + // TODO: make this the default value + spark.conf().set("enableListInference", "true"); + } + + // tests from superclass + +} diff --git a/spark-bigquery-dsv2/spark-3.2-bigquery/src/test/java/com/google/cloud/spark/bigquery/integration/Spark32QueryPushdownIntegrationTest.java b/spark-bigquery-dsv2/spark-3.2-bigquery/src/test/java/com/google/cloud/spark/bigquery/integration/Spark32QueryPushdownIntegrationTest.java new file mode 100644 index 000000000..f2a39ae0a --- /dev/null +++ b/spark-bigquery-dsv2/spark-3.2-bigquery/src/test/java/com/google/cloud/spark/bigquery/integration/Spark32QueryPushdownIntegrationTest.java @@ -0,0 +1,27 @@ +/* + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.spark.bigquery.integration; + +import com.google.cloud.spark.bigquery.BigQueryConnectorUtils; +import org.junit.Before; + +public class Spark32QueryPushdownIntegrationTest extends QueryPushdownIntegrationTestBase { + + @Before + public void before() { + BigQueryConnectorUtils.enablePushdownSession(spark); + } +} diff --git a/spark-bigquery-dsv2/spark-3.2-bigquery/src/test/java/com/google/cloud/spark/bigquery/integration/Spark32ReadByFormatIntegrationTest.java b/spark-bigquery-dsv2/spark-3.2-bigquery/src/test/java/com/google/cloud/spark/bigquery/integration/Spark32ReadByFormatIntegrationTest.java new file mode 100644 index 000000000..3e76cc937 --- /dev/null +++ b/spark-bigquery-dsv2/spark-3.2-bigquery/src/test/java/com/google/cloud/spark/bigquery/integration/Spark32ReadByFormatIntegrationTest.java @@ -0,0 +1,32 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.spark.bigquery.integration; + +import org.junit.Test; + +public class Spark32ReadByFormatIntegrationTest extends ReadByFormatIntegrationTestBase { + + public Spark32ReadByFormatIntegrationTest() { + super("ARROW", /* userProvidedSchemaAllowed */ false); + } + + // tests are from the super-class + + @Test + public void foo() { + testCachedViewWithDifferentColumnsForSelectAndFilter(); + } +} diff --git a/spark-bigquery-dsv2/spark-3.2-bigquery/src/test/java/com/google/cloud/spark/bigquery/integration/Spark32ReadFromQueryIntegrationTest.java b/spark-bigquery-dsv2/spark-3.2-bigquery/src/test/java/com/google/cloud/spark/bigquery/integration/Spark32ReadFromQueryIntegrationTest.java new file mode 100644 index 000000000..0fb72b66f --- /dev/null +++ b/spark-bigquery-dsv2/spark-3.2-bigquery/src/test/java/com/google/cloud/spark/bigquery/integration/Spark32ReadFromQueryIntegrationTest.java @@ -0,0 +1,22 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.spark.bigquery.integration; + +public class Spark32ReadFromQueryIntegrationTest extends ReadFromQueryIntegrationTestBase { + + // tests are from the super-class + +} diff --git a/spark-bigquery-dsv2/spark-3.2-bigquery/src/test/java/com/google/cloud/spark/bigquery/integration/Spark32ReadIntegrationTest.java b/spark-bigquery-dsv2/spark-3.2-bigquery/src/test/java/com/google/cloud/spark/bigquery/integration/Spark32ReadIntegrationTest.java new file mode 100644 index 000000000..74df43ae9 --- /dev/null +++ b/spark-bigquery-dsv2/spark-3.2-bigquery/src/test/java/com/google/cloud/spark/bigquery/integration/Spark32ReadIntegrationTest.java @@ -0,0 +1,25 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.spark.bigquery.integration; + +public class Spark32ReadIntegrationTest extends ReadIntegrationTestBase { + public Spark32ReadIntegrationTest() { + super(/* userProvidedSchemaAllowed */ false); + } + + // tests are from the super-class + +}