From 30a219a33a97479d1214d7afd727b4d04b28e026 Mon Sep 17 00:00:00 2001 From: Christoph Pirkl <4711730+kaklakariada@users.noreply.github.com> Date: Mon, 17 Jun 2024 14:14:00 +0200 Subject: [PATCH] #58: Added option to keep original column name for auto inference (#59) * #58: Update dependencies * Upgrade azurite --- .github/workflows/dependencies_update.yml | 2 +- .vscode/settings.json | 44 +++++++++---------- dependencies.md | 6 --- doc/changes/changelog.md | 1 + doc/changes/changes_2.1.0.md | 38 ++++++++++++++++ doc/user_guide/user_guide.md | 4 +- pk_generated_parent.pom | 2 +- pom.xml | 25 ++++------- .../document/files/IntegrationTestSetup.java | 2 +- .../files/abstestsetup/LocalAbsTestSetup.java | 2 +- 10 files changed, 75 insertions(+), 51 deletions(-) create mode 100644 doc/changes/changes_2.1.0.md diff --git a/.github/workflows/dependencies_update.yml b/.github/workflows/dependencies_update.yml index 1bf502f..0fa7180 100644 --- a/.github/workflows/dependencies_update.yml +++ b/.github/workflows/dependencies_update.yml @@ -75,7 +75,7 @@ jobs: echo >> "$GITHUB_OUTPUT" echo '# ⚠️ Notes ⚠️' >> "$GITHUB_OUTPUT" echo '## Run PK fix manually' >> "$GITHUB_OUTPUT" - echo 'Due to restrictions workflow `dependencies_update.yml` can't update other workflows, see https://github.com/exasol/project-keeper/issues/578 for details.' >> "$GITHUB_OUTPUT" + echo 'Due to restrictions workflow `dependencies_update.yml` cannot update other workflows, see https://github.com/exasol/project-keeper/issues/578 for details.' >> "$GITHUB_OUTPUT" echo 'Please checkout this PR locally and run `mvn com.exasol:project-keeper-maven-plugin:fix --projects .`' >> "$GITHUB_OUTPUT" echo '## This PR does not trigger CI workflows' >> "$GITHUB_OUTPUT" echo 'Please click the **Close pull request** button and then **Reopen pull request** to trigger running checks.' >> "$GITHUB_OUTPUT" diff --git a/.vscode/settings.json b/.vscode/settings.json index 768b55c..32d79b8 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,25 +1,23 @@ { - "editor.formatOnSave": true, - "editor.codeActionsOnSave": { - "source.organizeImports": "explicit", - "source.generate.finalModifiers": "explicit", - "source.fixAll": "explicit" - }, - "java.codeGeneration.useBlocks": true, - "java.saveActions.organizeImports": true, - "java.sources.organizeImports.starThreshold": 3, - "java.sources.organizeImports.staticStarThreshold": 3, - "java.test.config": { - "vmArgs": [ - "-Djava.util.logging.config.file=src/test/resources/logging.properties", - "-Djavax.xml.parsers.DocumentBuilderFactory=com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl" - ] - }, - "[json]": { - "editor.indentSize": 2 - }, - "sonarlint.connectedMode.project": { - "connectionId": "exasol", - "projectKey": "com.exasol:azure-blob-storage-document-files-virtual-schema" - } + "editor.formatOnSave": true, + "editor.codeActionsOnSave": { + "source.organizeImports": "explicit", + "source.generate.finalModifiers": "explicit", + "source.fixAll": "explicit" + }, + "java.codeGeneration.useBlocks": true, + "java.saveActions.organizeImports": true, + "java.sources.organizeImports.starThreshold": 3, + "java.sources.organizeImports.staticStarThreshold": 3, + "java.test.config": { + "vmArgs": [ + "-Djava.util.logging.config.file=src/test/resources/logging.properties", + "-Djavax.xml.parsers.DocumentBuilderFactory=com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl", + "-Dcom.exasol.dockerdb.image=8.27.0" + ] + }, + "sonarlint.connectedMode.project": { + "connectionId": "exasol", + "projectKey": "com.exasol:azure-blob-storage-document-files-virtual-schema" + } } diff --git a/dependencies.md b/dependencies.md index ce6bf53..e8d2be0 100644 --- a/dependencies.md +++ b/dependencies.md @@ -26,12 +26,6 @@ | [Performance Test Recorder Java][23] | [MIT License][24] | | [JaCoCo :: Agent][25] | [EPL-2.0][26] | -## Runtime Dependencies - -| Dependency | License | -| ---------------------------------------------- | -------------------------- | -| [Microsoft Azure Netty HTTP Client Library][4] | [The MIT License (MIT)][5] | - ## Plugin Dependencies | Dependency | License | diff --git a/doc/changes/changelog.md b/doc/changes/changelog.md index cac6d82..1d7d0ad 100644 --- a/doc/changes/changelog.md +++ b/doc/changes/changelog.md @@ -1,5 +1,6 @@ # Changes +* [2.1.0](changes_2.1.0.md) * [2.0.5](changes_2.0.5.md) * [2.0.4](changes_2.0.4.md) * [2.0.3](changes_2.0.3.md) diff --git a/doc/changes/changes_2.1.0.md b/doc/changes/changes_2.1.0.md new file mode 100644 index 0000000..4180904 --- /dev/null +++ b/doc/changes/changes_2.1.0.md @@ -0,0 +1,38 @@ +# Virtual Schema for Document Data in Files on Azure Blob Storage 2.1.0, released 2024-06-17 + +Code name: Configure column names for automatic mapping inference + +## Summary + +This release allows configuring the mapping of column names for the automatic mapping inference in Parquet and CSV files. Before, the virtual schema always converted source column names to `UPPER_SNAKE_CASE` to create the Exasol column names. This is now configurable with EDML property `autoInferenceColumnNames`. This property supports the following values: +* `CONVERT_TO_UPPER_SNAKE_CASE`: Convert column names to `UPPER_SNAKE_CASE` (default). +* `KEEP_ORIGINAL_NAME`: Do not convert column names, use column name from source. + +See the [EDML user guide](https://github.com/exasol/virtual-schema-common-document/blob/main/doc/user_guide/edml_user_guide.md#column-name-conversion) for details. + +## Features + +* #58: Added option to keep original column name for auto inference + +## Dependency Updates + +### Compile Dependency Updates + +* Updated `com.azure:azure-storage-blob:12.25.3` to `12.26.1` +* Updated `com.exasol:virtual-schema-common-document-files:8.0.4` to `8.1.0` +* Updated `org.slf4j:slf4j-jdk14:2.0.12` to `2.0.13` + +### Runtime Dependency Updates + +* Removed `com.azure:azure-core-http-netty:1.14.2` + +### Test Dependency Updates + +* Updated `com.exasol:exasol-test-setup-abstraction-java:2.1.2` to `2.1.4` +* Updated `com.exasol:virtual-schema-common-document-files:8.0.4` to `8.1.0` +* Updated `org.mockito:mockito-core:5.11.0` to `5.12.0` +* Updated `org.testcontainers:junit-jupiter:1.19.7` to `1.19.8` + +### Plugin Dependency Updates + +* Updated `com.exasol:project-keeper-maven-plugin:4.3.2` to `4.3.3` diff --git a/doc/user_guide/user_guide.md b/doc/user_guide/user_guide.md index bdc4e4f..bd0f9d4 100644 --- a/doc/user_guide/user_guide.md +++ b/doc/user_guide/user_guide.md @@ -17,7 +17,7 @@ Next create the Adapter Script: ```sql CREATE OR REPLACE JAVA ADAPTER SCRIPT ADAPTER.AZURE_BLOB_STORAGE_FILES_ADAPTER AS %scriptclass com.exasol.adapter.RequestDispatcher; - %jar /buckets/bfsdefault/default/document-files-virtual-schema-dist-8.0.4-azure-blob-storage-2.0.5.jar; + %jar /buckets/bfsdefault/default/document-files-virtual-schema-dist-8.1.0-azure-blob-storage-2.1.0.jar; / ``` @@ -30,7 +30,7 @@ CREATE OR REPLACE JAVA SET SCRIPT ADAPTER.IMPORT_FROM_AZURE_BLOB_STORAGE_DOCUMEN CONNECTION_NAME VARCHAR(500)) EMITS(...) AS %scriptclass com.exasol.adapter.document.UdfEntryPoint; - %jar /buckets/bfsdefault/default/document-files-virtual-schema-dist-8.0.4-azure-blob-storage-2.0.5.jar; + %jar /buckets/bfsdefault/default/document-files-virtual-schema-dist-8.1.0-azure-blob-storage-2.1.0.jar; / ``` diff --git a/pk_generated_parent.pom b/pk_generated_parent.pom index 4aee705..ef786d9 100644 --- a/pk_generated_parent.pom +++ b/pk_generated_parent.pom @@ -3,7 +3,7 @@ 4.0.0 com.exasol azure-blob-storage-document-files-virtual-schema-generated-parent - 2.0.5 + 2.1.0 pom UTF-8 diff --git a/pom.xml b/pom.xml index 4f8fca1..81e2f45 100644 --- a/pom.xml +++ b/pom.xml @@ -2,12 +2,12 @@ 4.0.0 azure-blob-storage-document-files-virtual-schema - 2.0.5 + 2.1.0 Virtual Schema for document data in files on Azure Blob Storage Adapter for document data access from files from Azure Blob Storage. https://github.com/exasol/azure-blob-storage-document-files-virtual-schema/ - 8.0.4 + 8.1.0 @@ -40,14 +40,7 @@ com.azure azure-storage-blob - 12.25.3 - - - - com.azure - azure-core-http-netty - 1.14.2 - runtime + 12.26.1 @@ -72,13 +65,13 @@ org.mockito mockito-core - 5.11.0 + 5.12.0 test org.testcontainers junit-jupiter - 1.19.7 + 1.19.8 test @@ -102,7 +95,7 @@ com.exasol exasol-test-setup-abstraction-java - 2.1.2 + 2.1.4 test @@ -115,7 +108,7 @@ org.slf4j slf4j-jdk14 - 2.0.12 + 2.0.13 @@ -153,7 +146,7 @@ com.exasol project-keeper-maven-plugin - 4.3.2 + 4.3.3 @@ -188,7 +181,7 @@ azure-blob-storage-document-files-virtual-schema-generated-parent com.exasol - 2.0.5 + 2.1.0 pk_generated_parent.pom diff --git a/src/test/java/com/exasol/adapter/document/files/IntegrationTestSetup.java b/src/test/java/com/exasol/adapter/document/files/IntegrationTestSetup.java index 7b6f818..a036771 100644 --- a/src/test/java/com/exasol/adapter/document/files/IntegrationTestSetup.java +++ b/src/test/java/com/exasol/adapter/document/files/IntegrationTestSetup.java @@ -26,7 +26,7 @@ import jakarta.json.*; public class IntegrationTestSetup implements AutoCloseable { - private static final String ADAPTER_JAR = "document-files-virtual-schema-dist-8.0.4-azure-blob-storage-2.0.5.jar"; + private static final String ADAPTER_JAR = "document-files-virtual-schema-dist-8.1.0-azure-blob-storage-2.1.0.jar"; private final ExasolTestSetup exasolTestSetup; private final Connection exasolConnection; private final Statement exasolStatement; diff --git a/src/test/java/com/exasol/adapter/document/files/abstestsetup/LocalAbsTestSetup.java b/src/test/java/com/exasol/adapter/document/files/abstestsetup/LocalAbsTestSetup.java index f066829..44cfb74 100644 --- a/src/test/java/com/exasol/adapter/document/files/abstestsetup/LocalAbsTestSetup.java +++ b/src/test/java/com/exasol/adapter/document/files/abstestsetup/LocalAbsTestSetup.java @@ -21,7 +21,7 @@ public class LocalAbsTestSetup implements AbsTestSetup { public LocalAbsTestSetup() { // https://mcr.microsoft.com/en-us/product/azure-storage/azurite/tags - this.azuriteContainer = new GenericContainer<>("mcr.microsoft.com/azure-storage/azurite:3.29.0"); + this.azuriteContainer = new GenericContainer<>("mcr.microsoft.com/azure-storage/azurite:3.30.0"); this.azuriteContainer.addExposedPort(PORT_IN_CONTAINER); this.azuriteContainer.start(); createAzuriteBlobServiceClient();