From 30a219a33a97479d1214d7afd727b4d04b28e026 Mon Sep 17 00:00:00 2001
From: Christoph Pirkl <4711730+kaklakariada@users.noreply.github.com>
Date: Mon, 17 Jun 2024 14:14:00 +0200
Subject: [PATCH] #58: Added option to keep original column name for auto
inference (#59)
* #58: Update dependencies
* Upgrade azurite
---
.github/workflows/dependencies_update.yml | 2 +-
.vscode/settings.json | 44 +++++++++----------
dependencies.md | 6 ---
doc/changes/changelog.md | 1 +
doc/changes/changes_2.1.0.md | 38 ++++++++++++++++
doc/user_guide/user_guide.md | 4 +-
pk_generated_parent.pom | 2 +-
pom.xml | 25 ++++-------
.../document/files/IntegrationTestSetup.java | 2 +-
.../files/abstestsetup/LocalAbsTestSetup.java | 2 +-
10 files changed, 75 insertions(+), 51 deletions(-)
create mode 100644 doc/changes/changes_2.1.0.md
diff --git a/.github/workflows/dependencies_update.yml b/.github/workflows/dependencies_update.yml
index 1bf502f..0fa7180 100644
--- a/.github/workflows/dependencies_update.yml
+++ b/.github/workflows/dependencies_update.yml
@@ -75,7 +75,7 @@ jobs:
echo >> "$GITHUB_OUTPUT"
echo '# ⚠️ Notes ⚠️' >> "$GITHUB_OUTPUT"
echo '## Run PK fix manually' >> "$GITHUB_OUTPUT"
- echo 'Due to restrictions workflow `dependencies_update.yml` can't update other workflows, see https://github.com/exasol/project-keeper/issues/578 for details.' >> "$GITHUB_OUTPUT"
+ echo 'Due to restrictions workflow `dependencies_update.yml` cannot update other workflows, see https://github.com/exasol/project-keeper/issues/578 for details.' >> "$GITHUB_OUTPUT"
echo 'Please checkout this PR locally and run `mvn com.exasol:project-keeper-maven-plugin:fix --projects .`' >> "$GITHUB_OUTPUT"
echo '## This PR does not trigger CI workflows' >> "$GITHUB_OUTPUT"
echo 'Please click the **Close pull request** button and then **Reopen pull request** to trigger running checks.' >> "$GITHUB_OUTPUT"
diff --git a/.vscode/settings.json b/.vscode/settings.json
index 768b55c..32d79b8 100644
--- a/.vscode/settings.json
+++ b/.vscode/settings.json
@@ -1,25 +1,23 @@
{
- "editor.formatOnSave": true,
- "editor.codeActionsOnSave": {
- "source.organizeImports": "explicit",
- "source.generate.finalModifiers": "explicit",
- "source.fixAll": "explicit"
- },
- "java.codeGeneration.useBlocks": true,
- "java.saveActions.organizeImports": true,
- "java.sources.organizeImports.starThreshold": 3,
- "java.sources.organizeImports.staticStarThreshold": 3,
- "java.test.config": {
- "vmArgs": [
- "-Djava.util.logging.config.file=src/test/resources/logging.properties",
- "-Djavax.xml.parsers.DocumentBuilderFactory=com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl"
- ]
- },
- "[json]": {
- "editor.indentSize": 2
- },
- "sonarlint.connectedMode.project": {
- "connectionId": "exasol",
- "projectKey": "com.exasol:azure-blob-storage-document-files-virtual-schema"
- }
+ "editor.formatOnSave": true,
+ "editor.codeActionsOnSave": {
+ "source.organizeImports": "explicit",
+ "source.generate.finalModifiers": "explicit",
+ "source.fixAll": "explicit"
+ },
+ "java.codeGeneration.useBlocks": true,
+ "java.saveActions.organizeImports": true,
+ "java.sources.organizeImports.starThreshold": 3,
+ "java.sources.organizeImports.staticStarThreshold": 3,
+ "java.test.config": {
+ "vmArgs": [
+ "-Djava.util.logging.config.file=src/test/resources/logging.properties",
+ "-Djavax.xml.parsers.DocumentBuilderFactory=com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl",
+ "-Dcom.exasol.dockerdb.image=8.27.0"
+ ]
+ },
+ "sonarlint.connectedMode.project": {
+ "connectionId": "exasol",
+ "projectKey": "com.exasol:azure-blob-storage-document-files-virtual-schema"
+ }
}
diff --git a/dependencies.md b/dependencies.md
index ce6bf53..e8d2be0 100644
--- a/dependencies.md
+++ b/dependencies.md
@@ -26,12 +26,6 @@
| [Performance Test Recorder Java][23] | [MIT License][24] |
| [JaCoCo :: Agent][25] | [EPL-2.0][26] |
-## Runtime Dependencies
-
-| Dependency | License |
-| ---------------------------------------------- | -------------------------- |
-| [Microsoft Azure Netty HTTP Client Library][4] | [The MIT License (MIT)][5] |
-
## Plugin Dependencies
| Dependency | License |
diff --git a/doc/changes/changelog.md b/doc/changes/changelog.md
index cac6d82..1d7d0ad 100644
--- a/doc/changes/changelog.md
+++ b/doc/changes/changelog.md
@@ -1,5 +1,6 @@
# Changes
+* [2.1.0](changes_2.1.0.md)
* [2.0.5](changes_2.0.5.md)
* [2.0.4](changes_2.0.4.md)
* [2.0.3](changes_2.0.3.md)
diff --git a/doc/changes/changes_2.1.0.md b/doc/changes/changes_2.1.0.md
new file mode 100644
index 0000000..4180904
--- /dev/null
+++ b/doc/changes/changes_2.1.0.md
@@ -0,0 +1,38 @@
+# Virtual Schema for Document Data in Files on Azure Blob Storage 2.1.0, released 2024-06-17
+
+Code name: Configure column names for automatic mapping inference
+
+## Summary
+
+This release allows configuring the mapping of column names for the automatic mapping inference in Parquet and CSV files. Before, the virtual schema always converted source column names to `UPPER_SNAKE_CASE` to create the Exasol column names. This is now configurable with EDML property `autoInferenceColumnNames`. This property supports the following values:
+* `CONVERT_TO_UPPER_SNAKE_CASE`: Convert column names to `UPPER_SNAKE_CASE` (default).
+* `KEEP_ORIGINAL_NAME`: Do not convert column names, use column name from source.
+
+See the [EDML user guide](https://github.com/exasol/virtual-schema-common-document/blob/main/doc/user_guide/edml_user_guide.md#column-name-conversion) for details.
+
+## Features
+
+* #58: Added option to keep original column name for auto inference
+
+## Dependency Updates
+
+### Compile Dependency Updates
+
+* Updated `com.azure:azure-storage-blob:12.25.3` to `12.26.1`
+* Updated `com.exasol:virtual-schema-common-document-files:8.0.4` to `8.1.0`
+* Updated `org.slf4j:slf4j-jdk14:2.0.12` to `2.0.13`
+
+### Runtime Dependency Updates
+
+* Removed `com.azure:azure-core-http-netty:1.14.2`
+
+### Test Dependency Updates
+
+* Updated `com.exasol:exasol-test-setup-abstraction-java:2.1.2` to `2.1.4`
+* Updated `com.exasol:virtual-schema-common-document-files:8.0.4` to `8.1.0`
+* Updated `org.mockito:mockito-core:5.11.0` to `5.12.0`
+* Updated `org.testcontainers:junit-jupiter:1.19.7` to `1.19.8`
+
+### Plugin Dependency Updates
+
+* Updated `com.exasol:project-keeper-maven-plugin:4.3.2` to `4.3.3`
diff --git a/doc/user_guide/user_guide.md b/doc/user_guide/user_guide.md
index bdc4e4f..bd0f9d4 100644
--- a/doc/user_guide/user_guide.md
+++ b/doc/user_guide/user_guide.md
@@ -17,7 +17,7 @@ Next create the Adapter Script:
```sql
CREATE OR REPLACE JAVA ADAPTER SCRIPT ADAPTER.AZURE_BLOB_STORAGE_FILES_ADAPTER AS
%scriptclass com.exasol.adapter.RequestDispatcher;
- %jar /buckets/bfsdefault/default/document-files-virtual-schema-dist-8.0.4-azure-blob-storage-2.0.5.jar;
+ %jar /buckets/bfsdefault/default/document-files-virtual-schema-dist-8.1.0-azure-blob-storage-2.1.0.jar;
/
```
@@ -30,7 +30,7 @@ CREATE OR REPLACE JAVA SET SCRIPT ADAPTER.IMPORT_FROM_AZURE_BLOB_STORAGE_DOCUMEN
CONNECTION_NAME VARCHAR(500))
EMITS(...) AS
%scriptclass com.exasol.adapter.document.UdfEntryPoint;
- %jar /buckets/bfsdefault/default/document-files-virtual-schema-dist-8.0.4-azure-blob-storage-2.0.5.jar;
+ %jar /buckets/bfsdefault/default/document-files-virtual-schema-dist-8.1.0-azure-blob-storage-2.1.0.jar;
/
```
diff --git a/pk_generated_parent.pom b/pk_generated_parent.pom
index 4aee705..ef786d9 100644
--- a/pk_generated_parent.pom
+++ b/pk_generated_parent.pom
@@ -3,7 +3,7 @@
4.0.0
com.exasol
azure-blob-storage-document-files-virtual-schema-generated-parent
- 2.0.5
+ 2.1.0
pom
UTF-8
diff --git a/pom.xml b/pom.xml
index 4f8fca1..81e2f45 100644
--- a/pom.xml
+++ b/pom.xml
@@ -2,12 +2,12 @@
4.0.0
azure-blob-storage-document-files-virtual-schema
- 2.0.5
+ 2.1.0
Virtual Schema for document data in files on Azure Blob Storage
Adapter for document data access from files from Azure Blob Storage.
https://github.com/exasol/azure-blob-storage-document-files-virtual-schema/
- 8.0.4
+ 8.1.0
@@ -40,14 +40,7 @@
com.azure
azure-storage-blob
- 12.25.3
-
-
-
- com.azure
- azure-core-http-netty
- 1.14.2
- runtime
+ 12.26.1
@@ -72,13 +65,13 @@
org.mockito
mockito-core
- 5.11.0
+ 5.12.0
test
org.testcontainers
junit-jupiter
- 1.19.7
+ 1.19.8
test
@@ -102,7 +95,7 @@
com.exasol
exasol-test-setup-abstraction-java
- 2.1.2
+ 2.1.4
test
@@ -115,7 +108,7 @@
org.slf4j
slf4j-jdk14
- 2.0.12
+ 2.0.13
@@ -153,7 +146,7 @@
com.exasol
project-keeper-maven-plugin
- 4.3.2
+ 4.3.3
@@ -188,7 +181,7 @@
azure-blob-storage-document-files-virtual-schema-generated-parent
com.exasol
- 2.0.5
+ 2.1.0
pk_generated_parent.pom
diff --git a/src/test/java/com/exasol/adapter/document/files/IntegrationTestSetup.java b/src/test/java/com/exasol/adapter/document/files/IntegrationTestSetup.java
index 7b6f818..a036771 100644
--- a/src/test/java/com/exasol/adapter/document/files/IntegrationTestSetup.java
+++ b/src/test/java/com/exasol/adapter/document/files/IntegrationTestSetup.java
@@ -26,7 +26,7 @@
import jakarta.json.*;
public class IntegrationTestSetup implements AutoCloseable {
- private static final String ADAPTER_JAR = "document-files-virtual-schema-dist-8.0.4-azure-blob-storage-2.0.5.jar";
+ private static final String ADAPTER_JAR = "document-files-virtual-schema-dist-8.1.0-azure-blob-storage-2.1.0.jar";
private final ExasolTestSetup exasolTestSetup;
private final Connection exasolConnection;
private final Statement exasolStatement;
diff --git a/src/test/java/com/exasol/adapter/document/files/abstestsetup/LocalAbsTestSetup.java b/src/test/java/com/exasol/adapter/document/files/abstestsetup/LocalAbsTestSetup.java
index f066829..44cfb74 100644
--- a/src/test/java/com/exasol/adapter/document/files/abstestsetup/LocalAbsTestSetup.java
+++ b/src/test/java/com/exasol/adapter/document/files/abstestsetup/LocalAbsTestSetup.java
@@ -21,7 +21,7 @@ public class LocalAbsTestSetup implements AbsTestSetup {
public LocalAbsTestSetup() {
// https://mcr.microsoft.com/en-us/product/azure-storage/azurite/tags
- this.azuriteContainer = new GenericContainer<>("mcr.microsoft.com/azure-storage/azurite:3.29.0");
+ this.azuriteContainer = new GenericContainer<>("mcr.microsoft.com/azure-storage/azurite:3.30.0");
this.azuriteContainer.addExposedPort(PORT_IN_CONTAINER);
this.azuriteContainer.start();
createAzuriteBlobServiceClient();