More testing, support profiles, fix issue

.github/workflows/Linux.yml

@@ -100,28 +100,4 @@ jobs:
         with:
           name: ${{matrix.arch}}-extensions
           path: |
-            build/release/extension/aws/aws.duckdb_extension
-
-      - name: Deploy
-        env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.S3_DEPLOY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DEPLOY_KEY }}
-          AWS_DEFAULT_REGION: ${{ secrets.S3_REGION }}
-          BUCKET_NAME: ${{ secrets.S3_BUCKET }}
-          DUCKDB_EXTENSION_SIGNING_PK: ${{ secrets.DUCKDB_EXTENSION_SIGNING_KEY }}
-        run: |
-          git config --global --add safe.directory '*'
-          cd duckdb
-          git fetch --tags
-          export DUCKDB_VERSION=`git tag --points-at HEAD`
-          export DUCKDB_VERSION=${DUCKDB_VERSION:=`git log -1 --format=%h`}
-          cd ..
-          if [[ "$AWS_ACCESS_KEY_ID" == "" ]] ; then
-            echo 'No key set, skipping'
-          elif [[ "$GITHUB_REF" =~ ^(refs/tags/v.+)$ ]] ; then
-            python3 -m pip install pip awscli
-            ./scripts/extension-upload.sh aws ${{ github.ref_name }} $DUCKDB_VERSION ${{matrix.arch}} $BUCKET_NAME true
-          elif [[ "$GITHUB_REF" =~ ^(refs/heads/main)$ ]] ; then
-            python3 -m pip install pip awscli
-            ./scripts/extension-upload.sh aws `git log -1 --format=%h` $DUCKDB_VERSION ${{matrix.arch}} $BUCKET_NAME false
-          fi
+            build/release/extension/aws/aws.duckdb_extension

.github/workflows/MinioTests.yml

@@ -0,0 +1,77 @@
+name: Minio Tests
+on: [push, pull_request,repository_dispatch]
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }}
+  cancel-in-progress: true
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  minio-tests:
+    name: Minio Tests
+    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        duckdb_version: [ '<submodule_version>' ]
+    env:
+      S3_TEST_SERVER_AVAILABLE: 1
+      AWS_DEFAULT_REGION: eu-west-1
+      DUCKDB_S3_ENDPOINT: duckdb-minio.com:9000
+      DUCKDB_S3_USE_SSL: false
+      GEN: ninja
+      VCPKG_TARGET_TRIPLET: x64-linux
+      VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          submodules: 'true'
+
+      - name: Checkout DuckDB to version
+        if: ${{ matrix.duckdb_version != '<submodule_version>'}}
+        run: |
+          cd duckdb
+          git checkout ${{ matrix.duckdb_version }}
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.7'
+
+      - name: Install Ninja
+        shell: bash
+        run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build
+
+      - name: Setup Ccache
+        uses: hendrikmuhs/ccache-action@main
+        with:
+          key: ${{ github.job }}
+          save: ${{ github.ref == 'refs/heads/master' || github.repository != 'duckdb/duckdb' }}
+
+      - name: Setup vcpkg
+        uses: lukka/run-vcpkg@v11
+        with:
+          vcpkgGitCommitId: 501db0f17ef6df184fcdbfbe0f87cde2313b6ab1
+
+      - name: Build
+        shell: bash
+        run: make debug
+
+      - name: Start S3/HTTP test server
+        shell: bash
+        run: |
+          cd duckdb
+          sudo ./scripts/install_s3_test_server.sh
+          source ./scripts/run_s3_test_server.sh
+          sleep 30
+
+      - name: Write AWS credentials file
+        shell: bash
+        run: |
+          ./scripts/create_minio_credential_file.sh
+
+      - name: Test
+        shell: bash
+        run: |
+          make test_debug

Makefile

@@ -31,7 +31,14 @@ BUILD_FLAGS=-DEXTENSION_STATIC_BUILD=1 -DBUILD_EXTENSIONS="tpch;httpfs" ${OSX_BU
 CLIENT_FLAGS :=
 
 # These flags will make DuckDB build the extension
-EXTENSION_FLAGS=-DDUCKDB_EXTENSION_NAMES="aws" -DDUCKDB_EXTENSION_AWS_PATH="$(PROJ_DIR)" -DDUCKDB_EXTENSION_AWS_SHOULD_LINK=1 -DDUCKDB_EXTENSION_AWS_INCLUDE_PATH="$(PROJ_DIR)src/include"
+EXTENSION_FLAGS=\
+-DDUCKDB_EXTENSION_NAMES="aws" \
+-DDUCKDB_EXTENSION_AWS_PATH="$(PROJ_DIR)" \
+-DDUCKDB_EXTENSION_AWS_SHOULD_LINK=1 \
+-DDUCKDB_EXTENSION_AWS_LOAD_TESTS=1 \
+-DDUCKDB_EXTENSION_AWS_TEST_PATH="$(PROJ_DIR)test" \
+-DDUCKDB_EXTENSION_AWS_INCLUDE_PATH="$(PROJ_DIR)src/include" \
+
 
 pull:
 	git submodule init
@@ -76,10 +83,10 @@ release_python: release
 test: test_release
 
 test_release: release
-	./build/release/test/unittest --test-dir . "[sql]"
+	./build/release/test/unittest "$(PROJ_DIR)test/*"
 
 test_debug: debug
-	./build/debug/test/unittest --test-dir . "[sql]"
+	./build/debug/test/unittest "$(PROJ_DIR)test/*"
 
 # Client tests
 test_js: test_debug_js

scripts/create_minio_credential_file.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+# Warning: overwrites your existing aws credentials file!
+
+# Set the file path for the credentials file
+credentials_file=~/.aws/credentials
+
+# create dir if not already existend
+mkdir -p ~/.aws
+
+# Create the credentials configuration
+credentials_config="[default]
+aws_access_key_id=minio_duckdb_user
+aws_secret_access_key=minio_duckdb_user_password
+
+[minio-testing-2]
+aws_access_key_id=minio_duckdb_user_2
+aws_secret_access_key=minio_duckdb_user_password_2
+
+[minio-testing-invalid]
+aws_access_key_id=minio_duckdb_user_invalid
+aws_secret_access_key=thispasswordiscompletelywrong
+"
+
+# Write the credentials configuration to the file
+echo "$credentials_config" > "$credentials_file"

src/aws_extension.cpp

@@ -12,17 +12,26 @@
 namespace duckdb {
 
 //! Set the DuckDB AWS Credentials using the DefaultAWSCredentialsProviderChain
-static string TrySetAwsCredentials(DBConfig& config) {
+static string TrySetAwsCredentials(DBConfig& config, const string& profile) {
 	Aws::SDKOptions options;
 	Aws::InitAPI(options);
-	Aws::Auth::DefaultAWSCredentialsProviderChain provider;
-	auto credentials = provider.GetAWSCredentials();
+	Aws::Auth::AWSCredentials credentials;
+
+	if (!profile.empty()) {
+		// The user has specified a specific profile they want to use instead of the current profile specified by the
+		// system
+		Aws::Auth::ProfileConfigFileAWSCredentialsProvider provider(profile.c_str());
+		credentials = provider.GetAWSCredentials();
+	} else {
+		Aws::Auth::DefaultAWSCredentialsProviderChain provider;
+		credentials = provider.GetAWSCredentials();
+	}
 
 	string ret;
 	if (!credentials.IsExpiredOrEmpty()) {
-		config.SetOptionByName("s3_access_key_id", credentials.GetAWSAccessKeyId());
-		config.SetOptionByName("s3_access_key_id", credentials.GetAWSSecretKey());
-		config.SetOptionByName("s3_session_token", credentials.GetSessionToken());
+		config.SetOption("s3_access_key_id", Value(credentials.GetAWSAccessKeyId()));
+		config.SetOption("s3_secret_access_key", Value(credentials.GetAWSSecretKey()));
+		config.SetOption("s3_session_token", Value(credentials.GetSessionToken()));
 		ret = credentials.GetAWSAccessKeyId();
 	}
 
@@ -31,12 +40,18 @@ static string TrySetAwsCredentials(DBConfig& config) {
 }
 
 struct SetAWSCredentialsFunctionData : public TableFunctionData {
+	string profile_name;
 	bool finished = false;
 };
 
 static unique_ptr<FunctionData> LoadAWSCredentialsBind(ClientContext &context, TableFunctionBindInput &input,
                                           vector<LogicalType> &return_types, vector<string> &names) {
 	auto result = make_uniq<SetAWSCredentialsFunctionData>();
+
+	if (input.inputs.size() >= 1) {
+		result->profile_name = input.inputs[0].ToString();
+	}
+
 	return_types.emplace_back(LogicalType::VARCHAR);
 	names.emplace_back("loaded_key");
 	return std::move(result);
@@ -53,7 +68,7 @@ static void LoadAWSCredentialsFun(ClientContext &context, TableFunctionInput &da
 	}
 
 	//! Return the Key ID of the key we found, or NULL if none was found
-	auto key_loaded = TrySetAwsCredentials(DBConfig::GetConfig(context));
+	auto key_loaded = TrySetAwsCredentials(DBConfig::GetConfig(context), data.profile_name);
 	auto ret_val = !key_loaded.empty() ? Value(key_loaded) : Value(nullptr);
 	output.SetValue(0,0,ret_val);
 	output.SetCardinality(1);
@@ -62,8 +77,10 @@ static void LoadAWSCredentialsFun(ClientContext &context, TableFunctionInput &da
 }
 
 static void LoadInternal(DuckDB &db) {
-	auto load_credentials_func = TableFunction("load_aws_credentials", {}, LoadAWSCredentialsFun, LoadAWSCredentialsBind);
-	ExtensionUtil::RegisterFunction(*db.instance, load_credentials_func);
+	TableFunctionSet function_set("load_aws_credentials");
+	function_set.AddFunction(TableFunction("load_aws_credentials", {}, LoadAWSCredentialsFun, LoadAWSCredentialsBind));
+	function_set.AddFunction(TableFunction("load_aws_credentials", {LogicalTypeId::VARCHAR}, LoadAWSCredentialsFun, LoadAWSCredentialsBind));
+	ExtensionUtil::RegisterFunction(*db.instance, function_set);
 }
 
 void AwsExtension::Load(DuckDB &db) {

test/sql/aws_env_var.test

@@ -13,4 +13,32 @@ require-env AWS_SECRET_ACCESS_KEY
 query I
 CALL load_aws_credentials();
 ----
+minio_duckdb_user
+
+query I
+select value from duckdb_settings() where name='s3_secret_access_key';
+----
+minio_duckdb_user_password
+
+query I
+select value from duckdb_settings() where name='s3_access_key_id';
+----
+minio_duckdb_user
+
+# Trying to access a profile that doesn't exist should return NULL and not change anything
+query I
+CALL load_aws_credentials('profile-doesnt-exists-altogether');
+----
+NULL
+
+# Same for passing null as a profile, it does nothing.
+query I
+CALL load_aws_credentials(NULL);
+----
+NULL
+
+# Key is untouched
+query I
+select value from duckdb_settings() where name='s3_access_key_id';
+----
 minio_duckdb_user

test/sql/aws_minio.test

@@ -0,0 +1,103 @@
+# name: test/sql/aws_minio.test
+# description: test aws extension with minio test server
+# group: [aws]
+
+require aws
+
+require httpfs
+
+require-env DUCKDB_S3_ENDPOINT
+
+require-env DUCKDB_S3_USE_SSL
+
+require-env AWS_DEFAULT_REGION
+
+require-env S3_TEST_SERVER_AVAILABLE 1
+
+set ignore_error_messages
+
+# Without params, this will use the DefaultAWSCredentialsProviderChain (https://sdk.amazonaws.com/cpp/api/LATEST/root/html/md_docs_2_credentials___providers.html)
+query I
+CALL load_aws_credentials();
+----
+minio_duckdb_user
+
+query I
+select value from duckdb_settings() where name='s3_secret_access_key';
+----
+minio_duckdb_user_password
+
+query I
+select value from duckdb_settings() where name='s3_access_key_id';
+----
+minio_duckdb_user
+
+# You can specify which config profile to use, this uses the ProfileConfigFileAWSCredentialsProvider directly
+query I
+CALL load_aws_credentials('minio-testing-2');
+----
+minio_duckdb_user_2
+
+query I
+select value from duckdb_settings() where name='s3_secret_access_key';
+----
+minio_duckdb_user_password_2
+
+query I
+select value from duckdb_settings() where name='s3_access_key_id';
+----
+minio_duckdb_user_2
+
+statement ok
+CALL load_aws_credentials();
+
+statement ok
+COPY (select 123 as column) to 's3://test-bucket/test_basic/test.csv';
+
+query I
+SELECT * FROM 's3://test-bucket/test_basic/test.csv';
+----
+123
+
+# Now when we select a failing profile, the query should fail
+query I
+CALL load_aws_credentials('minio-testing-invalid');
+----
+minio_duckdb_user_invalid
+
+statement error
+SELECT * FROM 's3://test-bucket/test_basic/test.csv';
+----
+HTTP 403
+
+# Now resetting to one of the working profiles and it works again
+statement ok
+CALL load_aws_credentials();
+
+query I
+SELECT * FROM 's3://test-bucket/test_basic/test.csv';
+----
+123
+
+# Trying to access a profile that doesn't exist should return NULL and not change anything
+query I
+CALL load_aws_credentials('profile-doesnt-exists-altogether');
+----
+NULL
+
+query I
+CALL load_aws_credentials(NULL);
+----
+NULL
+
+# Key is untouched
+query I
+select value from duckdb_settings() where name='s3_secret_access_key';
+----
+minio_duckdb_user_password
+
+# Query still works
+query I
+SELECT * FROM 's3://test-bucket/test_basic/test.csv';
+----
+123