diff --git a/.changes/unreleased/Changed-20240729-145310.yaml b/.changes/unreleased/Changed-20240729-145310.yaml new file mode 100644 index 000000000..338a4a16c --- /dev/null +++ b/.changes/unreleased/Changed-20240729-145310.yaml @@ -0,0 +1,3 @@ +kind: Changed +body: External table authentication to inlcude optional role external id. +time: 2024-07-29T14:53:10.992711+01:00 diff --git a/.github/workflows/code-check.yml b/.github/workflows/code-check.yml index df272117d..ac65a7569 100644 --- a/.github/workflows/code-check.yml +++ b/.github/workflows/code-check.yml @@ -20,7 +20,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install ".[dev]" + python -m pip install ".[dev]" - name: Run pre-commit checks uses: pre-commit/action@v2.0.3 diff --git a/.github/workflows/jaffle-shop-v2.yml b/.github/workflows/jaffle-shop-v2.yml index cebaeb02f..2738a66d8 100644 --- a/.github/workflows/jaffle-shop-v2.yml +++ b/.github/workflows/jaffle-shop-v2.yml @@ -30,7 +30,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install "dbt-firebolt/.[dev]" + python -m pip install "dbt-firebolt/.[dev]" - name: Setup database and engine id: setup @@ -53,6 +53,7 @@ jobs: SECURE_BUCKET_PATH: ${{ vars.SECURE_BUCKET_PATH }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_ACCESS_SECRET_KEY: ${{ secrets.AWS_ACCESS_SECRET_KEY }} + AWS_ACCESS_ROLE_ARN: ${{ secrets.AWS_ACCESS_ROLE_ARN }} DBT_PROFILES_DIR: "../dbt-firebolt/.github/workflows/jaffle_shop" working-directory: jaffle-shop run: diff --git a/.github/workflows/jaffle_shop/run_test_workflow.sh b/.github/workflows/jaffle_shop/run_test_workflow.sh index def2530f9..65915ae25 100755 --- a/.github/workflows/jaffle_shop/run_test_workflow.sh +++ b/.github/workflows/jaffle_shop/run_test_workflow.sh @@ -6,6 +6,11 @@ dbt compile dbt run-operation stage_external_sources cp ../dbt-firebolt/.github/workflows/jaffle_shop/sources_external_tables_id_secret.yml models/staging/sources_external_tables.yml dbt run-operation stage_external_sources --vars "ext_full_refresh: true" +if [[ -n "$AWS_ACCESS_ROLE_ARN" ]]; then + # Can't test this on FB 1.0 + cp ../dbt-firebolt/.github/workflows/jaffle_shop/sources_external_tables_iam.yml models/staging/sources_external_tables.yml + dbt run-operation stage_external_sources --vars "ext_full_refresh: true" +fi dbt seed dbt seed --full-refresh dbt run diff --git a/.github/workflows/jaffle_shop/sources_external_tables_iam.yml b/.github/workflows/jaffle_shop/sources_external_tables_iam.yml new file mode 100644 index 000000000..665eff755 --- /dev/null +++ b/.github/workflows/jaffle_shop/sources_external_tables_iam.yml @@ -0,0 +1,19 @@ +version: 2 + +sources: + - name: s3 + tables: + - name: raw_customers + external: + url: "{{ env_var('SECURE_BUCKET_PATH') }}" + object_pattern: '*raw_customers.csv' + credentials: + internal_role_arn: "{{ env_var('AWS_ACCESS_ROLE_ARN') }}" + type: '(CSV SKIP_HEADER_ROWS=true)' + columns: + - name: id + data_type: int + - name: first_name + data_type: TEXT + - name: last_name + data_type: TEXT diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 21b068f5c..f140c155b 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -57,6 +57,7 @@ jobs: SECURE_BUCKET_PATH: ${{ vars.SECURE_BUCKET_PATH }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_ACCESS_SECRET_KEY: ${{ secrets.AWS_ACCESS_SECRET_KEY }} + AWS_ACCESS_ROLE_ARN: ${{ secrets.AWS_ACCESS_ROLE_ARN }} working-directory: jaffle-shop shell: bash run: | diff --git a/dbt/include/firebolt/macros/dbt_external_tables/create_external_table.sql b/dbt/include/firebolt/macros/dbt_external_tables/create_external_table.sql index 89452229f..cccb7da1e 100644 --- a/dbt/include/firebolt/macros/dbt_external_tables/create_external_table.sql +++ b/dbt/include/firebolt/macros/dbt_external_tables/create_external_table.sql @@ -20,7 +20,10 @@ {% if external.url %} URL = '{{external.url}}' {%- endif %} {%- if credentials and credentials.internal_role_arn %} CREDENTIALS = (AWS_ROLE_ARN = '{{credentials.internal_role_arn}}' - AWS_ROLE_EXTERNAL_ID = '{{credentials.external_role_id}}') + {%- if credentials.external_role_id %} + AWS_ROLE_EXTERNAL_ID = '{{credentials.external_role_id}}' + {%- endif -%} + ) {% elif credentials and credentials.aws_key_id %} CREDENTIALS = (AWS_KEY_ID = '{{credentials.aws_key_id}}' AWS_SECRET_KEY = '{{credentials.aws_secret_key}}')