Release branch 1.6.5

.github/workflows/ci-integration-tests-new.yml

@@ -9,11 +9,42 @@ on:
 jobs:
   test:
     strategy:
+      max-parallel: 1
       matrix:
-        python: ['3.8']
+        python: ['3.10', '3.11']
     runs-on: ubuntu-latest
     name: Functional test
     steps:
+      # - name: Start Clearscape DBS
+      #   shell: bash
+      #   run: |
+      #     curl -X 'PATCH' \
+      #       'https://api.clearscape.teradata.com/environments/dbt-teradata-1' \
+      #       -H 'accept: application/json' \
+      #       -H 'Authorization: Bearer 2OKQYw4jhaCO3T7UBLjYmsoTxs73IG9q4Ps3kZT6DpnKBKO2mLYd4k6QkGuwW9db' \
+      #       -H 'Content-Type: application/json' \
+      #       -d '{
+      #       "operation": "start"
+      #     }'
+
+      #     curl -X 'PATCH' \
+      #       'https://api.clearscape.teradata.com/environments/dbt-teradata-2' \
+      #       -H 'accept: application/json' \
+      #       -H 'Authorization: Bearer 2OKQYw4jhaCO3T7UBLjYmsoTxs73IG9q4Ps3kZT6DpnKBKO2mLYd4k6QkGuwW9db' \
+      #       -H 'Content-Type: application/json' \
+      #       -d '{
+      #       "operation": "start"
+      #     }'
+
+      #     curl -X 'PATCH' \
+      #       'https://api.clearscape.teradata.com/environments/dbt-teradata-3' \
+      #       -H 'accept: application/json' \
+      #       -H 'Authorization: Bearer 2OKQYw4jhaCO3T7UBLjYmsoTxs73IG9q4Ps3kZT6DpnKBKO2mLYd4k6QkGuwW9db' \
+      #       -H 'Content-Type: application/json' \
+      #       -d '{
+      #       "operation": "start"
+      #     }'
+
       - name: Setup Python
         uses: actions/setup-python@v2
         with:
@@ -45,7 +76,9 @@ jobs:
       #     cd md5_20080530/md5/src
 
       #     # prepare bteq script
+
       #     cat << EOF > /tmp/install_md5.bteq
+
       #     .SET EXITONDELAY ON MAXREQTIME 20
       #     .logon $DBT_TERADATA_SERVER_NAME/$DBT_TERADATA_USERNAME,$DBT_TERADATA_PASSWORD
       #     CREATE DATABASE GLOBAL_FUNCTIONS AS PERMANENT = 60e6, SPOOL = 120e6;
@@ -61,29 +94,17 @@ jobs:
       #     DBT_TERADATA_SERVER_NAME: ${{ vars.DBT_TERADATA_SERVER_NAME }}
       #     DBT_TERADATA_USERNAME: ${{ vars.DBT_TERADATA_USERNAME }}
       #     DBT_TERADATA_PASSWORD: ${{ vars.DBT_TERADATA_PASSWORD }}
+
 
       - name: Run pytest tests
         run: |
+          sed -i "s/localhost/$DBT_TERADATA_SERVER_NAME/g" tests/conftest.py
+          sed -i "s/username='dbc'/username='$DBT_TERADATA_USERNAME'/g" tests/conftest.py
+          sed -i "s/password='dbc'/password='$DBT_TERADATA_PASSWORD'/g" tests/conftest.py
           export DBT_TEST_USER_1=test_grants_user1
           export DBT_TEST_USER_2=test_grants_user2
           export DBT_TEST_USER_3=test_grants_user3
-          pytest tests/functional
-      - name: Run pytest tests with existing database
-        run: |
-          export DBT_TEST_USER_1=test_grants_user1
-          export DBT_TEST_USER_2=test_grants_user2
-          export DBT_TEST_USER_3=test_grants_user3
-          cat << EOF > /tmp/pytestdatabase.bteq
-          .SET EXITONDELAY ON MAXREQTIME 20
-          .logon $DBT_TERADATA_SERVER_NAME/$DBT_TERADATA_USERNAME,$DBT_TERADATA_PASSWORD
-          CREATE DATABASE DBT_TEST
-            AS PERMANENT = 60e6, SPOOL = 120e6;
-          .logoff
-          EOF
-          # strip the random suffix from the pytest config
-          sed -i'.original' -e "s/_{prefix}//" tests/conftest.py
-
-          pytest tests/functional
+          pytest -v -k "not TestCurrentTimestamp" tests/functional
         env:
           DBT_TERADATA_SERVER_NAME: ${{ vars.DBT_TERADATA_SERVER_NAME }}
           DBT_TERADATA_USERNAME: ${{ vars.DBT_TERADATA_USERNAME }}
@@ -109,9 +130,9 @@ jobs:
                 retries: 1
               dbt_catalog_test:
                 type: teradata
-                host: $DBT_TERADATA_SERVER_NAME
-                user: $DBT_TERADATA_USERNAME
-                password: $DBT_TERADATA_PASSWORD
+                host: $DBT_TERADATA_SERVER_NAME_1
+                user: $DBT_TERADATA_USERNAME_1
+                password: $DBT_TERADATA_PASSWORD_1
                 logmech: TD2
                 schema: dbt_catalog_test
                 tmode: ANSI
@@ -126,37 +147,59 @@ jobs:
           DBT_TERADATA_SERVER_NAME: ${{ vars.DBT_TERADATA_SERVER_NAME }}
           DBT_TERADATA_USERNAME: ${{ vars.DBT_TERADATA_USERNAME }}
           DBT_TERADATA_PASSWORD: ${{ vars.DBT_TERADATA_PASSWORD }}
+          DBT_TERADATA_SERVER_NAME_1: ${{ vars.DBT_TERADATA_SERVER_NAME_1 }}
+          DBT_TERADATA_USERNAME_1: ${{ vars.DBT_TERADATA_USERNAME_1 }}
+          DBT_TERADATA_PASSWORD_1: ${{ vars.DBT_TERADATA_PASSWORD_1 }}
 
       - name: Run catalog tests
         run: |
-          # set DisableQVCI to false - this will give us data type info for views in DBC.ColumnsJQV
-          sshpass -p root ssh -o StrictHostKeyChecking=no -p 4422 root@localhost  "dbscontrol << EOF
-          M internal 551=false
-          W
-          EOF
-
-          tpareset -y Enable QVCI
-          "
-
-          $GITHUB_WORKSPACE/.github/workflows/scripts/verifyVantageIsRunning.sh
-
           cd $GITHUB_WORKSPACE/test/catalog/with_qvci
           ./run.sh
           cd $GITHUB_WORKSPACE/test/catalog/without_qvci
           ./run.sh
+
       - name: Run nopi tests
         run: |
-          sshpass -p root ssh -o StrictHostKeyChecking=no -p 4422 root@localhost  "dbscontrol << EOF
-          M general 53=N
-          W
-          EOF
+          sed -i "s/hostname=dbt.*/hostname='$DBT_TERADATA_SERVER_NAME_2'/g" tests/conftest.py
+          sed -i "s/username=demo.*/username='$DBT_TERADATA_USERNAME_2'/g" tests/conftest.py
+          sed -i "s/password=dbt.*/password='$DBT_TERADATA_PASSWORD_2'/g" tests/conftest.py
+          pytest tests/functional/adapter/test_nopi.py
+        env:
+          DBT_TERADATA_SERVER_NAME_2: ${{ vars.DBT_TERADATA_SERVER_NAME_2 }}
+          DBT_TERADATA_USERNAME_2: ${{ vars.DBT_TERADATA_USERNAME_2 }}
+          DBT_TERADATA_PASSWORD_2: ${{ vars.DBT_TERADATA_PASSWORD_2 }}
 
-          tpareset -y Disable PrimaryIndexDefault
-          "
+      # - name: Stop Clearscape DBS
+      #   if: always()
+      #   shell: bash
+      #   run: |
+      #     curl -X 'PATCH' \
+      #       'https://api.clearscape.teradata.com/environments/dbt-teradata-1' \
+      #       -H 'accept: application/json' \
+      #       -H 'Authorization: Bearer 2OKQYw4jhaCO3T7UBLjYmsoTxs73IG9q4Ps3kZT6DpnKBKO2mLYd4k6QkGuwW9db' \
+      #       -H 'Content-Type: application/json' \
+      #       -d '{
+      #       "operation": "stop"
+      #     }'
 
-          $GITHUB_WORKSPACE/.github/workflows/scripts/verifyVantageIsRunning.sh
+      #     curl -X 'PATCH' \
+      #       'https://api.clearscape.teradata.com/environments/dbt-teradata-2' \
+      #       -H 'accept: application/json' \
+      #       -H 'Authorization: Bearer 2OKQYw4jhaCO3T7UBLjYmsoTxs73IG9q4Ps3kZT6DpnKBKO2mLYd4k6QkGuwW9db' \
+      #       -H 'Content-Type: application/json' \
+      #       -d '{
+      #       "operation": "stop"
+      #     }'
 
-          pytest tests/functional/adapter/test_nopi.py
+      #     curl -X 'PATCH' \
+      #       'https://api.clearscape.teradata.com/environments/dbt-teradata-3' \
+      #       -H 'accept: application/json' \
+      #       -H 'Authorization: Bearer 2OKQYw4jhaCO3T7UBLjYmsoTxs73IG9q4Ps3kZT6DpnKBKO2mLYd4k6QkGuwW9db' \
+      #       -H 'Content-Type: application/json' \
+      #       -d '{
+      #       "operation": "stop"
+      #     }'
+
       - uses: actions/upload-artifact@v2
         if: ${{ failure() ||  cancelled() }}
         with:

.github/workflows/ci-integration-tests.yml

@@ -10,7 +10,7 @@ jobs:
   test:
     strategy:
       matrix:
-        python: ['3.8', '3.9','3.10', '3.11']
+        python: ['3.8', '3.9', '3.10', '3.11']
     runs-on: macos-12
     name: Functional test
     steps:
@@ -28,7 +28,7 @@ jobs:
           echo "my IP address is: " $(ipconfig getifaddr en0)
           mkdir /tmp/downloads
           cd /tmp/downloads
-          curl -L "https://d289lrf5tw1zls.cloudfront.net/database/teradata-express/VantageExpress17.20_Sles12_20220819081111.7z?Expires=1704067200&Key-Pair-Id=APKAJ3SWQUPWKYVMO2WQ&Signature=$VE_URL_SIGNATURE" \
+          curl -L "https://d289lrf5tw1zls.cloudfront.net/database/teradata-express/VantageExpress17.20_Sles12_20220819081111.7z?Expires=1735689600&Signature=$VE_URL_SIGNATURE&Key-Pair-Id=APKAJ3SWQUPWKYVMO2WQ" \
             --compressed -o ve.7z
         env:
           VE_URL_SIGNATURE: ${{ secrets.VE_URL_SIGNATURE }}
@@ -58,7 +58,7 @@ jobs:
       - name: Install TTU
         shell: bash
         run: |
-          curl -L "https://d289lrf5tw1zls.cloudfront.net/tools/ttu/TeradataToolsAndUtilitiesBase__macosx_x86_64.17.20.08.00.tar.gz?Expires=1704067200&Key-Pair-Id=APKAJ3SWQUPWKYVMO2WQ&Signature=$TTU_URL_SIGNATURE" \
+          curl -L "https://d289lrf5tw1zls.cloudfront.net/tools/ttu/TeradataToolsAndUtilitiesBase__macosx_x86_64.17.20.08.00.tar.gz?Expires=1735689600&Signature=$TTU_URL_SIGNATURE&Key-Pair-Id=APKAJ3SWQUPWKYVMO2WQ" \
             --compressed -o ttu.tar.gz
           tar -xzf ttu.tar.gz
 

dbt/include/teradata/macros/materializations/snapshot/helpers.sql

@@ -0,0 +1,127 @@
+{% macro teradata__snapshot_staging_table(strategy, source_sql, target_relation) -%}
+
+    with snapshot_query as (
+
+        {{ source_sql }}
+
+    ),
+
+    snapshotted_data as (
+
+        select snapshot.*,
+            {{ strategy.unique_key }} as dbt_unique_key
+
+        from {{ target_relation }} as snapshot
+        where dbt_valid_to is null
+
+    ),
+
+    insertions_source_data as (
+
+        select
+            snapshot_query.*,
+            {{ strategy.unique_key }} as dbt_unique_key,
+            {{ strategy.updated_at }} as dbt_updated_at,
+            {{ strategy.updated_at }} as dbt_valid_from,
+            nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as dbt_valid_to,
+            {{ strategy.scd_id }} as dbt_scd_id
+
+        from snapshot_query
+    ),
+
+    updates_source_data as (
+
+        select
+            snapshot_query.*,
+            {{ strategy.unique_key }} as dbt_unique_key,
+            {{ strategy.updated_at }} as dbt_updated_at,
+            {{ strategy.updated_at }} as dbt_valid_from,
+            {{ strategy.updated_at }} as dbt_valid_to
+
+        from snapshot_query
+    ),
+
+    {%- if strategy.invalidate_hard_deletes %}
+
+    deletes_source_data as (
+
+        select
+            snapshot_query.*,
+            {{ strategy.unique_key }} as dbt_unique_key
+        from snapshot_query
+    ),
+    {% endif %}
+
+    insertions as (
+
+        select
+            'insert' as dbt_change_type,
+            source_data.*
+
+        from insertions_source_data as source_data
+        left outer join snapshotted_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key
+        where snapshotted_data.dbt_unique_key is null
+           or (
+                snapshotted_data.dbt_unique_key is not null
+            and (
+                {{ strategy.row_changed }}
+            )
+        )
+
+    ),
+
+    updates as (
+
+        select
+            'update' as dbt_change_type,
+            source_data.*,
+            snapshotted_data.dbt_scd_id
+
+        from updates_source_data as source_data
+        join snapshotted_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key
+        where (
+            {{ strategy.row_changed }}
+        )
+    )
+
+    {%- if strategy.invalidate_hard_deletes -%}
+    ,
+
+    deletes as (
+
+        select
+            'delete' as dbt_change_type,
+            source_data.*,
+            {{ snapshot_get_time() }} as dbt_valid_from,
+            {{ snapshot_get_time() }} as dbt_updated_at,
+            {{ snapshot_get_time() }} as dbt_valid_to,
+            snapshotted_data.dbt_scd_id
+
+        from snapshotted_data
+        left join deletes_source_data as source_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key
+        where source_data.dbt_unique_key is null
+    )
+    {%- endif %}
+
+    select * from insertions
+    union all
+    select * from updates
+    {%- if strategy.invalidate_hard_deletes %}
+    union all
+    select * from deletes
+    {%- endif %}
+
+{%- endmacro %}
+
+{% macro teradata__build_snapshot_table(strategy, sql) %}
+
+    select sbq.*,
+        {{ strategy.scd_id }} as dbt_scd_id,
+        {{ strategy.updated_at }} as dbt_updated_at,
+        {{ strategy.updated_at }} as dbt_valid_from,
+        nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as dbt_valid_to
+    from (
+        {{ sql }}
+    ) sbq
+
+{% endmacro %}