Merge branch 'master' into master

Ruy-Araujo · Dec 5, 2023 · 8d8b548 · 8d8b548
2 parents 2f75b6d + 7863298
commit 8d8b548
Show file tree

Hide file tree

Showing 533 changed files with 11,713 additions and 6,989 deletions.
diff --git a/.bumpversion.cfg b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.50.34
+current_version = 0.50.35
 commit = False
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-[a-z]+)?

diff --git a/.github/workflows/airbyte-ci-tests.yml b/.github/workflows/airbyte-ci-tests.yml
@@ -6,14 +6,18 @@ concurrency:
 
 on:
   workflow_dispatch:
+    inputs:
+      airbyte_ci_binary_url:
+        description: "URL to airbyte-ci binary"
+        required: false
+        default: https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci
   pull_request:
     types:
       - opened
       - reopened
       - synchronize
 jobs:
   run-airbyte-ci-tests:
-    # Note if you are changing this name you must also change it in the approve-and-merge-dispatch.yml workflow
     name: Run Airbyte CI tests
     runs-on: "ci-runner-connector-test-large-dagger-0-6-4"
     steps:
@@ -70,6 +74,7 @@ jobs:
           sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
           github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
           subcommand: "test airbyte-ci/connectors/connector_ops"
+          airbyte_ci_binary_url: ${{ inputs.airbyte_ci_binary_url || 'https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci' }}
 
       - name: Run airbyte-ci/connectors/pipelines tests
         id: run-airbyte-ci-connectors-pipelines-tests
@@ -83,6 +88,7 @@ jobs:
           sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
           github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
           subcommand: "test airbyte-ci/connectors/pipelines"
+          airbyte_ci_binary_url: ${{ inputs.airbyte_ci_binary_url || 'https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci' }}
 
       - name: Run airbyte-ci/connectors/base_images tests
         id: run-airbyte-ci-connectors-base-images-tests
@@ -96,6 +102,7 @@ jobs:
           sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
           github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
           subcommand: "test airbyte-ci/connectors/base_images"
+          airbyte_ci_binary_url: ${{ inputs.airbyte_ci_binary_url || 'https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci' }}
 
       - name: Run test pipeline for the metadata lib
         id: metadata_lib-test-pipeline
@@ -107,6 +114,7 @@ jobs:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
           docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
+          airbyte_ci_binary_url: ${{ inputs.airbyte_ci_binary_url || 'https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci' }}
       - name: Run test for the metadata orchestrator
         id: metadata_orchestrator-test-pipeline
         if: steps.changes.outputs.metadata_orchestrator_any_changed == 'true'
@@ -117,3 +125,4 @@ jobs:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
           docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
+          airbyte_ci_binary_url: ${{ inputs.airbyte_ci_binary_url || 'https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci' }}
diff --git a/.github/workflows/approve-and-merge-demo-dispatch.yml b/.github/workflows/approve-and-merge-demo-dispatch.yml
diff --git a/.github/workflows/approve-and-merge-dispatch.yml b/.github/workflows/approve-and-merge-dispatch.yml
@@ -11,9 +11,9 @@ jobs:
         id: scd
         with:
           token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }}
-          permission: write
+          permission: admin
           issue-type: pull-request
-          repository: airbytehq/airbyte-cloud
+          repository: airbytehq/airbyte-platform-internal
           dispatch-type: repository
           commands: |
             approve-and-merge
@@ -24,4 +24,25 @@ jobs:
         with:
           comment-id: ${{ github.event.comment.id }}
           body: |
-            > Error: ${{ steps.scd.outputs.error-message }}
+            > Error!: ${{ steps.scd.outputs.error-message }}
+
+      - name: Checkout Airbyte
+        id: checkout
+        if: failure() || steps.scd.outputs.error-message
+        uses: actions/checkout@v2
+
+      - name: Run get_repo_admins.sh
+        if: failure() || steps.scd.outputs.error-message
+        id: repo_admins
+        run: |
+          echo "REPO_ADMINS=$(./tools/bin/get_repo_admins.sh ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }} airbytehq/airbyte)" >> $GITHUB_ENV
+
+      - name: Edit comment with repo admins
+        if: failure() || steps.scd.outputs.error-message
+        uses: peter-evans/create-or-update-comment@v1
+        with:
+          comment-id: ${{ github.event.comment.id }}
+          body: |
+            >
+            > Important: This command can only be run by one of the repository admins:
+            > ${{ env.REPO_ADMINS }}
diff --git a/.github/workflows/connectors_nightly_build.yml b/.github/workflows/connectors_nightly_build.yml
@@ -34,7 +34,7 @@ jobs:
       - name: Test connectors
         uses: ./.github/actions/run-dagger-pipeline
         with:
-          context: "nightly_builds"
+          context: "master"
           docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
           docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
           gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }}

diff --git a/.github/workflows/connectors_tests.yml b/.github/workflows/connectors_tests.yml
@@ -20,6 +20,10 @@ on:
       runner:
         description: "The runner to use for this job"
         default: "ci-runner-connector-test-large-dagger-0-6-4"
+      airbyte_ci_binary_url:
+        description: "The URL to download the airbyte-ci binary from"
+        required: false
+        default: https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci
   pull_request:
     types:
       - opened
@@ -66,6 +70,7 @@ jobs:
           s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
           s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
           subcommand: "connectors ${{ github.event.inputs.test-connectors-options }} test"
+          airbyte_ci_binary_url: ${{ github.event.inputs.airbyte_ci_binary_url }}
       - name: Test connectors [PULL REQUESTS]
         if: github.event_name == 'pull_request'
         uses: ./.github/actions/run-dagger-pipeline

diff --git a/.github/workflows/connectors_weekly_build.yml b/.github/workflows/connectors_weekly_build.yml
@@ -34,7 +34,7 @@ jobs:
       - name: Test connectors
         uses: ./.github/actions/run-dagger-pipeline
         with:
-          context: "nightly_builds"
+          context: "master"
           ci_job_key: "weekly_alpha_test"
           docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
           docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}

diff --git a/.github/workflows/format_fix.yml b/.github/workflows/format_fix.yml
@@ -11,7 +11,6 @@ on:
 jobs:
   format-fix:
     runs-on: "ci-runner-connector-format-medium-dagger-0-6-4"
-    # Note if you are changing this name you must also change it in the approve-and-merge-dispatch.yml workflow
     name: "Apply All Formatting Rules"
     timeout-minutes: 40
     steps:

diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml
@@ -54,7 +54,6 @@ jobs:
     # In case of self-hosted EC2 errors, removed the `needs` line and switch back to running on ubuntu-latest.
     needs: start-check-runner # required to start the main job when the runner is ready
     runs-on: ${{ needs.start-check-runner.outputs.label }} # run the job on the newly created runner
-    # Note if you are changing this name you must also change it in the approve-and-merge-dispatch.yml workflow
     name: Gradle Check
     timeout-minutes: 30
     steps:
@@ -79,6 +78,7 @@ jobs:
             gradlecheck:
               - '**/*'
               - '!**/*.md'
+              - '!.github/*'
 
       - uses: actions/setup-java@v3
         with:

diff --git a/.github/workflows/publish_connectors.yml b/.github/workflows/publish_connectors.yml
@@ -18,6 +18,10 @@ on:
         type: string
         default: ci-runner-connector-publish-large-dagger-0-6-4
         required: true
+      airbyte-ci-binary-url:
+        description: "URL to airbyte-ci binary"
+        required: false
+        default: https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci
 jobs:
   publish_connectors:
     name: Publish connectors
@@ -62,6 +66,7 @@ jobs:
           s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
           s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
           subcommand: "connectors ${{ github.event.inputs.connectors-options }} publish ${{ github.event.inputs.publish-options }}"
+          airbyte_ci_binary_url: ${{ github.event.inputs.airbyte-ci-binary-url }}
 
   set-instatus-incident-on-failure:
     name: Create Instatus Incident on Failure

diff --git a/.github/workflows/slash-commands.yml b/.github/workflows/slash-commands.yml
@@ -35,16 +35,6 @@ jobs:
             comment-id=${{ github.event.comment.id }}
           dispatch-type: workflow
 
-      - name: Slash Command Dispatch (Repository)
-        id: scdr
-        uses: peter-evans/slash-command-dispatch@v3
-        with:
-          token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
-          permission: write
-          commands: |
-            approve-and-merge-demo
-          dispatch-type: repository
-
       - name: Edit comment with error message
         if: steps.scd.outputs.error-message
         uses: peter-evans/create-or-update-comment@v1

diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md
@@ -108,6 +108,7 @@
 * [carlonuccio](https://github.com/carlonuccio)
 * [catpineapple](https://github.com/catpineapple)
 * [cgardens](https://github.com/cgardens)
+* [chadthman](https://github.com/chadthman)
 * [chandrasekharan98](https://github.com/chandrasekharan98)
 * [ChristoGrab](https://github.com/ChristoGrab)
 * [ChristopheDuong](https://github.com/ChristopheDuong)

diff --git a/Makefile b/Makefile
@@ -0,0 +1,21 @@
+##@ Makefile
+
+##@ Define the default airbyte-ci version
+AIRBYTE_CI_VERSION ?= latest
+
+tools.airbyte-ci.install: ## Install airbyte-ci
+	@python airbyte-ci/connectors/pipelines/pipelines/external_scripts/airbyte_ci_install.py ${AIRBYTE_CI_VERSION}
+
+tools.airbyte-ci-dev.install: ## Install the development version of airbyte-ci
+	@python airbyte-ci/connectors/pipelines/pipelines/external_scripts/airbyte_ci_dev_install.py
+
+tools.airbyte-ci.check: # Check if airbyte-ci is installed correctly
+	@./airbyte-ci/connectors/pipelines/pipelines/external_scripts/airbyte_ci_check.sh
+
+tools.airbyte-ci.clean: ## Clean airbyte-ci installations
+	@./airbyte-ci/connectors/pipelines/pipelines/external_scripts/airbyte_ci_clean.sh
+
+tools.install: tools.airbyte-ci.install tools.airbyte-ci.check
+
+.PHONY: tools.install tools.airbyte-ci.install tools.airbyte-ci-dev.install tools.airbyte-ci.check tools.airbyte-ci.clean
+
diff --git a/airbyte-cdk/java/airbyte-cdk/README.md b/airbyte-cdk/java/airbyte-cdk/README.md
@@ -155,7 +155,12 @@ MavenLocal debugging steps:
 ### Java CDK
 
 | Version | Date       | Pull Request                                               | Subject                                                                                                                                                        |
-| :------ | :--------- | :--------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+|:--------|:-----------|:-----------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| 0.6.2   | 2023-11-30 | [\#32573](https://github.com/airbytehq/airbyte/pull/32573) | Update MSSQLConverter to enforce 6-digit microsecond precision for timestamp fields                                                                            |
+| 0.6.1   | 2023-11-30 | [\#32610](https://github.com/airbytehq/airbyte/pull/32610) | Support DB inital sync using binary as primary key.                                                                                                            |
+| 0.6.0   | 2023-11-30 | [\#32888](https://github.com/airbytehq/airbyte/pull/32888) | JDBC destinations now use the async framework                                                                                                                  |
+| 0.5.3   | 2023-11-28 | [\#32686](https://github.com/airbytehq/airbyte/pull/32686) | Better attribution of debezium engine shutdown due to heartbeat.                                                                                               |
+| 0.5.1   | 2023-11-27 | [\#32662](https://github.com/airbytehq/airbyte/pull/32662) | Debezium initialization wait time will now read from initial setup time.                                                                                       |
 | 0.5.0   | 2023-11-22 | [\#32656](https://github.com/airbytehq/airbyte/pull/32656) | Introduce TestDatabase test fixture, refactor database source test base classes.                                                                               |
 | 0.4.11  | 2023-11-14 | [\#32526](https://github.com/airbytehq/airbyte/pull/32526) | Clean up memory manager logs.                                                                                                                                  |
 | 0.4.10  | 2023-11-13 | [\#32285](https://github.com/airbytehq/airbyte/pull/32285) | Fix UUID codec ordering for MongoDB connector                                                                                                                  |

diff --git a/airbyte-cdk/java/airbyte-cdk/build.gradle b/airbyte-cdk/java/airbyte-cdk/build.gradle
@@ -29,12 +29,12 @@ subprojects { subproject ->
             }
         }
         // This repository is only defined and used in the context of an artifact publishing
-        // It's different from the 'airbyte-public-jars' defined in settings.graddle only in its omission 
+        // It's different from the 'airbyte-public-jars' defined in settings.graddle only in its omission
         // of the 'public' directory. Any artifacts publish here will be available in the 'airbyte-public-jars' repo
         repositories {
             maven {
                 name 'airbyte-repo'
-                url 'https://airbyte.mycloudrepo.io/repositories/airbyte-public-jars/' 
+                url 'https://airbyte.mycloudrepo.io/repositories/airbyte-public-jars/'
                 credentials {
                     username System.getenv('CLOUDREPO_USER')
                     password System.getenv('CLOUDREPO_PASSWORD')
@@ -53,10 +53,10 @@ subprojects { subproject ->
 description = "Airbyte Connector Development Kit (CDK) for Java."
 
 def recursiveTasks = [
-    'assemble', 
-    'build', 
-    'integrationTestJava', 
-    'publish', 
+    'assemble',
+    'build',
+    'integrationTestJava',
+    'publish',
     'publishToMavenLocal',
     'test',
 ]
@@ -117,4 +117,4 @@ def cleanLocalCache = tasks.register('cleanLocalCache') {
 cleanLocalCache.configure {
     dependsOn tasks.named('clean')
     dependsOn subprojects.collect { it.tasks.named('clean') }
-}
+}
diff --git a/airbyte-cdk/java/airbyte-cdk/core/build.gradle b/airbyte-cdk/java/airbyte-cdk/core/build.gradle
@@ -90,7 +90,7 @@ dependencies {
     implementation 'org.projectlombok:lombok:1.18.20'
     annotationProcessor 'org.projectlombok:lombok:1.18.20'
     testFixturesImplementation 'org.projectlombok:lombok:1.18.20'
-    testFixturesAnnotationProcessor 'org.projectlombok:lombok:1.18.20' 
+    testFixturesAnnotationProcessor 'org.projectlombok:lombok:1.18.20'
 
     testImplementation libs.junit.jupiter.system.stubs
 

diff --git a/...cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/AbstractJdbcCompatibleSourceOperations.java b/...cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/AbstractJdbcCompatibleSourceOperations.java
@@ -222,7 +222,7 @@ protected void setString(final PreparedStatement preparedStatement, final int pa
   }
 
   protected void setBinary(final PreparedStatement preparedStatement, final int parameterIndex, final String value) throws SQLException {
-    preparedStatement.setBytes(parameterIndex, DatatypeConverter.parseHexBinary(value));
+    preparedStatement.setBytes(parameterIndex, DatatypeConverter.parseBase64Binary(value));
   }
 
   protected <ObjectType> ObjectType getObject(final ResultSet resultSet, final int index, final Class<ObjectType> clazz) throws SQLException {

diff --git a/...ain/java/io/airbyte/cdk/integrations/base/spec_modification/SpecModifyingDestination.java b/...ain/java/io/airbyte/cdk/integrations/base/spec_modification/SpecModifyingDestination.java
@@ -7,6 +7,7 @@
 import com.fasterxml.jackson.databind.JsonNode;
 import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer;
 import io.airbyte.cdk.integrations.base.Destination;
+import io.airbyte.cdk.integrations.base.SerializedAirbyteMessageConsumer;
 import io.airbyte.protocol.models.v0.AirbyteConnectionStatus;
 import io.airbyte.protocol.models.v0.AirbyteMessage;
 import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog;
@@ -41,4 +42,12 @@ public AirbyteMessageConsumer getConsumer(final JsonNode config,
     return destination.getConsumer(config, catalog, outputRecordCollector);
   }
 
+  @Override
+  public SerializedAirbyteMessageConsumer getSerializedMessageConsumer(final JsonNode config,
+                                                                       final ConfiguredAirbyteCatalog catalog,
+                                                                       final Consumer<AirbyteMessage> outputRecordCollector)
+      throws Exception {
+    return destination.getSerializedMessageConsumer(config, catalog, outputRecordCollector);
+  }
+
 }
diff --git a/...cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/OnCloseFunction.java b/...cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/OnCloseFunction.java
@@ -11,4 +11,6 @@
  * {@link io.airbyte.cdk.integrations.destination.buffered_stream_consumer.OnCloseFunction}.
  * Separately out for easier versioning.
  */
-public interface OnCloseFunction extends Consumer<Boolean> {}
+public interface OnCloseFunction extends Consumer<Boolean> {
+
+}
diff --git a/...io/airbyte/cdk/integrations/destination_async/partial_messages/PartialAirbyteMessage.java b/...io/airbyte/cdk/integrations/destination_async/partial_messages/PartialAirbyteMessage.java
@@ -71,6 +71,15 @@ public PartialAirbyteMessage withState(final PartialAirbyteStateMessage state) {
     return this;
   }
 
+  /**
+   * For record messages, this stores the serialized data blob (i.e.
+   * {@code Jsons.serialize(message.getRecord().getData())}). For state messages, this stores the
+   * _entire_ message (i.e. {@code Jsons.serialize(message)}).
+   * <p>
+   * See
+   * {@link io.airbyte.cdk.integrations.destination_async.AsyncStreamConsumer#deserializeAirbyteMessage(String)}
+   * for the exact logic of how this field is populated.
+   */
   @JsonProperty("serialized")
   public String getSerialized() {
     return serialized;

diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties b/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties
@@ -1 +1 @@
-version=0.5.0
+version=0.6.2