Skip to content

Commit

Permalink
Merge branch 'master' into master
Browse files Browse the repository at this point in the history
  • Loading branch information
Ruy-Araujo authored Dec 5, 2023
2 parents 2f75b6d + 7863298 commit 8d8b548
Show file tree
Hide file tree
Showing 533 changed files with 11,713 additions and 6,989 deletions.
2 changes: 1 addition & 1 deletion .bumpversion.cfg
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
[bumpversion]
current_version = 0.50.34
current_version = 0.50.35
commit = False
tag = False
parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-[a-z]+)?
Expand Down
11 changes: 10 additions & 1 deletion .github/workflows/airbyte-ci-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,14 +6,18 @@ concurrency:

on:
workflow_dispatch:
inputs:
airbyte_ci_binary_url:
description: "URL to airbyte-ci binary"
required: false
default: https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci
pull_request:
types:
- opened
- reopened
- synchronize
jobs:
run-airbyte-ci-tests:
# Note if you are changing this name you must also change it in the approve-and-merge-dispatch.yml workflow
name: Run Airbyte CI tests
runs-on: "ci-runner-connector-test-large-dagger-0-6-4"
steps:
Expand Down Expand Up @@ -70,6 +74,7 @@ jobs:
sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
subcommand: "test airbyte-ci/connectors/connector_ops"
airbyte_ci_binary_url: ${{ inputs.airbyte_ci_binary_url || 'https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci' }}

- name: Run airbyte-ci/connectors/pipelines tests
id: run-airbyte-ci-connectors-pipelines-tests
Expand All @@ -83,6 +88,7 @@ jobs:
sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
subcommand: "test airbyte-ci/connectors/pipelines"
airbyte_ci_binary_url: ${{ inputs.airbyte_ci_binary_url || 'https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci' }}

- name: Run airbyte-ci/connectors/base_images tests
id: run-airbyte-ci-connectors-base-images-tests
Expand All @@ -96,6 +102,7 @@ jobs:
sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
subcommand: "test airbyte-ci/connectors/base_images"
airbyte_ci_binary_url: ${{ inputs.airbyte_ci_binary_url || 'https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci' }}

- name: Run test pipeline for the metadata lib
id: metadata_lib-test-pipeline
Expand All @@ -107,6 +114,7 @@ jobs:
github_token: ${{ secrets.GITHUB_TOKEN }}
docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
airbyte_ci_binary_url: ${{ inputs.airbyte_ci_binary_url || 'https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci' }}
- name: Run test for the metadata orchestrator
id: metadata_orchestrator-test-pipeline
if: steps.changes.outputs.metadata_orchestrator_any_changed == 'true'
Expand All @@ -117,3 +125,4 @@ jobs:
github_token: ${{ secrets.GITHUB_TOKEN }}
docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
airbyte_ci_binary_url: ${{ inputs.airbyte_ci_binary_url || 'https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci' }}
50 changes: 0 additions & 50 deletions .github/workflows/approve-and-merge-demo-dispatch.yml

This file was deleted.

27 changes: 24 additions & 3 deletions .github/workflows/approve-and-merge-dispatch.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,9 @@ jobs:
id: scd
with:
token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }}
permission: write
permission: admin
issue-type: pull-request
repository: airbytehq/airbyte-cloud
repository: airbytehq/airbyte-platform-internal
dispatch-type: repository
commands: |
approve-and-merge
Expand All @@ -24,4 +24,25 @@ jobs:
with:
comment-id: ${{ github.event.comment.id }}
body: |
> Error: ${{ steps.scd.outputs.error-message }}
> Error!: ${{ steps.scd.outputs.error-message }}
- name: Checkout Airbyte
id: checkout
if: failure() || steps.scd.outputs.error-message
uses: actions/checkout@v2

- name: Run get_repo_admins.sh
if: failure() || steps.scd.outputs.error-message
id: repo_admins
run: |
echo "REPO_ADMINS=$(./tools/bin/get_repo_admins.sh ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }} airbytehq/airbyte)" >> $GITHUB_ENV
- name: Edit comment with repo admins
if: failure() || steps.scd.outputs.error-message
uses: peter-evans/create-or-update-comment@v1
with:
comment-id: ${{ github.event.comment.id }}
body: |
>
> Important: This command can only be run by one of the repository admins:
> ${{ env.REPO_ADMINS }}
2 changes: 1 addition & 1 deletion .github/workflows/connectors_nightly_build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ jobs:
- name: Test connectors
uses: ./.github/actions/run-dagger-pipeline
with:
context: "nightly_builds"
context: "master"
docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }}
Expand Down
5 changes: 5 additions & 0 deletions .github/workflows/connectors_tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,10 @@ on:
runner:
description: "The runner to use for this job"
default: "ci-runner-connector-test-large-dagger-0-6-4"
airbyte_ci_binary_url:
description: "The URL to download the airbyte-ci binary from"
required: false
default: https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci
pull_request:
types:
- opened
Expand Down Expand Up @@ -66,6 +70,7 @@ jobs:
s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
subcommand: "connectors ${{ github.event.inputs.test-connectors-options }} test"
airbyte_ci_binary_url: ${{ github.event.inputs.airbyte_ci_binary_url }}
- name: Test connectors [PULL REQUESTS]
if: github.event_name == 'pull_request'
uses: ./.github/actions/run-dagger-pipeline
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/connectors_weekly_build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ jobs:
- name: Test connectors
uses: ./.github/actions/run-dagger-pipeline
with:
context: "nightly_builds"
context: "master"
ci_job_key: "weekly_alpha_test"
docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
Expand Down
1 change: 0 additions & 1 deletion .github/workflows/format_fix.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ on:
jobs:
format-fix:
runs-on: "ci-runner-connector-format-medium-dagger-0-6-4"
# Note if you are changing this name you must also change it in the approve-and-merge-dispatch.yml workflow
name: "Apply All Formatting Rules"
timeout-minutes: 40
steps:
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/gradle.yml
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,6 @@ jobs:
# In case of self-hosted EC2 errors, removed the `needs` line and switch back to running on ubuntu-latest.
needs: start-check-runner # required to start the main job when the runner is ready
runs-on: ${{ needs.start-check-runner.outputs.label }} # run the job on the newly created runner
# Note if you are changing this name you must also change it in the approve-and-merge-dispatch.yml workflow
name: Gradle Check
timeout-minutes: 30
steps:
Expand All @@ -79,6 +78,7 @@ jobs:
gradlecheck:
- '**/*'
- '!**/*.md'
- '!.github/*'
- uses: actions/setup-java@v3
with:
Expand Down
5 changes: 5 additions & 0 deletions .github/workflows/publish_connectors.yml
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,10 @@ on:
type: string
default: ci-runner-connector-publish-large-dagger-0-6-4
required: true
airbyte-ci-binary-url:
description: "URL to airbyte-ci binary"
required: false
default: https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci
jobs:
publish_connectors:
name: Publish connectors
Expand Down Expand Up @@ -62,6 +66,7 @@ jobs:
s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
subcommand: "connectors ${{ github.event.inputs.connectors-options }} publish ${{ github.event.inputs.publish-options }}"
airbyte_ci_binary_url: ${{ github.event.inputs.airbyte-ci-binary-url }}

set-instatus-incident-on-failure:
name: Create Instatus Incident on Failure
Expand Down
10 changes: 0 additions & 10 deletions .github/workflows/slash-commands.yml
Original file line number Diff line number Diff line change
Expand Up @@ -35,16 +35,6 @@ jobs:
comment-id=${{ github.event.comment.id }}
dispatch-type: workflow

- name: Slash Command Dispatch (Repository)
id: scdr
uses: peter-evans/slash-command-dispatch@v3
with:
token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
permission: write
commands: |
approve-and-merge-demo
dispatch-type: repository

- name: Edit comment with error message
if: steps.scd.outputs.error-message
uses: peter-evans/create-or-update-comment@v1
Expand Down
1 change: 1 addition & 0 deletions CONTRIBUTORS.md
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,7 @@
* [carlonuccio](https://github.com/carlonuccio)
* [catpineapple](https://github.com/catpineapple)
* [cgardens](https://github.com/cgardens)
* [chadthman](https://github.com/chadthman)
* [chandrasekharan98](https://github.com/chandrasekharan98)
* [ChristoGrab](https://github.com/ChristoGrab)
* [ChristopheDuong](https://github.com/ChristopheDuong)
Expand Down
21 changes: 21 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
##@ Makefile

##@ Define the default airbyte-ci version
AIRBYTE_CI_VERSION ?= latest

tools.airbyte-ci.install: ## Install airbyte-ci
@python airbyte-ci/connectors/pipelines/pipelines/external_scripts/airbyte_ci_install.py ${AIRBYTE_CI_VERSION}

tools.airbyte-ci-dev.install: ## Install the development version of airbyte-ci
@python airbyte-ci/connectors/pipelines/pipelines/external_scripts/airbyte_ci_dev_install.py

tools.airbyte-ci.check: # Check if airbyte-ci is installed correctly
@./airbyte-ci/connectors/pipelines/pipelines/external_scripts/airbyte_ci_check.sh

tools.airbyte-ci.clean: ## Clean airbyte-ci installations
@./airbyte-ci/connectors/pipelines/pipelines/external_scripts/airbyte_ci_clean.sh

tools.install: tools.airbyte-ci.install tools.airbyte-ci.check

.PHONY: tools.install tools.airbyte-ci.install tools.airbyte-ci-dev.install tools.airbyte-ci.check tools.airbyte-ci.clean

7 changes: 6 additions & 1 deletion airbyte-cdk/java/airbyte-cdk/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,12 @@ MavenLocal debugging steps:
### Java CDK

| Version | Date | Pull Request | Subject |
| :------ | :--------- | :--------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|:--------|:-----------|:-----------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------|
| 0.6.2 | 2023-11-30 | [\#32573](https://github.com/airbytehq/airbyte/pull/32573) | Update MSSQLConverter to enforce 6-digit microsecond precision for timestamp fields |
| 0.6.1 | 2023-11-30 | [\#32610](https://github.com/airbytehq/airbyte/pull/32610) | Support DB inital sync using binary as primary key. |
| 0.6.0 | 2023-11-30 | [\#32888](https://github.com/airbytehq/airbyte/pull/32888) | JDBC destinations now use the async framework |
| 0.5.3 | 2023-11-28 | [\#32686](https://github.com/airbytehq/airbyte/pull/32686) | Better attribution of debezium engine shutdown due to heartbeat. |
| 0.5.1 | 2023-11-27 | [\#32662](https://github.com/airbytehq/airbyte/pull/32662) | Debezium initialization wait time will now read from initial setup time. |
| 0.5.0 | 2023-11-22 | [\#32656](https://github.com/airbytehq/airbyte/pull/32656) | Introduce TestDatabase test fixture, refactor database source test base classes. |
| 0.4.11 | 2023-11-14 | [\#32526](https://github.com/airbytehq/airbyte/pull/32526) | Clean up memory manager logs. |
| 0.4.10 | 2023-11-13 | [\#32285](https://github.com/airbytehq/airbyte/pull/32285) | Fix UUID codec ordering for MongoDB connector |
Expand Down
14 changes: 7 additions & 7 deletions airbyte-cdk/java/airbyte-cdk/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -29,12 +29,12 @@ subprojects { subproject ->
}
}
// This repository is only defined and used in the context of an artifact publishing
// It's different from the 'airbyte-public-jars' defined in settings.graddle only in its omission
// It's different from the 'airbyte-public-jars' defined in settings.graddle only in its omission
// of the 'public' directory. Any artifacts publish here will be available in the 'airbyte-public-jars' repo
repositories {
maven {
name 'airbyte-repo'
url 'https://airbyte.mycloudrepo.io/repositories/airbyte-public-jars/'
url 'https://airbyte.mycloudrepo.io/repositories/airbyte-public-jars/'
credentials {
username System.getenv('CLOUDREPO_USER')
password System.getenv('CLOUDREPO_PASSWORD')
Expand All @@ -53,10 +53,10 @@ subprojects { subproject ->
description = "Airbyte Connector Development Kit (CDK) for Java."

def recursiveTasks = [
'assemble',
'build',
'integrationTestJava',
'publish',
'assemble',
'build',
'integrationTestJava',
'publish',
'publishToMavenLocal',
'test',
]
Expand Down Expand Up @@ -117,4 +117,4 @@ def cleanLocalCache = tasks.register('cleanLocalCache') {
cleanLocalCache.configure {
dependsOn tasks.named('clean')
dependsOn subprojects.collect { it.tasks.named('clean') }
}
}
2 changes: 1 addition & 1 deletion airbyte-cdk/java/airbyte-cdk/core/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ dependencies {
implementation 'org.projectlombok:lombok:1.18.20'
annotationProcessor 'org.projectlombok:lombok:1.18.20'
testFixturesImplementation 'org.projectlombok:lombok:1.18.20'
testFixturesAnnotationProcessor 'org.projectlombok:lombok:1.18.20'
testFixturesAnnotationProcessor 'org.projectlombok:lombok:1.18.20'

testImplementation libs.junit.jupiter.system.stubs

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -222,7 +222,7 @@ protected void setString(final PreparedStatement preparedStatement, final int pa
}

protected void setBinary(final PreparedStatement preparedStatement, final int parameterIndex, final String value) throws SQLException {
preparedStatement.setBytes(parameterIndex, DatatypeConverter.parseHexBinary(value));
preparedStatement.setBytes(parameterIndex, DatatypeConverter.parseBase64Binary(value));
}

protected <ObjectType> ObjectType getObject(final ResultSet resultSet, final int index, final Class<ObjectType> clazz) throws SQLException {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import com.fasterxml.jackson.databind.JsonNode;
import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer;
import io.airbyte.cdk.integrations.base.Destination;
import io.airbyte.cdk.integrations.base.SerializedAirbyteMessageConsumer;
import io.airbyte.protocol.models.v0.AirbyteConnectionStatus;
import io.airbyte.protocol.models.v0.AirbyteMessage;
import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog;
Expand Down Expand Up @@ -41,4 +42,12 @@ public AirbyteMessageConsumer getConsumer(final JsonNode config,
return destination.getConsumer(config, catalog, outputRecordCollector);
}

@Override
public SerializedAirbyteMessageConsumer getSerializedMessageConsumer(final JsonNode config,
final ConfiguredAirbyteCatalog catalog,
final Consumer<AirbyteMessage> outputRecordCollector)
throws Exception {
return destination.getSerializedMessageConsumer(config, catalog, outputRecordCollector);
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -11,4 +11,6 @@
* {@link io.airbyte.cdk.integrations.destination.buffered_stream_consumer.OnCloseFunction}.
* Separately out for easier versioning.
*/
public interface OnCloseFunction extends Consumer<Boolean> {}
public interface OnCloseFunction extends Consumer<Boolean> {

}
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,15 @@ public PartialAirbyteMessage withState(final PartialAirbyteStateMessage state) {
return this;
}

/**
* For record messages, this stores the serialized data blob (i.e.
* {@code Jsons.serialize(message.getRecord().getData())}). For state messages, this stores the
* _entire_ message (i.e. {@code Jsons.serialize(message)}).
* <p>
* See
* {@link io.airbyte.cdk.integrations.destination_async.AsyncStreamConsumer#deserializeAirbyteMessage(String)}
* for the exact logic of how this field is populated.
*/
@JsonProperty("serialized")
public String getSerialized() {
return serialized;
Expand Down
Original file line number Diff line number Diff line change
@@ -1 +1 @@
version=0.5.0
version=0.6.2
Loading

0 comments on commit 8d8b548

Please sign in to comment.