Merge branch 'airbytehq:master' into master

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.50.38
+current_version = 0.50.54
 commit = False
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-[a-z]+)?
@@ -10,6 +10,4 @@ serialize =
 
 [bumpversion:file:gradle.properties]
 
-[bumpversion:file:docs/operator-guides/upgrading-airbyte.md]
-
 [bumpversion:file:run-ab-platform.sh]

.devcontainer/java-connectors-generic/devcontainer.json

@@ -1,6 +1,6 @@
 // For format details, see https://aka.ms/devcontainer.json. For config options, see the
 {
-  "name": "Connector Development DevContainer (Generic)",
+  "name": "Java Development DevContainer (Generic)",
 
   "image": "mcr.microsoft.com/devcontainers/java:0-17",
   "features": {

.devcontainer/python-connectors-generic/devcontainer.json

@@ -0,0 +1,65 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the
+{
+  "name": "Python Development DevContainer (Generic)",
+
+  "image": "mcr.microsoft.com/devcontainers/python:0-3.10",
+  "features": {
+    "ghcr.io/devcontainers/features/docker-in-docker": {},
+    "ghcr.io/devcontainers/features/python:1": {
+      "installGradle": true,
+      "version": "3.10",
+      "installTools": true
+    },
+    "ghcr.io/devcontainers-contrib/features/poetry:2": {}
+  },
+
+  // Deterministic order reduces cache busting
+  "overrideFeatureInstallOrder": [
+    "ghcr.io/devcontainers/features/docker-in-docker",
+    "ghcr.io/devcontainers/features/python",
+    "ghcr.io/devcontainers-contrib/features/poetry"
+  ],
+
+  // Configure tool-specific properties.
+  "customizations": {
+    "vscode": {
+      "extensions": [
+        // Python extensions:
+        "charliermarsh.ruff",
+        "matangover.mypy",
+        "ms-python.python",
+        "ms-python.vscode-pylance",
+
+        // Toml support
+        "tamasfe.even-better-toml",
+
+        // Yaml and JSON Schema support:
+        "redhat.vscode-yaml",
+
+        // Contributing:
+        "GitHub.vscode-pull-request-github"
+      ],
+      "settings": {
+        "extensions.ignoreRecommendations": true,
+        "git.openRepositoryInParentFolders": "always"
+      }
+    }
+  },
+
+  // Mark the root directory as 'safe' for git.
+  "initializeCommand": "git config --add safe.directory /workspaces/airbyte",
+
+  // Setup airbyte-ci on the container:
+  "postCreateCommand": "make tools.airbyte-ci-dev.install",
+
+  "containerEnv": {
+    // Deterministic Poetry virtual env location: `./.venv`
+    "POETRY_VIRTUALENVS_IN_PROJECT": "true"
+  }
+
+  // Override to change the directory that the IDE opens by default:
+  // "workspaceFolder": "/workspaces/airbyte"
+
+  // Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root.
+  // "remoteUser": "root"
+}

.github/CODEOWNERS

@@ -9,7 +9,7 @@
 # CDK and Connector Acceptance Tests
 /airbyte-cdk/python @airbytehq/connector-extensibility
 /airbyte-integrations/connector-templates/ @airbytehq/connector-extensibility
-/airbyte-integrations/bases/connector-acceptance-test/ @airbytehq/connector-operations @lazebnyi @oustynova
+/airbyte-integrations/bases/connector-acceptance-test/ @airbytehq/connector-extensibility @lazebnyi @oustynova
 
 # Protocol related items
 /docs/understanding-airbyte/airbyte-protocol.md @airbytehq/protocol-reviewers
@@ -62,7 +62,7 @@ airbyte-cdk/java/airbyte-cdk/typing-deduping/ @airbytehq/destinations
 /airbyte-integrations/connectors/destination-tidb/ @airbytehq/destinations
 
 # Build customization file change
-/airbyte-integrations/connectors/**/build_customization.py @airbytehq/connector-operations
+/airbyte-integrations/connectors/**/build_customization.py @airbytehq/connector-extensibility
 
 # airbyte-ci
-/airbyte-ci @airbytehq/connector-operations
+/airbyte-ci @airbytehq/connector-extensibility

.github/actions/airbyte-ci-requirements/action.yml

@@ -0,0 +1,104 @@
+name: "Get airbyte-ci runner name"
+description: "Runs a given airbyte-ci command with the --ci-requirements flag to get the CI requirements for a given command"
+inputs:
+  runner_type:
+    description: "Type of runner to get requirements for. One of: format, test, nightly, publish"
+    required: true
+  runner_size:
+    description: "One of: format, test, nightly, publish"
+    required: true
+  airbyte_ci_command:
+    description: "airbyte-ci command to get CI requirements for."
+    required: true
+  runner_name_prefix:
+    description: "Prefix of runner name"
+    required: false
+    default: ci-runner-connector
+  github_token:
+    description: "GitHub token"
+    required: true
+  sentry_dsn:
+    description: "Sentry DSN"
+    required: false
+  airbyte_ci_binary_url:
+    description: "URL to airbyte-ci binary"
+    required: false
+    default: https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci
+
+runs:
+  using: "composite"
+  steps:
+    - name: Check if PR is from a fork
+      if: github.event_name == 'pull_request'
+      shell: bash
+      run: |
+        if [ "${{ github.event.pull_request.head.repo.fork }}" == "true" ]; then
+          echo "PR is from a fork. Exiting workflow..."
+          exit 78
+        fi
+
+    - name: Get changed files
+      uses: tj-actions/changed-files@v39
+      id: changes
+      with:
+        files_yaml: |
+          pipelines:
+            - 'airbyte-ci/connectors/pipelines/**'
+
+    - name: Determine how Airbyte CI should be installed
+      shell: bash
+      id: determine-install-mode
+      run: |
+        if [[ "${{ github.ref }}" != "refs/heads/master" ]] && [[ "${{ steps.changes.outputs.pipelines_any_changed }}" == "true" ]]; then
+          echo "Making changes to Airbyte CI on a non-master branch. Airbyte-CI will be installed from source."
+          echo "install-mode=dev" >> $GITHUB_OUTPUT
+        else
+          echo "install-mode=production" >> $GITHUB_OUTPUT
+        fi
+
+    - name: Install airbyte-ci binary
+      id: install-airbyte-ci
+      if: steps.determine-install-mode.outputs.install-mode == 'production'
+      shell: bash
+      run: |
+        curl -sSL ${{ inputs.airbyte_ci_binary_url }} --output airbyte-ci-bin
+        sudo mv airbyte-ci-bin /usr/local/bin/airbyte-ci
+        sudo chmod +x /usr/local/bin/airbyte-ci
+
+    - name: Install Python 3.10
+      uses: actions/setup-python@v4
+      if: steps.determine-install-mode.outputs.install-mode == 'dev'
+      with:
+        python-version: "3.10"
+        token: ${{ inputs.github_token }}
+
+    - name: Install ci-connector-ops package
+      if: steps.determine-install-mode.outputs.install-mode == 'dev'
+      shell: bash
+      run: |
+        pip install pipx
+        pipx ensurepath
+        pipx install airbyte-ci/connectors/pipelines/
+
+    - name: Get dagger version from airbyte-ci
+      id: get-dagger-version
+      shell: bash
+      run: |
+        dagger_version=$(airbyte-ci --disable-update-check ${{ inputs.airbyte_ci_command }} --ci-requirements | tail -n 1 | jq -r '.dagger_version')
+        echo "dagger_version=${dagger_version}" >> "$GITHUB_OUTPUT"
+
+    - name: Get runner name
+      id: get-runner-name
+      shell: bash
+      run: |
+        runner_name_prefix=${{ inputs.runner_name_prefix }}
+        runner_type=${{ inputs.runner_type }}
+        runner_size=${{ inputs.runner_size }}
+        dashed_dagger_version=$(echo "${{ steps.get-dagger-version.outputs.dagger_version }}" | tr '.' '-')
+        runner_name="${runner_name_prefix}-${runner_type}-${runner_size}-dagger-${dashed_dagger_version}"
+        echo ${runner_name}
+        echo "runner_name=${runner_name}" >> "$GITHUB_OUTPUT"
+outputs:
+  runner_name:
+    description: "Name of self hosted CI runner to use"
+    value: ${{ steps.get-runner-name.outputs.runner_name }}

.github/actions/get-dagger-engine-image/action.yml

@@ -0,0 +1,53 @@
+name: "Get Dagger Engine Image"
+description: "Pulls the Dagger Engine Image or load from cache"
+
+inputs:
+  dagger_engine_image:
+    description: "Image name of the Dagger Engine"
+    required: true
+  path_to_dagger_engine_image_cache:
+    description: "Path to the Dagger Engine image cache"
+    required: false
+    default: "/home/runner/dagger-engine-image-cache"
+
+runs:
+  using: "composite"
+  steps:
+    - name: Create local image cache directory
+      id: create-dagger-engine-image-cache-dir
+      shell: bash
+      run: mkdir -p ${{ inputs.path_to_dagger_engine_image_cache }}
+
+    - name: Restore dagger engine image cache
+      id: dagger-engine-image-cache-restore
+      uses: actions/cache/restore@v4
+      with:
+        path: ${{ inputs.path_to_dagger_engine_image_cache }}
+        key: ${{ inputs.dagger_engine_image }}
+
+    # If no GitHub Action cache hit, pull the image and save it locally as tar to the cache directory
+    - name: Pull dagger engine image
+      id: pull-dagger-engine-image
+      if: steps.dagger-engine-image-cache-restore.outputs.cache-hit != 'true'
+      shell: bash
+      run: |
+        set -x
+        docker pull ${{ inputs.dagger_engine_image }}
+        docker save -o ${{ inputs.path_to_dagger_engine_image_cache }}/image.tar ${{ inputs.dagger_engine_image }}
+
+    # If no GitHub Action  cache hit, save the path to the image cache directory to the Github Action cache
+    - name: Save dagger engine image cache
+      id: dagger-engine-image-cache-save
+      if: steps.dagger-engine-image-cache-restore.outputs.cache-hit != 'true'
+      uses: actions/cache/save@v4
+      with:
+        path: ${{ inputs.path_to_dagger_engine_image_cache }}
+        key: ${{ inputs.dagger_engine_image }}
+
+    # If GitHub Action cache hit, load the image tar restored from the cache
+    - name: Load dagger engine image from cache
+      if: steps.dagger-engine-image-cache-restore.outputs.cache-hit == 'true'
+      shell: bash
+      run: |
+        set -x
+        docker load -i ${{ inputs.path_to_dagger_engine_image_cache }}/image.tar

.github/actions/install-airbyte-ci/action.yml

@@ -0,0 +1,81 @@
+name: "Install Airbyte CI"
+description: "Install Airbyte CI from source or from a binary according to changed files. Pulls the Dagger Engine image according to the dagger version used in airbyte-ci."
+
+inputs:
+  airbyte_ci_binary_url:
+    description: "URL to airbyte-ci binary"
+    required: false
+    default: https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci
+  path_to_airbyte_ci_source:
+    description: "Path to airbyte-ci source"
+    required: false
+    default: airbyte-ci/connectors/pipelines
+runs:
+  using: "composite"
+  steps:
+    - name: Get changed files
+      uses: tj-actions/changed-files@v39
+      id: changes
+      with:
+        files_yaml: |
+          pipelines:
+            - '${{ inputs.path_to_airbyte_ci_source }}/**'
+
+    - name: Determine how Airbyte CI should be installed
+      shell: bash
+      id: determine-install-mode
+      run: |
+        if [[ "${{ github.ref }}" != "refs/heads/master" ]] && [[ "${{ steps.changes.outputs.pipelines_any_changed }}" == "true" ]]; then
+          echo "Making changes to Airbyte CI on a non-master branch. Airbyte-CI will be installed from source."
+          echo "install-mode=source" >> $GITHUB_OUTPUT
+        else
+          echo "install-mode=binary" >> $GITHUB_OUTPUT
+        fi
+
+    - name: Install Airbyte CI from binary
+      id: install-airbyte-ci-binary
+      if: steps.determine-install-mode.outputs.install-mode == 'binary'
+      shell: bash
+      run: |
+        curl -sSL ${{ inputs.airbyte_ci_binary_url }} --output airbyte-ci-bin
+        sudo mv airbyte-ci-bin /usr/local/bin/airbyte-ci
+        sudo chmod +x /usr/local/bin/airbyte-ci
+
+    - name: Install Python 3.10
+      id: install-python-3-10
+      uses: actions/setup-python@v4
+      if: steps.determine-install-mode.outputs.install-mode == 'source'
+      with:
+        python-version: "3.10"
+        token: ${{ inputs.github_token }}
+
+    - name: Install Airbyte CI from source
+      id: install-airbyte-ci-source
+      if: steps.determine-install-mode.outputs.install-mode == 'source'
+      shell: bash
+      run: |
+        pip install --upgrade pip
+        pip install pipx
+        pipx ensurepath
+        pipx install ${{ inputs.path_to_airbyte_ci_source }}
+
+    - name: Get dagger engine image name
+      id: get-dagger-engine-image-name
+      shell: bash
+      run: |
+        dagger_engine_image=$(airbyte-ci --ci-requirements | tail -n 1 | jq -r '.dagger_engine_image')
+        echo "dagger_engine_image=${dagger_engine_image}" >> "$GITHUB_OUTPUT"
+
+    - name: Get dagger engine image
+      id: get-dagger-engine-image
+      uses: ./.github/actions/get-dagger-engine-image
+      with:
+        dagger_engine_image: ${{ steps.get-dagger-engine-image-name.outputs.dagger_engine_image }}
+
+outputs:
+  install_mode:
+    description: "Whether Airbyte CI was installed from source or from a binary"
+    value: ${{ steps.determine-install-mode.outputs.install-mode }}
+  dagger_engine_image_name:
+    description: "Dagger engine image name"
+    value: ${{ steps.get-dagger-engine-image-name.outputs.dagger_engine_image }}