Skip to content

Commit

Permalink
Merge branch 'airbytehq:master' into master
Browse files Browse the repository at this point in the history
  • Loading branch information
Ruy-Araujo authored Feb 27, 2024
2 parents 60048db + ab2f67a commit 9cf67dc
Show file tree
Hide file tree
Showing 4,331 changed files with 229,338 additions and 140,108 deletions.
The diff you're trying to view is too large. We only load the first 3000 changed files.
4 changes: 1 addition & 3 deletions .bumpversion.cfg
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
[bumpversion]
current_version = 0.50.38
current_version = 0.50.54
commit = False
tag = False
parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-[a-z]+)?
Expand All @@ -10,6 +10,4 @@ serialize =

[bumpversion:file:gradle.properties]

[bumpversion:file:docs/operator-guides/upgrading-airbyte.md]

[bumpversion:file:run-ab-platform.sh]
2 changes: 1 addition & 1 deletion .devcontainer/java-connectors-generic/devcontainer.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
// For format details, see https://aka.ms/devcontainer.json. For config options, see the
{
"name": "Connector Development DevContainer (Generic)",
"name": "Java Development DevContainer (Generic)",

"image": "mcr.microsoft.com/devcontainers/java:0-17",
"features": {
Expand Down
65 changes: 65 additions & 0 deletions .devcontainer/python-connectors-generic/devcontainer.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
// For format details, see https://aka.ms/devcontainer.json. For config options, see the
{
"name": "Python Development DevContainer (Generic)",

"image": "mcr.microsoft.com/devcontainers/python:0-3.10",
"features": {
"ghcr.io/devcontainers/features/docker-in-docker": {},
"ghcr.io/devcontainers/features/python:1": {
"installGradle": true,
"version": "3.10",
"installTools": true
},
"ghcr.io/devcontainers-contrib/features/poetry:2": {}
},

// Deterministic order reduces cache busting
"overrideFeatureInstallOrder": [
"ghcr.io/devcontainers/features/docker-in-docker",
"ghcr.io/devcontainers/features/python",
"ghcr.io/devcontainers-contrib/features/poetry"
],

// Configure tool-specific properties.
"customizations": {
"vscode": {
"extensions": [
// Python extensions:
"charliermarsh.ruff",
"matangover.mypy",
"ms-python.python",
"ms-python.vscode-pylance",

// Toml support
"tamasfe.even-better-toml",

// Yaml and JSON Schema support:
"redhat.vscode-yaml",

// Contributing:
"GitHub.vscode-pull-request-github"
],
"settings": {
"extensions.ignoreRecommendations": true,
"git.openRepositoryInParentFolders": "always"
}
}
},

// Mark the root directory as 'safe' for git.
"initializeCommand": "git config --add safe.directory /workspaces/airbyte",

// Setup airbyte-ci on the container:
"postCreateCommand": "make tools.airbyte-ci-dev.install",

"containerEnv": {
// Deterministic Poetry virtual env location: `./.venv`
"POETRY_VIRTUALENVS_IN_PROJECT": "true"
}

// Override to change the directory that the IDE opens by default:
// "workspaceFolder": "/workspaces/airbyte"

// Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root.
// "remoteUser": "root"
}
6 changes: 3 additions & 3 deletions .github/CODEOWNERS
Validating CODEOWNERS rules …
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
# CDK and Connector Acceptance Tests
/airbyte-cdk/python @airbytehq/connector-extensibility
/airbyte-integrations/connector-templates/ @airbytehq/connector-extensibility
/airbyte-integrations/bases/connector-acceptance-test/ @airbytehq/connector-operations @lazebnyi @oustynova
/airbyte-integrations/bases/connector-acceptance-test/ @airbytehq/connector-extensibility @lazebnyi @oustynova

# Protocol related items
/docs/understanding-airbyte/airbyte-protocol.md @airbytehq/protocol-reviewers
Expand Down Expand Up @@ -62,7 +62,7 @@ airbyte-cdk/java/airbyte-cdk/typing-deduping/ @airbytehq/destinations
/airbyte-integrations/connectors/destination-tidb/ @airbytehq/destinations

# Build customization file change
/airbyte-integrations/connectors/**/build_customization.py @airbytehq/connector-operations
/airbyte-integrations/connectors/**/build_customization.py @airbytehq/connector-extensibility

# airbyte-ci
/airbyte-ci @airbytehq/connector-operations
/airbyte-ci @airbytehq/connector-extensibility
104 changes: 104 additions & 0 deletions .github/actions/airbyte-ci-requirements/action.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
name: "Get airbyte-ci runner name"
description: "Runs a given airbyte-ci command with the --ci-requirements flag to get the CI requirements for a given command"
inputs:
runner_type:
description: "Type of runner to get requirements for. One of: format, test, nightly, publish"
required: true
runner_size:
description: "One of: format, test, nightly, publish"
required: true
airbyte_ci_command:
description: "airbyte-ci command to get CI requirements for."
required: true
runner_name_prefix:
description: "Prefix of runner name"
required: false
default: ci-runner-connector
github_token:
description: "GitHub token"
required: true
sentry_dsn:
description: "Sentry DSN"
required: false
airbyte_ci_binary_url:
description: "URL to airbyte-ci binary"
required: false
default: https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci

runs:
using: "composite"
steps:
- name: Check if PR is from a fork
if: github.event_name == 'pull_request'
shell: bash
run: |
if [ "${{ github.event.pull_request.head.repo.fork }}" == "true" ]; then
echo "PR is from a fork. Exiting workflow..."
exit 78
fi
- name: Get changed files
uses: tj-actions/changed-files@v39
id: changes
with:
files_yaml: |
pipelines:
- 'airbyte-ci/connectors/pipelines/**'
- name: Determine how Airbyte CI should be installed
shell: bash
id: determine-install-mode
run: |
if [[ "${{ github.ref }}" != "refs/heads/master" ]] && [[ "${{ steps.changes.outputs.pipelines_any_changed }}" == "true" ]]; then
echo "Making changes to Airbyte CI on a non-master branch. Airbyte-CI will be installed from source."
echo "install-mode=dev" >> $GITHUB_OUTPUT
else
echo "install-mode=production" >> $GITHUB_OUTPUT
fi
- name: Install airbyte-ci binary
id: install-airbyte-ci
if: steps.determine-install-mode.outputs.install-mode == 'production'
shell: bash
run: |
curl -sSL ${{ inputs.airbyte_ci_binary_url }} --output airbyte-ci-bin
sudo mv airbyte-ci-bin /usr/local/bin/airbyte-ci
sudo chmod +x /usr/local/bin/airbyte-ci
- name: Install Python 3.10
uses: actions/setup-python@v4
if: steps.determine-install-mode.outputs.install-mode == 'dev'
with:
python-version: "3.10"
token: ${{ inputs.github_token }}

- name: Install ci-connector-ops package
if: steps.determine-install-mode.outputs.install-mode == 'dev'
shell: bash
run: |
pip install pipx
pipx ensurepath
pipx install airbyte-ci/connectors/pipelines/
- name: Get dagger version from airbyte-ci
id: get-dagger-version
shell: bash
run: |
dagger_version=$(airbyte-ci --disable-update-check ${{ inputs.airbyte_ci_command }} --ci-requirements | tail -n 1 | jq -r '.dagger_version')
echo "dagger_version=${dagger_version}" >> "$GITHUB_OUTPUT"
- name: Get runner name
id: get-runner-name
shell: bash
run: |
runner_name_prefix=${{ inputs.runner_name_prefix }}
runner_type=${{ inputs.runner_type }}
runner_size=${{ inputs.runner_size }}
dashed_dagger_version=$(echo "${{ steps.get-dagger-version.outputs.dagger_version }}" | tr '.' '-')
runner_name="${runner_name_prefix}-${runner_type}-${runner_size}-dagger-${dashed_dagger_version}"
echo ${runner_name}
echo "runner_name=${runner_name}" >> "$GITHUB_OUTPUT"
outputs:
runner_name:
description: "Name of self hosted CI runner to use"
value: ${{ steps.get-runner-name.outputs.runner_name }}
53 changes: 53 additions & 0 deletions .github/actions/get-dagger-engine-image/action.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
name: "Get Dagger Engine Image"
description: "Pulls the Dagger Engine Image or load from cache"

inputs:
dagger_engine_image:
description: "Image name of the Dagger Engine"
required: true
path_to_dagger_engine_image_cache:
description: "Path to the Dagger Engine image cache"
required: false
default: "/home/runner/dagger-engine-image-cache"

runs:
using: "composite"
steps:
- name: Create local image cache directory
id: create-dagger-engine-image-cache-dir
shell: bash
run: mkdir -p ${{ inputs.path_to_dagger_engine_image_cache }}

- name: Restore dagger engine image cache
id: dagger-engine-image-cache-restore
uses: actions/cache/restore@v4
with:
path: ${{ inputs.path_to_dagger_engine_image_cache }}
key: ${{ inputs.dagger_engine_image }}

# If no GitHub Action cache hit, pull the image and save it locally as tar to the cache directory
- name: Pull dagger engine image
id: pull-dagger-engine-image
if: steps.dagger-engine-image-cache-restore.outputs.cache-hit != 'true'
shell: bash
run: |
set -x
docker pull ${{ inputs.dagger_engine_image }}
docker save -o ${{ inputs.path_to_dagger_engine_image_cache }}/image.tar ${{ inputs.dagger_engine_image }}
# If no GitHub Action cache hit, save the path to the image cache directory to the Github Action cache
- name: Save dagger engine image cache
id: dagger-engine-image-cache-save
if: steps.dagger-engine-image-cache-restore.outputs.cache-hit != 'true'
uses: actions/cache/save@v4
with:
path: ${{ inputs.path_to_dagger_engine_image_cache }}
key: ${{ inputs.dagger_engine_image }}

# If GitHub Action cache hit, load the image tar restored from the cache
- name: Load dagger engine image from cache
if: steps.dagger-engine-image-cache-restore.outputs.cache-hit == 'true'
shell: bash
run: |
set -x
docker load -i ${{ inputs.path_to_dagger_engine_image_cache }}/image.tar
81 changes: 81 additions & 0 deletions .github/actions/install-airbyte-ci/action.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
name: "Install Airbyte CI"
description: "Install Airbyte CI from source or from a binary according to changed files. Pulls the Dagger Engine image according to the dagger version used in airbyte-ci."

inputs:
airbyte_ci_binary_url:
description: "URL to airbyte-ci binary"
required: false
default: https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci
path_to_airbyte_ci_source:
description: "Path to airbyte-ci source"
required: false
default: airbyte-ci/connectors/pipelines
runs:
using: "composite"
steps:
- name: Get changed files
uses: tj-actions/changed-files@v39
id: changes
with:
files_yaml: |
pipelines:
- '${{ inputs.path_to_airbyte_ci_source }}/**'
- name: Determine how Airbyte CI should be installed
shell: bash
id: determine-install-mode
run: |
if [[ "${{ github.ref }}" != "refs/heads/master" ]] && [[ "${{ steps.changes.outputs.pipelines_any_changed }}" == "true" ]]; then
echo "Making changes to Airbyte CI on a non-master branch. Airbyte-CI will be installed from source."
echo "install-mode=source" >> $GITHUB_OUTPUT
else
echo "install-mode=binary" >> $GITHUB_OUTPUT
fi
- name: Install Airbyte CI from binary
id: install-airbyte-ci-binary
if: steps.determine-install-mode.outputs.install-mode == 'binary'
shell: bash
run: |
curl -sSL ${{ inputs.airbyte_ci_binary_url }} --output airbyte-ci-bin
sudo mv airbyte-ci-bin /usr/local/bin/airbyte-ci
sudo chmod +x /usr/local/bin/airbyte-ci
- name: Install Python 3.10
id: install-python-3-10
uses: actions/setup-python@v4
if: steps.determine-install-mode.outputs.install-mode == 'source'
with:
python-version: "3.10"
token: ${{ inputs.github_token }}

- name: Install Airbyte CI from source
id: install-airbyte-ci-source
if: steps.determine-install-mode.outputs.install-mode == 'source'
shell: bash
run: |
pip install --upgrade pip
pip install pipx
pipx ensurepath
pipx install ${{ inputs.path_to_airbyte_ci_source }}
- name: Get dagger engine image name
id: get-dagger-engine-image-name
shell: bash
run: |
dagger_engine_image=$(airbyte-ci --ci-requirements | tail -n 1 | jq -r '.dagger_engine_image')
echo "dagger_engine_image=${dagger_engine_image}" >> "$GITHUB_OUTPUT"
- name: Get dagger engine image
id: get-dagger-engine-image
uses: ./.github/actions/get-dagger-engine-image
with:
dagger_engine_image: ${{ steps.get-dagger-engine-image-name.outputs.dagger_engine_image }}

outputs:
install_mode:
description: "Whether Airbyte CI was installed from source or from a binary"
value: ${{ steps.determine-install-mode.outputs.install-mode }}
dagger_engine_image_name:
description: "Dagger engine image name"
value: ${{ steps.get-dagger-engine-image-name.outputs.dagger_engine_image }}
Loading

0 comments on commit 9cf67dc

Please sign in to comment.