Skip to content

Commit

Permalink
Merge branch 'master' into feature/support-group-owners-in-dataflows
Browse files Browse the repository at this point in the history
  • Loading branch information
dnks23 authored Sep 1, 2023
2 parents f2db0d5 + 23f24f4 commit b5fa9b9
Show file tree
Hide file tree
Showing 885 changed files with 18,980 additions and 10,275 deletions.
5 changes: 5 additions & 0 deletions .github/actions/docker-custom-build-and-push/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,9 @@ inputs:
# e.g. latest,head,sha12345
description: "List of tags to use for the Docker image"
required: true
target:
description: "Sets the target stage to build"
required: false
outputs:
image_tag:
description: "Docker image tags"
Expand Down Expand Up @@ -62,6 +65,7 @@ runs:
platforms: linux/amd64
build-args: ${{ inputs.build-args }}
tags: ${{ steps.docker_meta.outputs.tags }}
target: ${{ inputs.target }}
load: true
push: false
cache-from: type=registry,ref=${{ steps.docker_meta.outputs.tags }}
Expand Down Expand Up @@ -94,6 +98,7 @@ runs:
platforms: ${{ inputs.platforms }}
build-args: ${{ inputs.build-args }}
tags: ${{ steps.docker_meta.outputs.tags }}
target: ${{ inputs.target }}
push: true
cache-from: type=registry,ref=${{ steps.docker_meta.outputs.tags }}
cache-to: type=inline
Expand Down
85 changes: 85 additions & 0 deletions .github/workflows/airflow-plugin.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
name: Airflow Plugin
on:
push:
branches:
- master
paths:
- ".github/workflows/airflow-plugin.yml"
- "metadata-ingestion-modules/airflow-plugin/**"
- "metadata-ingestion/**"
- "metadata-models/**"
pull_request:
branches:
- master
paths:
- ".github/**"
- "metadata-ingestion-modules/airflow-plugin/**"
- "metadata-ingestion/**"
- "metadata-models/**"
release:
types: [published]

concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true

jobs:
airflow-plugin:
runs-on: ubuntu-latest
env:
SPARK_VERSION: 3.0.3
DATAHUB_TELEMETRY_ENABLED: false
strategy:
matrix:
include:
- python-version: "3.7"
extraPythonRequirement: "apache-airflow~=2.1.0"
- python-version: "3.7"
extraPythonRequirement: "apache-airflow~=2.2.0"
- python-version: "3.10"
extraPythonRequirement: "apache-airflow~=2.4.0"
- python-version: "3.10"
extraPythonRequirement: "apache-airflow~=2.6.0"
- python-version: "3.10"
extraPythonRequirement: "apache-airflow>2.6.0"
fail-fast: false
steps:
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
cache: "pip"
- name: Install dependencies
run: ./metadata-ingestion/scripts/install_deps.sh
- name: Install airflow package and test (extras ${{ matrix.extraPythonRequirement }})
run: ./gradlew -Pextra_pip_requirements='${{ matrix.extraPythonRequirement }}' :metadata-ingestion-modules:airflow-plugin:lint :metadata-ingestion-modules:airflow-plugin:testQuick
- name: pip freeze show list installed
if: always()
run: source metadata-ingestion-modules/airflow-plugin/venv/bin/activate && pip freeze
- uses: actions/upload-artifact@v3
if: ${{ always() && matrix.python-version == '3.10' && matrix.extraPythonRequirement == 'apache-airflow>2.6.0' }}
with:
name: Test Results (Airflow Plugin ${{ matrix.python-version}})
path: |
**/build/reports/tests/test/**
**/build/test-results/test/**
**/junit.*.xml
- name: Upload coverage to Codecov
if: always()
uses: codecov/codecov-action@v3
with:
token: ${{ secrets.CODECOV_TOKEN }}
directory: .
fail_ci_if_error: false
flags: airflow-${{ matrix.python-version }}-${{ matrix.extraPythonRequirement }}
name: pytest-airflow
verbose: true

event-file:
runs-on: ubuntu-latest
steps:
- name: Upload
uses: actions/upload-artifact@v3
with:
name: Event File
path: ${{ github.event_path }}
45 changes: 0 additions & 45 deletions .github/workflows/docker-ingestion-base.yml

This file was deleted.

118 changes: 0 additions & 118 deletions .github/workflows/docker-ingestion.yml

This file was deleted.

Loading

0 comments on commit b5fa9b9

Please sign in to comment.