ci: separate great-expectations action package

.github/workflows/gx-plugin.yml

@@ -0,0 +1,89 @@
+name: GX Plugin
+on:
+  push:
+    branches:
+      - master
+    paths:
+      - ".github/workflows/gx-plugin.yml"
+      - "metadata-ingestion-modules/gx-plugin/**"
+      - "metadata-ingestion/**"
+      - "metadata-models/**"
+  pull_request:
+    branches:
+      - master
+    paths:
+      - ".github/**"
+      - "metadata-ingestion-modules/gx-plugin/**"
+      - "metadata-ingestion/**"
+      - "metadata-models/**"
+  release:
+    types: [published]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  gx-plugin:
+    runs-on: ubuntu-latest
+    env:
+      SPARK_VERSION: 3.0.3
+      DATAHUB_TELEMETRY_ENABLED: false
+    strategy:
+      matrix:
+        python-version: ["3.8", "3.10"]
+        include:
+          - python-version: "3.8"
+            extraPythonRequirement: "great-expectations~=0.15.12"
+          - python-version: "3.8"
+            extraPythonRequirement: "great-expectations~=0.16.0"
+          - python-version: "3.10"
+            extraPythonRequirement: "great-expectations~=0.15.12"
+          - python-version: "3.10"
+            extraPythonRequirement: "great-expectations~=0.16.0"
+      fail-fast: false
+    steps:
+      - name: Set up JDK 17
+        uses: actions/setup-java@v3
+        with:
+          distribution: "zulu"
+          java-version: 17
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: "pip"
+      - name: Install dependencies
+        run: ./metadata-ingestion/scripts/install_deps.sh
+      - name: Install GX package and test  (extras ${{ matrix.extraPythonRequirement }})
+        run: ./gradlew -Pextra_pip_requirements='${{ matrix.extraPythonRequirement }}' :metadata-ingestion-modules:gx-plugin:lint :metadata-ingestion-modules:gx-plugin:testQuick
+      - name: pip freeze show list installed
+        if: always()
+        run: source metadata-ingestion-modules/gx-plugin/venv/bin/activate && pip freeze
+      - uses: actions/upload-artifact@v3
+        if: ${{ always() && matrix.python-version == '3.10' && matrix.extraPythonRequirement == 'great-expectations>=0.15.12, <=0.15.50' }}
+        with:
+          name: Test Results (GX Plugin ${{ matrix.python-version}})
+          path: |
+            **/build/reports/tests/test/**
+            **/build/test-results/test/**
+            **/junit.*.xml
+      - name: Upload coverage to Codecov
+        if: always()
+        uses: codecov/codecov-action@v3
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          directory: .
+          fail_ci_if_error: false
+          flags: gx-${{ matrix.python-version }}-${{ matrix.extraPythonRequirement }}
+          name: pytest-gx
+          verbose: true
+
+  event-file:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Upload
+        uses: actions/upload-artifact@v3
+        with:
+          name: Event File
+          path: ${{ github.event_path }}

metadata-ingestion-modules/gx-plugin/.gitignore

@@ -0,0 +1,143 @@
+.envrc
+src/datahub_gx_plugin/__init__.py.bak
+.vscode/
+output
+pvenv36/
+bq_credentials.json
+/tmp
+*.bak
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# Generated classes
+src/datahub/metadata/
+wheels/
+junit.quick.xml

metadata-ingestion-modules/gx-plugin/README.md

@@ -0,0 +1,4 @@
+# Datahub GX Plugin
+
+See the DataHub GX docs for details.
+

metadata-ingestion-modules/gx-plugin/build.gradle

@@ -0,0 +1,129 @@
+plugins {
+    id 'base'
+}
+
+ext {
+  python_executable = 'python3'
+  venv_name = 'venv'
+}
+
+if (!project.hasProperty("extra_pip_requirements")) {
+    ext.extra_pip_requirements = ""
+}
+
+task checkPythonVersion(type: Exec) {
+  commandLine python_executable, '-c', 'import sys; assert sys.version_info >= (3, 8)'
+}
+
+task environmentSetup(type: Exec, dependsOn: checkPythonVersion) {
+  def sentinel_file = "${venv_name}/.venv_environment_sentinel"
+  inputs.file file('setup.py')
+  outputs.file(sentinel_file)
+  commandLine 'bash', '-c',
+      "${python_executable} -m venv ${venv_name} && " +
+      "${venv_name}/bin/python -m pip install --upgrade pip uv wheel 'setuptools>=63.0.0' && " +
+      "touch ${sentinel_file}"
+}
+
+task installPackage(type: Exec, dependsOn: environmentSetup) {
+  def sentinel_file = "${venv_name}/.build_install_package_sentinel"
+  inputs.file file('setup.py')
+  outputs.file(sentinel_file)
+  commandLine 'bash', '-c',
+      "source ${venv_name}/bin/activate && set -x && " +
+          "uv pip install -e . ${extra_pip_requirements} && " +
+          "touch ${sentinel_file}"
+}
+
+task install(dependsOn: [installPackage])
+
+task installDev(type: Exec, dependsOn: [install]) {
+  def sentinel_file = "${venv_name}/.build_install_dev_sentinel"
+  inputs.file file('setup.py')
+  outputs.file(sentinel_file)
+  commandLine 'bash', '-c',
+      "source ${venv_name}/bin/activate && set -x && " +
+      "uv pip install -e .[dev] ${extra_pip_requirements} && " +
+      "touch ${sentinel_file}"
+}
+
+task lint(type: Exec, dependsOn: installDev) {
+  /*
+    The find/sed combo below is a temporary work-around for the following mypy issue with airflow 2.2.0:
+   "venv/lib/python3.8/site-packages/airflow/_vendor/connexion/spec.py:169: error: invalid syntax".
+   */
+  commandLine 'bash', '-c',
+    "source ${venv_name}/bin/activate && set -x && " +
+    "black --check --diff src/ tests/ && " +
+    "isort --check --diff src/ tests/ && " +
+    "flake8 --count --statistics src/ tests/ && " +
+    "mypy --show-traceback --show-error-codes src/ tests/"
+}
+task lintFix(type: Exec, dependsOn: installDev) {
+  commandLine 'bash', '-x', '-c',
+    "source ${venv_name}/bin/activate && " +
+    "black src/ tests/ && " +
+    "isort src/ tests/ && " +
+    "flake8 src/ tests/ && " +
+    "mypy src/ tests/"
+}
+
+task installDevTest(type: Exec, dependsOn: [installDev]) {
+  def sentinel_file = "${venv_name}/.build_install_dev_test_sentinel"
+  inputs.file file('setup.py')
+  outputs.dir("${venv_name}")
+  outputs.file(sentinel_file)
+  commandLine 'bash', '-c',
+      "source ${venv_name}/bin/activate && set -x && " +
+      "uv pip install -e .[dev,integration-tests] ${extra_pip_requirements} && " +
+      "touch ${sentinel_file}"
+}
+
+def testFile = hasProperty('testFile') ? testFile : 'unknown'
+task testSingle(dependsOn: [installDevTest]) {
+  doLast {
+  if (testFile != 'unknown')  {
+  exec {
+    commandLine 'bash', '-x', '-c',
+     "source ${venv_name}/bin/activate && pytest ${testFile}"
+    }
+  } else {
+   throw new GradleException("No file provided. Use -PtestFile=<test_file>")
+    }
+  }
+}
+
+task testQuick(type: Exec, dependsOn: installDevTest) {
+  // We can't enforce the coverage requirements if we run a subset of the tests.
+  inputs.files(project.fileTree(dir: "src/", include: "**/*.py"))
+  inputs.files(project.fileTree(dir: "tests/"))
+  outputs.dir("${venv_name}")
+  commandLine 'bash', '-x', '-c',
+    "source ${venv_name}/bin/activate && pytest -vv --continue-on-collection-errors --junit-xml=junit.quick.xml"
+}
+
+
+task testFull(type: Exec, dependsOn: [testQuick, installDevTest]) {
+  commandLine 'bash', '-x', '-c',
+    "source ${venv_name}/bin/activate && pytest -m 'not slow_integration' -vv --continue-on-collection-errors --junit-xml=junit.full.xml"
+}
+task buildWheel(type: Exec, dependsOn: [environmentSetup]) {
+  commandLine 'bash', '-c', "source ${venv_name}/bin/activate && " +
+  'uv pip install build && RELEASE_VERSION="\${RELEASE_VERSION:-0.0.0.dev1}" RELEASE_SKIP_INSTALL=1 RELEASE_SKIP_UPLOAD=1 ./scripts/release.sh'
+}
+
+task cleanPythonCache(type: Exec) {
+  commandLine 'bash', '-c',
+    "find src -type f -name '*.py[co]' -delete -o -type d -name __pycache__ -delete -o -type d -empty -delete"
+}
+
+build.dependsOn install
+check.dependsOn lint
+check.dependsOn testQuick
+
+clean {
+  delete venv_name
+  delete 'build'
+  delete 'dist'
+}
+clean.dependsOn cleanPythonCache

metadata-ingestion-modules/gx-plugin/pyproject.toml

@@ -0,0 +1,19 @@
+[build-system]
+build-backend = "setuptools.build_meta"
+requires = ["setuptools>=54.0.0", "wheel", "pip>=21.0.0"]
+
+[tool.black]
+extend-exclude = '''
+# A regex preceded with ^/ will apply only to files and directories
+# in the root of the project.
+^/tmp
+'''
+include = '\.pyi?$'
+
+[tool.isort]
+indent = '    '
+profile = 'black'
+sections = 'FUTURE,STDLIB,THIRDPARTY,FIRSTPARTY,LOCALFOLDER'
+
+[tool.pyright]
+extraPaths = ['tests']