fix(ingestion/prefect-plugin): fixed the unit tests

.github/workflows/build-and-test.yml

@@ -91,6 +91,8 @@ jobs:
             -x :metadata-ingestion-modules:airflow-plugin:check \
             -x :metadata-ingestion-modules:dagster-plugin:build \
             -x :metadata-ingestion-modules:dagster-plugin:check \
+            -x :metadata-ingestion-modules:prefect-plugin:build \
+            -x :metadata-ingestion-modules:prefect-plugin:check \
             -x :metadata-ingestion-modules:gx-plugin:build \
             -x :metadata-ingestion-modules:gx-plugin:check \
             -x :datahub-frontend:build \
@@ -138,4 +140,4 @@ jobs:
         uses: actions/upload-artifact@v3
         with:
           name: Event File
-          path: ${{ github.event_path }}
+          path: ${{ github.event_path }}

.github/workflows/prefect-plugin.yml

@@ -0,0 +1,86 @@
+name: Prefect Plugin
+on:
+  push:
+    branches:
+      - master
+    paths:
+      - ".github/workflows/prefect-plugin.yml"
+      - "metadata-ingestion-modules/prefect-plugin/**"
+      - "metadata-ingestion/**"
+      - "metadata-models/**"
+  pull_request:
+    branches:
+      - "**"
+    paths:
+      - ".github/workflows/prefect-plugin.yml"
+      - "metadata-ingestion-modules/prefect-plugin/**"
+      - "metadata-ingestion/**"
+      - "metadata-models/**"
+  release:
+    types: [published]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  prefect-plugin:
+    runs-on: ubuntu-latest
+    env:
+      SPARK_VERSION: 3.0.3
+      DATAHUB_TELEMETRY_ENABLED: false
+    strategy:
+      matrix:
+        python-version: ["3.8", "3.9", "3.10"]
+        include:
+          - python-version: "3.8"
+          - python-version: "3.9"
+          - python-version: "3.10"
+      fail-fast: false
+    steps:
+      - name: Set up JDK 17
+        uses: actions/setup-java@v3
+        with:
+          distribution: "zulu"
+          java-version: 17
+      - uses: gradle/gradle-build-action@v2
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: "pip"
+      - name: Install dependencies
+        run: ./metadata-ingestion/scripts/install_deps.sh
+      - name: Install prefect package
+        run: ./gradlew :metadata-ingestion-modules:prefect-plugin:lint :metadata-ingestion-modules:prefect-plugin:testQuick
+      - name: pip freeze show list installed
+        if: always()
+        run: source metadata-ingestion-modules/prefect-plugin/venv/bin/activate && pip freeze
+      - uses: actions/upload-artifact@v3
+        if: ${{ always() && matrix.python-version == '3.10'}}
+        with:
+          name: Test Results (Prefect Plugin ${{ matrix.python-version}})
+          path: |
+            **/build/reports/tests/test/**
+            **/build/test-results/test/**
+            **/junit.*.xml
+            !**/binary/**
+      - name: Upload coverage to Codecov
+        if: always()
+        uses: codecov/codecov-action@v3
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          directory: .
+          fail_ci_if_error: false
+          flags: prefect,prefect-${{ matrix.extra_pip_extras }}
+          name: pytest-prefect-${{ matrix.python-version }}
+          verbose: true
+
+  event-file:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Upload
+        uses: actions/upload-artifact@v3
+        with:
+          name: Event File
+          path: ${{ github.event_path }}

.github/workflows/test-results.yml

@@ -2,7 +2,7 @@ name: Test Results
 
 on:
   workflow_run:
-    workflows: ["build & test", "metadata ingestion", "Airflow Plugin", "Dagster Plugin", "GX Plugin"]
+    workflows: ["build & test", "metadata ingestion", "Airflow Plugin", "Dagster Plugin", "Prefect Plugin", "GX Plugin"]
     types:
       - completed
 

docs-website/build.gradle

@@ -86,6 +86,7 @@ task yarnGenerate(type: YarnTask, dependsOn: [yarnInstall,
     ':metadata-ingestion:buildWheel',
     ':metadata-ingestion-modules:airflow-plugin:buildWheel',
     ':metadata-ingestion-modules:dagster-plugin:buildWheel',
+    ':metadata-ingestion-modules:prefect-plugin:buildWheel',
     ':metadata-ingestion-modules:gx-plugin:buildWheel',
     ]) {
   inputs.files(projectMdFiles)

docs-website/filterTagIndexes.json

@@ -85,7 +85,7 @@
       "tags": {
         "Platform Type": "Orchestrator",
         "Connection Type": "Pull",
-        "Features": "Stateful Ingestion, UI Ingestion, Status Aspect"
+        "Features": "Status Aspect"
       }
     },
     {
@@ -429,6 +429,17 @@
         "Features": "Stateful Ingestion, Lower Casing, Status Aspect"
       }
     },
+    {
+      "Path": "docs/lineage/prefect",
+      "imgPath": "img/logos/platforms/prefect.svg",
+      "Title": "Prefect",
+      "Description": "Prefect is a modern workflow orchestration for data and ML engineers.",
+      "tags": {
+        "Platform Type": "Orchestrator",
+        "Connection Type": "Pull",
+        "Features": "Status Aspect"
+      }
+    },
     {
       "Path": "docs/generated/ingestion/sources/presto",
       "imgPath": "img/logos/platforms/presto.svg",

docs-website/generateDocsDir.ts

@@ -573,6 +573,7 @@ function copy_python_wheels(): void {
     "../metadata-ingestion/dist",
     "../metadata-ingestion-modules/airflow-plugin/dist",
     "../metadata-ingestion-modules/dagster-plugin/dist",
+    "../metadata-ingestion-modules/prefect-plugin/dist",
     "../metadata-ingestion-modules/gx-plugin/dist",
   ];
 

docs-website/sidebars.js

@@ -444,6 +444,11 @@ module.exports = {
           id: "docs/lineage/openlineage",
           label: "OpenLineage",
         },
+        {
+          type: "doc",
+          id: "docs/lineage/prefect",
+          label: "Prefect",
+        },
         {
           type: "doc",
           id: "metadata-integration/java/acryl-spark-lineage/README",
@@ -917,6 +922,7 @@ module.exports = {
     // "metadata-integration/java/openlineage-converter/README"
     //"metadata-ingestion-modules/airflow-plugin/README"
     //"metadata-ingestion-modules/dagster-plugin/README"
+    //"metadata-ingestion-modules/prefect-plugin/README"
     //"metadata-ingestion-modules/gx-plugin/README"
     // "metadata-ingestion/schedule_docs/datahub", // we can delete this
     // TODO: change the titles of these, removing the "What is..." portion from the sidebar"

docs-website/src/pages/_components/Logos/index.js

@@ -40,6 +40,7 @@ const platformLogos = [
     name: "CouchBase",
     imageUrl: "/img/logos/platforms/couchbase.svg",
   },
+  { name: "Dagster", imageUrl: "/img/logos/platforms/dagster.png" },
   { name: "Databricks", imageUrl: "/img/logos/platforms/databricks.png" },
   { name: "DBT", imageUrl: "/img/logos/platforms/dbt.svg" },
   { name: "Deltalake", imageUrl: "/img/logos/platforms/deltalake.svg" },
@@ -87,6 +88,7 @@ const platformLogos = [
   { name: "Pinot", imageUrl: "/img/logos/platforms/pinot.svg" },
   { name: "PostgreSQL", imageUrl: "/img/logos/platforms/postgres.svg" },
   { name: "PowerBI", imageUrl: "/img/logos/platforms/powerbi.png" },
+  { name: "Prefect", imageUrl: "/img/logos/platforms/prefect.svg" },
   { name: "Presto", imageUrl: "/img/logos/platforms/presto.svg" },
   { name: "Protobuf", imageUrl: "/img/logos/platforms/protobuf.png" },
   { name: "Pulsar", imageUrl: "/img/logos/platforms/pulsar.png" },