feat(ingest/dremio): Dremio Source Ingestion

datahub-web-react/src/app/ingest/source/builder/constants.ts

@@ -19,6 +19,7 @@ import clickhouseLogo from '../../../../images/clickhouselogo.png';
 import cockroachdbLogo from '../../../../images/cockroachdblogo.png';
 import trinoLogo from '../../../../images/trinologo.png';
 import dbtLogo from '../../../../images/dbtlogo.png';
+import dremioLogo from '../../../../images/dremiologo.png';
 import druidLogo from '../../../../images/druidlogo.png';
 import elasticsearchLogo from '../../../../images/elasticsearchlogo.png';
 import feastLogo from '../../../../images/feastlogo.png';
@@ -52,6 +53,8 @@ export const COCKROACHDB = 'cockroachdb';
 export const COCKROACHDB_URN = `urn:li:dataPlatform:${COCKROACHDB}`;
 export const DBT = 'dbt';
 export const DBT_URN = `urn:li:dataPlatform:${DBT}`;
+export const DREMIO = 'dremio';
+export const DREMIO_URN = `urn:li:dataPlatform:${DREMIO}`;
 export const DRUID = 'druid';
 export const DRUID_URN = `urn:li:dataPlatform:${DRUID}`;
 export const DYNAMODB = 'dynamodb';
@@ -139,6 +142,7 @@ export const PLATFORM_URN_TO_LOGO = {
     [CLICKHOUSE_URN]: clickhouseLogo,
     [COCKROACHDB_URN]: cockroachdbLogo,
     [DBT_URN]: dbtLogo,
+    [DREMIO_URN]: dremioLogo,
     [DRUID_URN]: druidLogo,
     [DYNAMODB_URN]: dynamodbLogo,
     [ELASTICSEARCH_URN]: elasticsearchLogo,

datahub-web-react/src/app/ingest/source/builder/sources.json

@@ -302,5 +302,13 @@
         "description": "Configure a custom recipe using YAML.",
         "docsUrl": "https://datahubproject.io/docs/metadata-ingestion/",
         "recipe": "source:\n  type: <source-type>\n  config:\n    # Source-type specifics config\n    <source-configs>"
+    },
+    {
+        "urn": "urn:li:dataPlatform:dremio",
+        "name": "dremio",
+        "displayName": "Dremio",
+        "description": "Import Tables, Databases, Schemas, and statistics from Dremio.",
+        "docsUrl": "https://datahubproject.io/docs/metadata-ingestion/",
+        "recipe": "source:\n    type: dremio\n    config:\n        # Coordinates\n        hostname: null\n        port: null\n        # Credentials\n        authentication_method: password\n        username: null\n        password: null\n        stateful_ingestion:\n            enabled: true"
     }
 ]

docs/cli.md

@@ -705,6 +705,7 @@ Please see our [Integrations page](https://datahubproject.io/integrations) if yo
 | [datahub-lineage-file](./generated/ingestion/sources/file-based-lineage.md)                    | _no additional dependencies_                               | Lineage File source                     |
 | [datahub-business-glossary](./generated/ingestion/sources/business-glossary.md)                | _no additional dependencies_                               | Business Glossary File source           |
 | [dbt](./generated/ingestion/sources/dbt.md)                                                    | _no additional dependencies_                               | dbt source                              |
+| [dremio](./generated/ingestion/sources/dremio.md)                                              | `pip install 'acryl-datahub[dremio]'`                      | Dremio Source                          |
 | [druid](./generated/ingestion/sources/druid.md)                                                | `pip install 'acryl-datahub[druid]'`                       | Druid Source                            |
 | [feast](./generated/ingestion/sources/feast.md)                                                | `pip install 'acryl-datahub[feast]'`                       | Feast source (0.26.0)                   |
 | [glue](./generated/ingestion/sources/glue.md)                                                  | `pip install 'acryl-datahub[glue]'`                        | AWS Glue source                         |

metadata-ingestion/docs/sources/dremio/README.md

@@ -0,0 +1,23 @@
+### Concept Mapping
+
+- **Dremio Datasets**: Mapped to DataHub’s `Dataset` entity.
+    - A dataset can be physical or virtual.
+- **Lineage**: Mapped to DataHub’s `UpstreamLineage` aspect, representing the flow of data between datasets and columns.
+- **Containers**: Spaces, folders, and sources in Dremio are mapped to DataHub’s `Container` aspect, organizing datasets logically.
+
+Here's a table for **Concept Mapping** between Dremio and DataHub to provide a clear overview of how entities and concepts in Dremio are mapped to corresponding entities in DataHub:
+
+| **Dremio Concept** | **DataHub Entity/Aspect** | **Description** |  |
+| --- | --- | --- | --- |
+| **Physical Dataset** | `Dataset` | A dataset directly queried from an external source without modifications. |  |
+| **Virtual Dataset** | `Dataset` | A dataset built from SQL-based transformations on other datasets. |  |
+| **Spaces** | `Container` | Top-level organizational unit in Dremio, used to group datasets. Mapped to DataHub’s `Container` aspect. |  |
+| **Folders** | `Container` | Substructure inside spaces, used for organizing datasets. Mapped as a `Container` in DataHub. |  |
+| **Sources** | `Container` | External data sources connected to Dremio (e.g., S3, databases). Represented as a `Container` in DataHub. |  |
+| **Column Lineage** | `ColumnLineage` | Lineage between columns in datasets, showing how individual columns are transformed across datasets. |  |
+| **Dataset Lineage** | `UpstreamLineage` | Lineage between datasets, tracking the flow and transformations between different datasets. |  |
+| **Ownership (Dataset)** | `Ownership` | Ownership information for datasets, representing the technical owner in DataHub’s `Ownership` aspect. |  |
+| **Glossary Terms** | `GlossaryTerms` | Business terms associated with datasets, providing context. Mapped as `GlossaryTerms` in DataHub. |  |
+| **Schema Metadata** | `SchemaMetadata` | Schema details (columns, data types) for datasets. Mapped to DataHub’s `SchemaMetadata` aspect. |  |
+| **SQL Transformations** | `Dataset` (with lineage) | SQL queries in Dremio that transform datasets. Represented as `Dataset` in DataHub, with lineage showing dependency. |  |
+| **Queries** | `Query` (if mapped) | Historical SQL queries executed on Dremio datasets. These can be tracked for audit purposes in DataHub. |  |

metadata-ingestion/docs/sources/dremio/dremio_pre.md

@@ -0,0 +1,45 @@
+### Setup
+
+This integration pulls metadata directly from the Dremio APIs.
+
+You'll need to have a Dremio instance up and running with access to the necessary datasets, and API access should be enabled with a valid token.
+
+**Dremio instance can be one of following**:
+
+    - Dremio Cloud (Fully managed cloud SaaS)
+        - Standard
+        - Enterprise
+    - Dremio Software (self-managed on own infrastructure / on-premise)
+        - Community (oss)
+        - Enterprise
+
+The API token should have the necessary permissions to **read metadata** and **retrieve lineage**.
+
+#### Steps to Get the Required Information
+
+1. **Generate an API Token**:
+
+   - Log in to your Dremio instance.
+   - Navigate to your user profile in the top-right corner.
+   - Select **Generate API Token** to create an API token for programmatic access.
+   - Ensure that the API token has sufficient permissions to access datasets, spaces, sources, and lineage.
+
+2. **Identify the API Endpoint**:
+
+   - The Dremio API endpoint typically follows this format:  
+     `https://<your-dremio-instance>/api/v3/`
+   - This endpoint is used to query metadata and lineage information.
+
+3. **Get the Space, Folder, and Dataset Details**:
+   - To identify specific datasets or containers (spaces, folders, sources), navigate to the Dremio web interface.
+   - Explore the **Spaces** and **Sources** sections to identify the datasets you need to retrieve metadata for.
+4. **Permissions**:
+   - The token should have **read-only** or **admin** permissions that allow it to:
+     - View all datasets (physical and virtual).
+     - Access all spaces, folders, and sources.
+     - Retrieve dataset and column-level lineage information.
+5. **Verify External Data Source Permissions**:
+   - If Dremio is connected to external data sources (e.g., AWS S3, relational databases), ensure that Dremio has access to the credentials required for querying those sources.
+
+
+Ensure your API token has the correct permissions to interact with the Dremio metadata.

metadata-ingestion/docs/sources/dremio/dremio_recipe.yml

@@ -0,0 +1,25 @@
+source:
+  type: dremio
+  config:
+    # Coordinates
+    hostname: localhost
+    port: 9047
+    tls: true
+
+    # Credentials
+    authentication_method: password
+    username: user
+    password: pass
+
+    include_query_lineage: True
+
+    source_mappings:
+      - platform: s3
+        platform_name: samples
+
+    schema_pattern:
+      allow:
+        - ".*"
+
+sink:
+  # sink configs

metadata-ingestion/setup.py

@@ -382,6 +382,7 @@
     "delta-lake": {*data_lake_profiling, *delta_lake},
     "dbt": {"requests"} | dbt_common | aws_common,
     "dbt-cloud": {"requests"} | dbt_common,
+    "dremio": {"requests"} | sqlglot_lib,
     "druid": sql_common | {"pydruid>=0.6.2"},
     "dynamodb": aws_common | classification_lib,
     # Starting with 7.14.0 python client is checking if it is connected to elasticsearch client. If its not it throws
@@ -592,6 +593,7 @@
             "clickhouse-usage",
             "cockroachdb",
             "delta-lake",
+            "dremio",
             "druid",
             "elasticsearch",
             "feast",
@@ -690,6 +692,7 @@
         "s3 = datahub.ingestion.source.s3:S3Source",
         "dbt = datahub.ingestion.source.dbt.dbt_core:DBTCoreSource",
         "dbt-cloud = datahub.ingestion.source.dbt.dbt_cloud:DBTCloudSource",
+        "dremio = datahub.ingestion.source.dremio.dremio_source:DremioSource",
         "druid = datahub.ingestion.source.sql.druid:DruidSource",
         "dynamodb = datahub.ingestion.source.dynamodb.dynamodb:DynamoDBSource",
         "elasticsearch = datahub.ingestion.source.elastic_search:ElasticsearchSource",