From 55f99956b02c934bbe73256897cb9413504198d8 Mon Sep 17 00:00:00 2001 From: open-metadata Date: Thu, 16 Jan 2025 10:15:26 +0000 Subject: [PATCH] See https://github.com/open-metadata/OpenMetadata/commit/4af97643903cce6817a326e244385151c9e61211 from refs/heads/main --- .../yaml/messaging/source-config-def.md | 10 +++- .../yaml/messaging/source-config.md | 4 ++ .../yaml/messaging/source-config-def.md | 10 +++- .../yaml/messaging/source-config.md | 4 ++ .../v1.6.x/connectors/messaging/kafka/yaml.md | 28 +++++++++++ .../connectors/messaging/redpanda/yaml.md | 15 ++++++ .../connectors/messaging/kafka/yaml.md | 46 +++++++++++++++---- .../connectors/messaging/redpanda/yaml.md | 33 +++++++++---- 8 files changed, 128 insertions(+), 22 deletions(-) diff --git a/content/partials/v1.6/connectors/yaml/messaging/source-config-def.md b/content/partials/v1.6/connectors/yaml/messaging/source-config-def.md index 43a77ddad..371c48053 100644 --- a/content/partials/v1.6/connectors/yaml/messaging/source-config-def.md +++ b/content/partials/v1.6/connectors/yaml/messaging/source-config-def.md @@ -4,8 +4,14 @@ The sourceConfig is defined [here](https://github.com/open-metadata/OpenMetadata/blob/main/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/messagingServiceMetadataPipeline.json): -**generateSampleData:** Option to turn on/off generating sample data during metadata extraction. +- **generateSampleData:** Option to turn on/off generating sample data during metadata extraction. -**topicFilterPattern:** Note that the `topicFilterPattern` supports regex as include or exclude. +- **topicFilterPattern:** Note that the `topicFilterPattern` supports regex as include or exclude. + +- **generateSampleData:** Option to turn on/off generating sample data during metadata extraction. `generateSampleData` supports boolean value either `true` or `false`. + +- **markDeletedTopics:** Optional configuration to soft delete topics in OpenMetadata if the source topics are deleted. Also, if the topic is deleted, all the associated entities like sample data, lineage, etc., with that topic will be deleted. `markDeletedTopics` supports boolean value either `true` or `false`. + +- **overrideMetadata:** Set the 'Override Metadata' toggle to control whether to override the existing metadata in the OpenMetadata server with the metadata fetched from the source. If the toggle is set to true, the metadata fetched from the source will override the existing metadata in the OpenMetadata server. If the toggle is set to false, the metadata fetched from the source will not override the existing metadata in the OpenMetadata server. This is applicable for fields like description, tags, owner and displayName. `overrideMetadata` supports boolean value either `true` or `false`. {% /codeInfo %} \ No newline at end of file diff --git a/content/partials/v1.6/connectors/yaml/messaging/source-config.md b/content/partials/v1.6/connectors/yaml/messaging/source-config.md index cb9bd5e14..1b64379b7 100644 --- a/content/partials/v1.6/connectors/yaml/messaging/source-config.md +++ b/content/partials/v1.6/connectors/yaml/messaging/source-config.md @@ -8,4 +8,8 @@ # includes: # - topic1 # generateSampleData: true + # generateSampleData: false # true + # markDeletedTopics: true # false + # overrideMetadata: false # true + ``` \ No newline at end of file diff --git a/content/partials/v1.7/connectors/yaml/messaging/source-config-def.md b/content/partials/v1.7/connectors/yaml/messaging/source-config-def.md index 43a77ddad..371c48053 100644 --- a/content/partials/v1.7/connectors/yaml/messaging/source-config-def.md +++ b/content/partials/v1.7/connectors/yaml/messaging/source-config-def.md @@ -4,8 +4,14 @@ The sourceConfig is defined [here](https://github.com/open-metadata/OpenMetadata/blob/main/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/messagingServiceMetadataPipeline.json): -**generateSampleData:** Option to turn on/off generating sample data during metadata extraction. +- **generateSampleData:** Option to turn on/off generating sample data during metadata extraction. -**topicFilterPattern:** Note that the `topicFilterPattern` supports regex as include or exclude. +- **topicFilterPattern:** Note that the `topicFilterPattern` supports regex as include or exclude. + +- **generateSampleData:** Option to turn on/off generating sample data during metadata extraction. `generateSampleData` supports boolean value either `true` or `false`. + +- **markDeletedTopics:** Optional configuration to soft delete topics in OpenMetadata if the source topics are deleted. Also, if the topic is deleted, all the associated entities like sample data, lineage, etc., with that topic will be deleted. `markDeletedTopics` supports boolean value either `true` or `false`. + +- **overrideMetadata:** Set the 'Override Metadata' toggle to control whether to override the existing metadata in the OpenMetadata server with the metadata fetched from the source. If the toggle is set to true, the metadata fetched from the source will override the existing metadata in the OpenMetadata server. If the toggle is set to false, the metadata fetched from the source will not override the existing metadata in the OpenMetadata server. This is applicable for fields like description, tags, owner and displayName. `overrideMetadata` supports boolean value either `true` or `false`. {% /codeInfo %} \ No newline at end of file diff --git a/content/partials/v1.7/connectors/yaml/messaging/source-config.md b/content/partials/v1.7/connectors/yaml/messaging/source-config.md index cb9bd5e14..1b64379b7 100644 --- a/content/partials/v1.7/connectors/yaml/messaging/source-config.md +++ b/content/partials/v1.7/connectors/yaml/messaging/source-config.md @@ -8,4 +8,8 @@ # includes: # - topic1 # generateSampleData: true + # generateSampleData: false # true + # markDeletedTopics: true # false + # overrideMetadata: false # true + ``` \ No newline at end of file diff --git a/content/v1.6.x/connectors/messaging/kafka/yaml.md b/content/v1.6.x/connectors/messaging/kafka/yaml.md index 46b1dad3d..849d52a43 100644 --- a/content/v1.6.x/connectors/messaging/kafka/yaml.md +++ b/content/v1.6.x/connectors/messaging/kafka/yaml.md @@ -122,6 +122,22 @@ following [link](https://docs.confluent.io/platform/current/clients/confluent-ka {% /codeInfo %} +{% codeInfo srNumber=9 %} +**securityProtocol**: security.protocol consumer config property. It accepts `PLAINTEXT`,`SASL_PLAINTEXT`, `SASL_SSL`, `SSL`. +{% /codeInfo %} + +{% codeInfo srNumber=10 %} +**schemaRegistryTopicSuffixName**: Schema Registry Topic Suffix Name. The suffix to be appended to the topic name to get topic schema from registry. +{% /codeInfo %} + +{% codeInfo srNumber=11 %} +**schemaRegistrySSL**: Schema Registry SSL Config. Configuration for enabling SSL for the Schema Registry connection. +{% /codeInfo %} + +{% codeInfo srNumber=12 %} +**supportsMetadataExtraction**: Supports Metadata Extraction. `supportsMetadataExtraction` supports boolean value either true or false. +{% /codeInfo %} + {% partial file="/v1.6/connectors/yaml/messaging/source-config-def.md" /%} {% partial file="/v1.6/connectors/yaml/ingestion-sink-def.md" /%} @@ -164,6 +180,18 @@ source: ```yaml {% srNumber=8 %} schemaRegistryConfig: {} ``` +```yaml {% srNumber=9 %} + # securityProtocol: PLAINTEXT +``` +```yaml {% srNumber=10 %} + # schemaRegistryTopicSuffixName: -value +``` +```yaml {% srNumber=11 %} + # schemaRegistrySSL: "" +``` +```yaml {% srNumber=12 %} + # supportsMetadataExtraction: true +``` {% partial file="/v1.6/connectors/yaml/messaging/source-config.md" /%} diff --git a/content/v1.6.x/connectors/messaging/redpanda/yaml.md b/content/v1.6.x/connectors/messaging/redpanda/yaml.md index 2bab32c86..e8eba0884 100644 --- a/content/v1.6.x/connectors/messaging/redpanda/yaml.md +++ b/content/v1.6.x/connectors/messaging/redpanda/yaml.md @@ -117,6 +117,14 @@ following [link](https://docs.confluent.io/platform/current/clients/confluent-ka {% /codeInfo %} +{% codeInfo srNumber=9 %} +**securityProtocol**: security.protocol consumer config property. It accepts `PLAINTEXT`,`SASL_PLAINTEXT`, `SASL_SSL`, `SSL`. +{% /codeInfo} + +{% codeInfo srNumber=10 %} +**supportsMetadataExtraction**: Supports Metadata Extraction. `supportsMetadataExtraction` supports boolean value either true or false. +{% /codeInfo %} + {% partial file="/v1.6/connectors/yaml/messaging/source-config-def.md" /%} {% partial file="/v1.6/connectors/yaml/ingestion-sink-def.md" /%} @@ -159,6 +167,13 @@ source: ```yaml {% srNumber=8 %} schemaRegistryConfig: {} ``` +```yaml {% srNumber=9 %} + # securityProtocol: PLAINTEXT +``` +```yaml {% srNumber=10 %} + # supportsMetadataExtraction: true +``` + {% partial file="/v1.6/connectors/yaml/messaging/source-config.md" /%} diff --git a/content/v1.7.x-SNAPSHOT/connectors/messaging/kafka/yaml.md b/content/v1.7.x-SNAPSHOT/connectors/messaging/kafka/yaml.md index f4b8d110c..849d52a43 100644 --- a/content/v1.7.x-SNAPSHOT/connectors/messaging/kafka/yaml.md +++ b/content/v1.7.x-SNAPSHOT/connectors/messaging/kafka/yaml.md @@ -19,13 +19,13 @@ Configure and schedule Kafka metadata and profiler workflows from the OpenMetada - [Metadata Ingestion](#metadata-ingestion) - [Enable Security](#securing-kafka-connection-with-ssl-in-openmetadata) -{% partial file="/v1.7/connectors/external-ingestion-deployment.md" /%} +{% partial file="/v1.6/connectors/external-ingestion-deployment.md" /%} ## Requirements ### Python Requirements -{% partial file="/v1.7/connectors/python-requirements.md" /%} +{% partial file="/v1.6/connectors/python-requirements.md" /%} To run the Kafka ingestion, you will need to install: @@ -122,11 +122,27 @@ following [link](https://docs.confluent.io/platform/current/clients/confluent-ka {% /codeInfo %} -{% partial file="/v1.7/connectors/yaml/messaging/source-config-def.md" /%} +{% codeInfo srNumber=9 %} +**securityProtocol**: security.protocol consumer config property. It accepts `PLAINTEXT`,`SASL_PLAINTEXT`, `SASL_SSL`, `SSL`. +{% /codeInfo %} + +{% codeInfo srNumber=10 %} +**schemaRegistryTopicSuffixName**: Schema Registry Topic Suffix Name. The suffix to be appended to the topic name to get topic schema from registry. +{% /codeInfo %} + +{% codeInfo srNumber=11 %} +**schemaRegistrySSL**: Schema Registry SSL Config. Configuration for enabling SSL for the Schema Registry connection. +{% /codeInfo %} + +{% codeInfo srNumber=12 %} +**supportsMetadataExtraction**: Supports Metadata Extraction. `supportsMetadataExtraction` supports boolean value either true or false. +{% /codeInfo %} + +{% partial file="/v1.6/connectors/yaml/messaging/source-config-def.md" /%} -{% partial file="/v1.7/connectors/yaml/ingestion-sink-def.md" /%} +{% partial file="/v1.6/connectors/yaml/ingestion-sink-def.md" /%} -{% partial file="/v1.7/connectors/yaml/workflow-config-def.md" /%} +{% partial file="/v1.6/connectors/yaml/workflow-config-def.md" /%} {% /codeInfoContainer %} @@ -164,12 +180,24 @@ source: ```yaml {% srNumber=8 %} schemaRegistryConfig: {} ``` +```yaml {% srNumber=9 %} + # securityProtocol: PLAINTEXT +``` +```yaml {% srNumber=10 %} + # schemaRegistryTopicSuffixName: -value +``` +```yaml {% srNumber=11 %} + # schemaRegistrySSL: "" +``` +```yaml {% srNumber=12 %} + # supportsMetadataExtraction: true +``` -{% partial file="/v1.7/connectors/yaml/messaging/source-config.md" /%} +{% partial file="/v1.6/connectors/yaml/messaging/source-config.md" /%} -{% partial file="/v1.7/connectors/yaml/ingestion-sink.md" /%} +{% partial file="/v1.6/connectors/yaml/ingestion-sink.md" /%} -{% partial file="/v1.7/connectors/yaml/workflow-config.md" /%} +{% partial file="/v1.6/connectors/yaml/workflow-config.md" /%} {% /codeBlock %} @@ -186,4 +214,4 @@ To establish secure connections between OpenMetadata and Kafka, in the `YAML` yo sslKey: "/path/to/your/ssl_key" ``` -{% partial file="/v1.7/connectors/yaml/ingestion-cli.md" /%} +{% partial file="/v1.6/connectors/yaml/ingestion-cli.md" /%} diff --git a/content/v1.7.x-SNAPSHOT/connectors/messaging/redpanda/yaml.md b/content/v1.7.x-SNAPSHOT/connectors/messaging/redpanda/yaml.md index ecdc1609b..e8eba0884 100644 --- a/content/v1.7.x-SNAPSHOT/connectors/messaging/redpanda/yaml.md +++ b/content/v1.7.x-SNAPSHOT/connectors/messaging/redpanda/yaml.md @@ -18,13 +18,13 @@ Configure and schedule Redpanda metadata and profiler workflows from the OpenMet - [Requirements](#requirements) - [Metadata Ingestion](#metadata-ingestion) -{% partial file="/v1.7/connectors/external-ingestion-deployment.md" /%} +{% partial file="/v1.6/connectors/external-ingestion-deployment.md" /%} ## Requirements ### Python Requirements -{% partial file="/v1.7/connectors/python-requirements.md" /%} +{% partial file="/v1.6/connectors/python-requirements.md" /%} To run the Redpanda ingestion, you will need to install: @@ -117,11 +117,19 @@ following [link](https://docs.confluent.io/platform/current/clients/confluent-ka {% /codeInfo %} -{% partial file="/v1.7/connectors/yaml/messaging/source-config-def.md" /%} +{% codeInfo srNumber=9 %} +**securityProtocol**: security.protocol consumer config property. It accepts `PLAINTEXT`,`SASL_PLAINTEXT`, `SASL_SSL`, `SSL`. +{% /codeInfo} -{% partial file="/v1.7/connectors/yaml/ingestion-sink-def.md" /%} +{% codeInfo srNumber=10 %} +**supportsMetadataExtraction**: Supports Metadata Extraction. `supportsMetadataExtraction` supports boolean value either true or false. +{% /codeInfo %} + +{% partial file="/v1.6/connectors/yaml/messaging/source-config-def.md" /%} + +{% partial file="/v1.6/connectors/yaml/ingestion-sink-def.md" /%} -{% partial file="/v1.7/connectors/yaml/workflow-config-def.md" /%} +{% partial file="/v1.6/connectors/yaml/workflow-config-def.md" /%} {% /codeInfoContainer %} @@ -159,15 +167,22 @@ source: ```yaml {% srNumber=8 %} schemaRegistryConfig: {} ``` +```yaml {% srNumber=9 %} + # securityProtocol: PLAINTEXT +``` +```yaml {% srNumber=10 %} + # supportsMetadataExtraction: true +``` + -{% partial file="/v1.7/connectors/yaml/messaging/source-config.md" /%} +{% partial file="/v1.6/connectors/yaml/messaging/source-config.md" /%} -{% partial file="/v1.7/connectors/yaml/ingestion-sink.md" /%} +{% partial file="/v1.6/connectors/yaml/ingestion-sink.md" /%} -{% partial file="/v1.7/connectors/yaml/workflow-config.md" /%} +{% partial file="/v1.6/connectors/yaml/workflow-config.md" /%} {% /codeBlock %} {% /codePreview %} -{% partial file="/v1.7/connectors/yaml/ingestion-cli.md" /%} +{% partial file="/v1.6/connectors/yaml/ingestion-cli.md" /%}