This repository has been archived by the owner on Oct 23, 2024. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 427
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Updates for Confluent Platform packages to latest release (3.2.2) (#1264
) * Updating packages for Confleunt 3.2.2 * Aligned Kafka health check default timings between config.json and *.mustache. More specific tool-tip around the broker memory and disk settings (based on feedback from customers that current settings are confusing). * Properly constrain memory allocation for docker image. For now, we don't allow the user to re-configure the JVM Heap allocation for the Control Center service (which defaults to 3 GB). So the minimum memory for the service should be 4 GB to allow room for the RocksDB instance.
- Loading branch information
Showing
25 changed files
with
1,849 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,62 @@ | ||
| { | ||
| "$schema": "http://json-schema.org/schema#", | ||
| "properties": { | ||
| "connect": { | ||
| "properties": { | ||
| "name": { | ||
| "default": "connect", | ||
| "description": "Service name for the connect worker application(s)", | ||
| "type": "string" | ||
| }, | ||
| "instances": { | ||
| "default": 1, | ||
| "description": "Number of instances to run.", | ||
| "minimum": 1, | ||
| "type": "integer" | ||
| }, | ||
| "cpus": { | ||
| "default": 2, | ||
| "description": "CPU shares to allocate to each connect worker instance.", | ||
| "minimum": 1, | ||
| "type": "number" | ||
| }, | ||
| "mem": { | ||
| "default": 1024, | ||
| "description": "Memory (MB) to allocate to each connect worker instance.", | ||
| "minimum": 512, | ||
| "type": "number" | ||
| }, | ||
| "heap": { | ||
| "default": 768, | ||
| "description": "JVM heap allocation (in MB) for connect worker task; should be ~256MB less than total memory for the instance.", | ||
| "minimum": 256, | ||
| "type": "number" | ||
| }, | ||
| "role": { | ||
| "default": "*", | ||
| "description": "Deploy connect worker only on nodes with this role.", | ||
| "type": "string" | ||
| }, | ||
| "kafka-service": { | ||
| "default": "confluent-kafka", | ||
| "description": "Target Apache Kafka by Confluent service to which these tasks will connect. ", | ||
| "type": "string" | ||
| }, | ||
| "zookeeper-connect": { | ||
| "default": "master.mesos:2181/dcos-service-confluent-kafka", | ||
| "description": "Zookeeper Connect string for service cluster. Format is comma-separated list of <zk-host>:<zkport>/<kservice>", | ||
| "type": "string" | ||
| }, | ||
| "schema-registry-service": { | ||
| "default": "schema-registry", | ||
| "description": "Schema Registry service to be used by connect workers. The named VIP associated with this service will be used to specify the converter-schema-registry-url's", | ||
|
|
||
| "type": "string" | ||
| } | ||
| }, | ||
| "required": ["cpus", "mem", "instances", "name"], | ||
| "type": "object" | ||
| } | ||
| }, | ||
| "type": "object" | ||
| } |
69 changes: 69 additions & 0 deletions
69
repo/packages/C/confluent-connect/7/marathon.json.mustache
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,69 @@ | ||
| { | ||
| "id": "/{{connect.name}}", | ||
| "instances": {{connect.instances}}, | ||
| "cpus": {{connect.cpus}}, | ||
| "mem": {{connect.mem}}, | ||
| "maintainer": "[email protected]", | ||
| "container": { | ||
| "type": "DOCKER", | ||
| "docker": { | ||
| "image": "{{resource.assets.container.docker.image}}", | ||
| "forcePullImage": true, | ||
| "network": "BRIDGE", | ||
| "portMappings": [ { | ||
| "containerPort": 8083, | ||
| "hostPort": 0, | ||
| "protocol": "tcp", | ||
| "labels": { | ||
| "VIP_0": "{{connect.name}}:8083" | ||
| } | ||
| } ] | ||
| } | ||
| }, | ||
| "portDefinitions": [ { | ||
| "name": "{{connect.name}}", | ||
| "port": 8083, | ||
| "protocol": "tcp", | ||
| "labels": { | ||
| "VIP_0": "{{connect.name}}:8083" | ||
| } | ||
| } ], | ||
| "env": { | ||
| "CONNECT_BOOTSTRAP_SERVERS": "broker.{{connect.kafka-service}}.l4lb.thisdcos.directory:9092", | ||
| "CONNECT_REST_PORT": "8083", | ||
| "CONNECT_GROUP_ID": "dcos-{{connect.name}}-group", | ||
| "CONNECT_CONFIG_STORAGE_TOPIC": "dcos-{{connect.name}}-configs", | ||
| "CONNECT_OFFSET_STORAGE_TOPIC": "dcos-{{connect.name}}-offsets", | ||
| "CONNECT_STATUS_STORAGE_TOPIC": "dcos-{{connect.name}}-status", | ||
| "CONNECT_KEY_CONVERTER" : "io.confluent.connect.avro.AvroConverter", | ||
| "CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL": "http://{{connect.schema-registry-service}}.marathon.l4lb.thisdcos.directory:8081", | ||
| "CONNECT_VALUE_CONVERTER" : "io.confluent.connect.avro.AvroConverter", | ||
| "CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL": "http://{{connect.schema-registry-service}}.marathon.l4lb.thisdcos.directory:8081", | ||
| "CONNECT_INTERNAL_KEY_CONVERTER" : "org.apache.kafka.connect.json.JsonConverter", | ||
| "CONNECT_INTERNAL_VALUE_CONVERTER" : "org.apache.kafka.connect.json.JsonConverter", | ||
| "CONNECT_PRODUCER_INTERCEPTOR_CLASSES" : "io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor", | ||
| "CONNECT_CONSUMER_INTERCEPTOR_CLASSES" : "io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor", | ||
| "CONNECT_ZOOKEEPER_CONNECT": "{{connect.zookeeper-connect}}", | ||
| "KAFKA_HEAP_OPTS": "-Xmx{{connect.heap}}M" | ||
| }, | ||
| "healthChecks": [ | ||
| { | ||
| "protocol": "HTTP", | ||
| "portIndex": 0, | ||
| "path": "/", | ||
| "gracePeriodSeconds": 60, | ||
| "intervalSeconds": 60, | ||
| "timeoutSeconds": 20, | ||
| "maxConsecutiveFailures": 3, | ||
| "ignoreHttp1xx": false | ||
| } | ||
| ], | ||
| "acceptedResourceRoles": [ | ||
| "{{connect.role}}" | ||
| ], | ||
| "labels": { | ||
| "DCOS_SERVICE_NAME": "{{connect.name}}", | ||
| "DCOS_SERVICE_SCHEME": "http", | ||
| "DCOS_SERVICE_PORT_INDEX": "0" | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,19 @@ | ||
| { | ||
| "packagingVersion": "3.0", | ||
| "name": "confluent-connect", | ||
| "version": "1.0.0-3.2.2", | ||
| "scm": "https://github.com/confluentinc/kafka", | ||
| "description": "Confluent Connect worker\n\n\tDocumentation: http://docs.confluent.io/3.2.2/connect/managing.html", | ||
| "maintainer": "[email protected]", | ||
| "tags": ["kafka", "confluent", "connect"], | ||
| "preInstallNotes": "This DC/OS Service is currently in preview. Preparing to install confluent-connect", | ||
| "postInstallNotes": "confluent-connect has been installed.", | ||
| "postUninstallNotes": "confluent-connect was uninstalled successfully.", | ||
| "minDcosReleaseVersion" : "1.8", | ||
| "licenses": [ | ||
| { | ||
| "name": "Apache License v2", | ||
| "url": "https://www.apache.org/licenses/LICENSE-2.0" | ||
| } | ||
| ] | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,14 @@ | ||
| { | ||
| "assets": { | ||
| "container": { | ||
| "docker": { | ||
| "image": "confluentinc/cp-kafka-connect:3.2.2" | ||
| } | ||
| } | ||
| }, | ||
| "images": { | ||
| "icon-small": "https://s3-us-west-2.amazonaws.com/confluent-mesos-devel/ConfIcon_small.png", | ||
| "icon-medium": "https://s3-us-west-2.amazonaws.com/confluent-mesos-devel/ConfIcon_medium.png", | ||
| "icon-large": "https://s3-us-west-2.amazonaws.com/confluent-mesos-devel/ConfIcon_large.png" | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,80 @@ | ||
| { | ||
| "$schema": "http://json-schema.org/schema#", | ||
| "properties": { | ||
| "control-center": { | ||
| "properties": { | ||
| "name": { | ||
| "default": "control-center", | ||
| "description": "Name for this control-center application", | ||
| "type": "string" | ||
| }, | ||
| "instances": { | ||
| "default": 1, | ||
| "description": "Number of instances to run.", | ||
| "minimum": 1, | ||
| "type": "integer" | ||
| }, | ||
| "cpus": { | ||
| "default": 2, | ||
| "description": "CPU shares to allocate to each control-center instance.", | ||
| "minimum": 2, | ||
| "type": "number" | ||
| }, | ||
| "mem": { | ||
| "default": 4096, | ||
| "description": "Memory (MB) to allocate to each control-center task.", | ||
| "minimum": 4096, | ||
| "type": "number" | ||
| }, | ||
| "role": { | ||
| "default": "*", | ||
| "description": "Deploy control-center only on nodes with this role.", | ||
| "type": "string" | ||
| }, | ||
| "kafka-service": { | ||
| "default": "confluent-kafka", | ||
| "description": "Target Apache Kafka by Confluent service to which these tasks will connect. ", | ||
| "type": "string" | ||
| }, | ||
| "connect-service": { | ||
| "default": "connect", | ||
| "description": "Service name of Kafka Connect Workers to which this instance will deploy connectors.", | ||
| "type": "string" | ||
| }, | ||
| "confluent-controlcenter-internal-topics-partitions": { | ||
| "default": 3, | ||
| "description": "Parition count for internal control-center kafka topics", | ||
| "type": "number" | ||
| }, | ||
| "confluent-controlcenter-internal-topics-replication": { | ||
| "default": 2, | ||
| "description": "Replication factor for internal control-center kafka topics", | ||
| "type": "number" | ||
| }, | ||
| "confluent-monitoring-interceptor-topic-partitions": { | ||
| "default": 3, | ||
| "description": "Parition count for kafka topics used to store data from the interceptor classes", | ||
| "type": "number" | ||
| }, | ||
| "confluent-monitoring-interceptor-topic-replication": { | ||
| "default": 2, | ||
| "description": "Replication factor for kafka topics used to store data from the interceptor classes", | ||
| "type": "number" | ||
| }, | ||
| "confluent-license": { | ||
| "default": "", | ||
| "description": "License key for Confluent Enterprise (default is 30-day trial)", | ||
| "type": "string" | ||
| }, | ||
| "zookeeper-connect": { | ||
| "default": "master.mesos:2181/dcos-service-confluent-kafka", | ||
| "description": "Zookeeper Connect string for service cluster. Format is comma-separated list of <zk-host>:<zkport>/<kservice>", | ||
| "type": "string" | ||
| } | ||
| }, | ||
| "required": ["cpus", "mem", "instances", "name"], | ||
| "type": "object" | ||
| } | ||
| }, | ||
| "type": "object" | ||
| } |
49 changes: 49 additions & 0 deletions
49
repo/packages/C/confluent-control-center/8/marathon.json.mustache
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,49 @@ | ||
| { | ||
| "id": "/{{control-center.name}}", | ||
| "instances": {{control-center.instances}}, | ||
| "cpus": {{control-center.cpus}}, | ||
| "mem": {{control-center.mem}}, | ||
| "maintainer": "[email protected]", | ||
| "container": { | ||
| "type": "DOCKER", | ||
| "docker": { | ||
| "image": "{{resource.assets.container.docker.image}}", | ||
| "forcePullImage": true, | ||
| "network": "BRIDGE", | ||
| "portMappings": [ { | ||
| "containerPort": 9021, | ||
| "hostPort": 0, | ||
| "protocol": "tcp" | ||
| } ] | ||
| } | ||
| }, | ||
| "env": { | ||
| "CONTROL_CENTER_BOOTSTRAP_SERVERS": "broker.{{control-center.kafka-service}}.l4lb.thisdcos.directory:9092", | ||
| "CONTROL_CENTER_CONNECT_CLUSTER": "{{control-center.connect-service}}.marathon.l4lb.thisdcos.directory:8083", | ||
| "CONTROL_CENTER_INTERNAL_TOPICS_PARTITIONS": "{{control-center.confluent-controlcenter-internal-topics-partitions}}", | ||
| "CONTROL-CENTER_MONITORING_INTERCEPTOR_TOPIC_PARTITIONS": "{{control-center.confluent-monitoring-interceptor-topic-partitions}}", | ||
| "CONTROL_CENTER_REPLICATION_FACTOR": "{{control-center.confluent-controlcenter-internal-topics-replication}}", | ||
| "CONTROL_CENTER_LICENSE": "{{control-center.confluent-license}}", | ||
| "CONTROL_CENTER_ZOOKEEPER_CONNECT": "{{control-center.zookeeper-connect}}" | ||
| }, | ||
| "healthChecks": [ | ||
| { | ||
| "protocol": "HTTP", | ||
| "portIndex": 0, | ||
| "path": "/", | ||
| "gracePeriodSeconds": 3600, | ||
| "intervalSeconds": 60, | ||
| "timeoutSeconds": 5, | ||
| "maxConsecutiveFailures": 10, | ||
| "ignoreHttp1xx": false | ||
| } | ||
| ], | ||
| "acceptedResourceRoles": [ | ||
| "{{control-center.role}}" | ||
| ], | ||
| "labels": { | ||
| "DCOS_SERVICE_NAME": "{{control-center.name}}", | ||
| "DCOS_SERVICE_SCHEME": "http", | ||
| "DCOS_SERVICE_PORT_INDEX": "0" | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,19 @@ | ||
| { | ||
| "packagingVersion": "3.0", | ||
| "name": "confluent-control-center", | ||
| "version": "1.0.0-3.2.2", | ||
| "scm": "https://github.com/confluentinc/control-center", | ||
| "description": "Confluent Control Center service\n\n\tDocumentation: http://docs.confluent.io/3.2.2/control-center/docs/userguide.html\n\tDC/OS Specifics: https://www.confluent.io/whitepaper/deploying-confluent-platform-with-mesosphere", | ||
| "maintainer": "[email protected]", | ||
| "tags": ["kafka", "confluent", "control", "center"], | ||
| "preInstallNotes": "Preparing to install confluent-control-center", | ||
| "postInstallNotes": "confluent-control-center has been installed.", | ||
| "postUninstallNotes": "confluent-control-center was uninstalled successfully.", | ||
| "minDcosReleaseVersion" : "1.8", | ||
| "licenses": [ | ||
| { | ||
| "name": "Apache License v2", | ||
| "url": "https://raw.githubusercontent.com/confluentinc/control-center/master/LICENSE" | ||
| } | ||
| ] | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,14 @@ | ||
| { | ||
| "assets": { | ||
| "container": { | ||
| "docker": { | ||
| "image": "confluentinc/cp-enterprise-control-center:3.2.2" | ||
| } | ||
| } | ||
| }, | ||
| "images": { | ||
| "icon-small": "https://s3-us-west-2.amazonaws.com/confluent-mesos-devel/ConfIcon_small.png", | ||
| "icon-medium": "https://s3-us-west-2.amazonaws.com/confluent-mesos-devel/ConfIcon_medium.png", | ||
| "icon-large": "https://s3-us-west-2.amazonaws.com/confluent-mesos-devel/ConfIcon_large.png" | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,5 @@ | ||
| { | ||
| "pip": [ | ||
| "https://downloads.mesosphere.com/kafka/assets/1.1.19-0.10.1.0/bin_wrapper-0.0.1-py2.py3-none-any.whl" | ||
| ] | ||
| } |
Oops, something went wrong.