From 7754a359aed8ad772d696794e607ae1bd529fc30 Mon Sep 17 00:00:00 2001 From: AJ Danelz Date: Wed, 18 Sep 2024 08:54:33 -0400 Subject: [PATCH 1/2] Model concept docs (#254) * feat: add instructions on mounting files in a zilla pod * feat: add modesl overview and catalog/model guides * feat: add basic catalog how-to * feat: add the model guide * feat: add guid links * fix: link * Update src/how-tos/catalogs/index.md Co-authored-by: John Fallows --------- Co-authored-by: John Fallows --- cspell.json | 1 + src/.vuepress/sidebar/en.ts | 21 +++++++- src/concepts/catalogs.md | 4 +- src/concepts/models.md | 47 +++++++++++++++++ src/how-tos/catalogs/index.md | 66 ++++++++++++++++++++++++ src/how-tos/deploy-operate.md | 47 ++++++++++++++++- src/how-tos/models/index.md | 96 +++++++++++++++++++++++++++++++++++ 7 files changed, 277 insertions(+), 5 deletions(-) create mode 100644 src/concepts/models.md create mode 100644 src/how-tos/catalogs/index.md create mode 100644 src/how-tos/models/index.md diff --git a/cspell.json b/cspell.json index a3a18a6c..95806632 100644 --- a/cspell.json +++ b/cspell.json @@ -11,6 +11,7 @@ "aklivity", "ALIES", "alpn", + "apicurio", "APIs", "artifacthub", "asyncapi", diff --git a/src/.vuepress/sidebar/en.ts b/src/.vuepress/sidebar/en.ts index b053dc11..381af478 100644 --- a/src/.vuepress/sidebar/en.ts +++ b/src/.vuepress/sidebar/en.ts @@ -244,8 +244,27 @@ export const enSidebar = sidebar({ }, { text: "Catalogs", + collapsible: true, link: "concepts/catalogs.md", - children: [], + children: [ + { + text: "Overview", + link: "concepts/catalogs.md", + }, + "how-tos/catalogs/", + ], + }, + { + text: "Models", + collapsible: true, + link: "concepts/models.md", + children: [ + { + text: "Overview", + link: "concepts/models.md", + }, + "how-tos/models/", + ], }, ], }, diff --git a/src/concepts/catalogs.md b/src/concepts/catalogs.md index 6aa3f98f..33d8156a 100644 --- a/src/concepts/catalogs.md +++ b/src/concepts/catalogs.md @@ -19,10 +19,10 @@ catalogs: The simplest catalogs allow Zilla to access resources directly. The [filesystem](../reference/config/catalogs/catalog-filesystem.md) catalog will look for resources on the host filesystem. To embed a resource directly into the Zilla config yaml an [inline](../reference/config/catalogs/catalog-inline.md) catalog can be used to define any text based resource. -> [http.proxy.schema.inline example](https://github.com/aklivity/zilla-examples/tree/main/http.proxy.schema.inline) +> [Local Catalogs Guide](../how-tos/catalogs/index.md#local-catalogs) | [http.proxy.schema.inline example](https://github.com/aklivity/zilla-examples/tree/main/http.proxy.schema.inline) ## Remote Catalogs Zilla can reference remote configs stored in third-party services like Schema Registry. The [schema-registry](../reference/config/catalogs/catalog-schema-registry.md) and [apicurio-registry](../reference/config/catalogs/catalog-apicurio-registry.md) catalogs allow zilla to interact with those services through their admin APIs. -> [Apicurio in the Petstore REST Demo](https://github.com/aklivity/zilla-demos/tree/main/petstore) | [http.kafka.karapace example](https://github.com/aklivity/zilla-examples/tree/main/http.kafka.karapace) +> [Remote Catalogs Guide](../how-tos/catalogs/index.md#remote-catalogs) | [Apicurio in the Petstore REST Demo](https://github.com/aklivity/zilla-demos/tree/main/petstore) | [http.kafka.karapace example](https://github.com/aklivity/zilla-examples/tree/main/http.kafka.karapace) diff --git a/src/concepts/models.md b/src/concepts/models.md new file mode 100644 index 00000000..473741b7 --- /dev/null +++ b/src/concepts/models.md @@ -0,0 +1,47 @@ +# Models + +When Zilla interacts with the data inside of a message, it only parses the necessary metadata with standard formats. The processing of messages is simple and uses fewer resources to proxy data streams. A [`model`](../reference/config/models/) adds the type syntax or structure definitions that Zilla needs to deserialize the remaining message parts. + +## Primitive Models + +Primitive models will have additional properties based on the type used. + +```yaml +model: string +encoding: utf_8 +``` + +> [Validating message keys](../how-tos/models/index.md#validating-message-keys) | [http.proxy.schema.inline example](https://github.com/aklivity/zilla-examples/tree/main/http.proxy.schema.inline) + +## Schema Models + +Schema-based models will reference a [catalog](./catalogs.md) to supply the binding with the configured model definition. Schemas referenced by their subject will fetch the latest version of that schema. + +- Fetch the latest schema by `subject`. + + ```yaml + model: avro + catalog: + my_catalog: + - subject: my_schema_subject + ``` + +- Fetch the latest schema by the schema definition on a Kafka `topic`. + + ```yaml + model: avro + catalog: + my_catalog: + - strategy: topic + ``` + +- Fetch a specific schema by its schema ID. + + ```yaml + model: avro + catalog: + my_catalog: + - id: 42 + ``` + +> [Validating a new message](../how-tos/models/index.md#validating-a-new-message) | [Expose a different model format](../how-tos/models/index.md#expose-a-different-model-format) | [http.kafka.karapace example](https://github.com/aklivity/zilla-examples/tree/main/http.kafka.karapace) diff --git a/src/how-tos/catalogs/index.md b/src/how-tos/catalogs/index.md new file mode 100644 index 00000000..06cc13fd --- /dev/null +++ b/src/how-tos/catalogs/index.md @@ -0,0 +1,66 @@ +# Registering a Catalog + +A registered catalog is a namespace scoped item in a zilla config that can be reused throughout the config. + +## Local Catalogs + +Local catalogs are used to quickly bootstrap a Zilla config or package a standard schema that doesn't change often with a Zilla install. + +A simple way to reference a file in Zilla is from the local filesystem. Adding [files to a Zilla pod](../deploy-operate.md#adding-files-to-the-zilla-pod) relative to the Zilla install directory. + +```yaml +catalogs: + my_host_filesystem_catalog: + type: filesystem + options: + subjects: + local_file: + path: relative/path/to/local.file +``` + +In some environments the local filesystem isn't accessible. Schema subjects can be definined inside of the Zilla config to keep all of the relevant model information in one place. + +```yaml +catalogs: + my_inline_catalog: + type: inline + options: + subjects: + my_avro_subject: + schema: | + { + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "status": { + "type": "string" + } + }, + "required": [ + "id", + "status" + ] + } +``` + +## Remote Catalogs + +Remote catalogs allow Zilla to fetch existing schemas stored in an external service. Zilla will pull and maintain an up to date version of the specified schema meaning the running Zilla doesn't need to be redeployed when new data models are pushed. Zilla can also reference specific versions. + +Registering remote catalogs is simple. Zilla needs to know the address for the registry and any relevant information about where the resources in the registry will be found. Once a remote catalog is registered the [model configs](../models/) will specify which resources and versions to fetch and use. + +```yaml +catalogs: + my_schema_registry_catalog: + type: schema-registry + options: + url: ${{env.SCHEMA_REGISTRY_URL}} + context: default + my_apicurio_registry_catalog: + type: apicurio-registry + options: + url: ${{env.APICURIO_URL}} + group-id: my_group_id +``` diff --git a/src/how-tos/deploy-operate.md b/src/how-tos/deploy-operate.md index 997577ac..c9449244 100644 --- a/src/how-tos/deploy-operate.md +++ b/src/how-tos/deploy-operate.md @@ -99,9 +99,52 @@ kubectl apply -f ingress-deploy.yaml The ingress controller will allow your ports to pass through, and you can configure which services should receive the requests made at those ports. -### Get diagnostics from zilla pods +### Adding files to the Zilla pod -For every running zilla pod you will need to first copy the `/var/run/zilla` directory to make sure no additional files are written while it is compressed then compress the full directory to make it easier to copy. +All local files referenced in a `zilla.yaml` config should be found in a location relative to the Zilla install location `/etc/zilla`. The best way to get your files into a pod is by using configmaps. Below you will find one option using configmaps and volume mounts to add your files into the Zilla pod. + +- From a single file. + + ```bash:no-line-numbers + kubectl create configmap my-files-configmap --from-file=my-file.txt -n $NAMESPACE -o yaml --dry-run=client | kubectl apply -f - + ``` + +- All files in a folder. This does not add folders recursively and each folder needs to be individually mapped + + ```bash:no-line-numbers + kubectl create configmap my-folder-configmap --from-file=path/to/my-folder/ -n $NAMESPACE -o yaml --dry-run=client | kubectl apply -f - + ``` + +Once you have the files you need stored in a configmap you can mount them as volumes into the Zilla pod at the install location `/etc/zilla`. + +::: + +::: code-tabs#bash + +@tab values.yaml + +```yaml +... +volumeMounts: + - name: my-files-volume + mountPath: /etc/zilla/files + - name: my-folder-volume + mountPath: /etc/zilla/folder + +volumes: + - name: my-files-volume + configMap: + name: my-files-configmap + - name: my-folder-volume + configMap: + name: my-folder-configmap +``` + +::: + +### Get diagnostics from Zilla pods + +For every running Zilla pod you will need to first copy the `/var/run/zilla` directory to make sure no additional files are written while it is compressed then compress the full directory to make it easier to copy. ```bash:no-line-numbers kubectl get pod \ diff --git a/src/how-tos/models/index.md b/src/how-tos/models/index.md new file mode 100644 index 00000000..d632c6d4 --- /dev/null +++ b/src/how-tos/models/index.md @@ -0,0 +1,96 @@ +# Structured Message data + +Adding structured types to the message data streams in Zilla. + +## Adding models structure to Kafka messages + +The `kafka` `cache_client` and `cache_server` bindings are responsible for interacting with the messages stored on Kafka topics. This is where Zilla can implement any structured type definitions. The schema for the message can come from the Kafka topic's schema definition using the topic `strategy` or be a reference to a schema's `subject` or `id`. The [catalog](../catalogs/index.md) definition will determine which methods are available when referencing schemas. + +### Validating message keys + +The message key for a topic can be set to any primitive model type and Zilla will validate the key when a message is Produced on a topic. + +```yaml + north_kafka_cache_client: + type: kafka + kind: cache_client + options: + topics: + - name: my-kafka-topic + key: + model: string +``` + +### Validating a new message + +The `kafka cache_client` binding can parse the message value, or body of the message, that is Produced on a topic. + +```yaml + north_kafka_cache_client: + type: kafka + kind: cache_client + options: + topics: + - name: my-kafka-topic + value: + model: avro + catalog: + my_catalog: + - strategy: topic +``` + +### Enforcing a schema on Fetch + +The `kafka cache_server` can enforce a schema on messages Fetched from a topic. This will prevent any messages that are pruduced on a Kafka topic from getting cosumed by a client if that messages doesn't match to the specified schema. + +```yaml + south_kafka_cache_server: + type: kafka + kind: cache_server + options: + bootstrap: + - my-kafka-topic + topics: + - name: my-kafka-topic + value: + model: avro + catalog: + my_catalog: + - strategy: topic +``` + +### Expose a different model format + +The `kafka cache_client` can read the `view` model and translate it into the specified `model` for when a message is produced on the Kafka topic. Then the `kafka cache_server` can read the `model` from the topic and translate it into the `view` model. + +In this case the `view` model that clients interact with needs to be a JSON object but the topic `model` is a serialize Avro object. + +```yaml + north_kafka_cache_client: + type: kafka + kind: cache_client + options: + topics: + - name: my-kafka-topic + value: + model: avro + view: json + catalog: + my_catalog: + - strategy: topic + exit: south_kafka_cache_server + south_kafka_cache_server: + type: kafka + kind: cache_server + options: + bootstrap: + - my-kafka-topic + topics: + - name: my-kafka-topic + value: + model: avro + view: json + catalog: + my_catalog: + - strategy: topic +``` From 47f9dd2861bac2d519b0bd4b8f0f0246305e626a Mon Sep 17 00:00:00 2001 From: vordimous Date: Fri, 20 Sep 2024 17:22:42 +0000 Subject: [PATCH 2/2] CI: update version to 2.1.3 --- deploy-versions.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy-versions.json b/deploy-versions.json index d7d505bc..f2717ac9 100644 --- a/deploy-versions.json +++ b/deploy-versions.json @@ -1 +1 @@ -[{"text":"Latest","icon":"fas fa-home","key":"latest","tag":"v2.1.2"}] +[{"text":"Latest","icon":"fas fa-home","key":"latest","tag":"v2.1.3"}] diff --git a/package.json b/package.json index 3d757a63..14a6b973 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "zilla-docs", "type": "module", - "version": "2.1.2", + "version": "2.1.3", "description": "The official documentation for the aklivity/zilla open-source project", "keywords": [], "author": "aklivity.io",