source-kafka: capture Avro encoded messages using schema registry

Implements capturing Avro encoded messages using schemas retrieved from the configured schema registry. The hard work is mostly translating an Avro value into a JSON value in a way that is maximally useful. The implementation here is a lot like the one in the Flow parser, but since we are unconstrained with backward-compatibility in the Kafka capture it includes some enhancements that would be difficult to include in the parser's version.
estuary · Oct 30, 2024 · c3b1323 · c3b1323
1 parent 1c194ae
commit c3b1323
Show file tree

Hide file tree

Showing 8 changed files with 685 additions and 155 deletions.
diff --git a/source-kafka/Cargo.lock b/source-kafka/Cargo.lock
diff --git a/source-kafka/Cargo.toml b/source-kafka/Cargo.toml
@@ -31,8 +31,11 @@ tracing-subscriber = { version = "0.3", features = [
 reqwest = { version = "0.12", features = ["json"] }
 futures = "0.3"
 apache-avro = "0.17"
+bigdecimal = "0.4"
+time = "0.3"
 
 
 [dev-dependencies]
 insta = { version = "1", features = ["json", "serde"] }
 schema_registry_converter = { version = "4.2.0", features = ["avro"] }
+uuid = "1.11"