From acc461ff95349a460c62b6995e9cbb129766318a Mon Sep 17 00:00:00 2001 From: Johnny Graettinger Date: Mon, 4 Nov 2024 10:28:23 -0600 Subject: [PATCH] derive-sqlite: improve transform stub generation Lead with an identity-transform example SELECT JSON($flow_document) Continue to emit an example selecting individual fields, but semantically disable it and also remove root document projections, because these are never desired. --- ...lite__validate__test__stub_generation.snap | 15 ++++ crates/derive-sqlite/src/validate.rs | 90 +++++++++++++++++-- 2 files changed, 96 insertions(+), 9 deletions(-) create mode 100644 crates/derive-sqlite/src/snapshots/derive_sqlite__validate__test__stub_generation.snap diff --git a/crates/derive-sqlite/src/snapshots/derive_sqlite__validate__test__stub_generation.snap b/crates/derive-sqlite/src/snapshots/derive_sqlite__validate__test__stub_generation.snap new file mode 100644 index 0000000000..56ca8daa8d --- /dev/null +++ b/crates/derive-sqlite/src/snapshots/derive_sqlite__validate__test__stub_generation.snap @@ -0,0 +1,15 @@ +--- +source: crates/derive-sqlite/src/validate.rs +expression: validated +--- +{ + "transforms": [ + { + "readOnly": true + } + ], + "generatedFiles": { + "file://path/to/migration.stub": "\n-- Use migrations to create or alter tables that your derivation will use.\n-- Each migration is run only once, and new migrations will be applied as needed.\n--\n-- For example, create the join table below, and then use it across multiple lambdas:\n--\n-- A first lambda that updates indexed state:\n--\n-- INSERT INTO my_join_table (id, joined_value) VALUES ($id, $my::value)\n-- ON CONFLICT REPLACE;\n--\n-- A second lambda that reads out and joins over the indexed state:\n--\n-- SELECT $id, $other$value, j.joined_value FROM my_join_table WHERE id = $id;\n\nCREATE TABLE my_join_table (\n -- A common ID that's joined over.\n id INTEGER PRIMARY KEY NOT NULL,\n -- A value that's updated by one lambda, and read by another.\n joined_value TEXT NOT NULL\n);\n\n", + "file://path/to/transform.stub.sql": "\n-- Example statement which passes-through source acmeCo/foo/bar documents without modification.\n-- Use a WHERE clause to filter, for example: WHERE $my$column = 1234\nSELECT JSON($flow_document);\n\n-- Example statement demonstrating how to SELECT specific locations from documents of acmeCo/foo/bar.\n-- This statement is effectively disabled by its WHERE FALSE clause and does not emit any documents.\n--\n-- You can rename a location by using the SQL \"AS\" syntax, for example:\n-- SELECT $some$column AS \"my_new_column_name;\n--\n-- You can also filter by using locations in a WHERE clause, for example:\n-- SELECT $some$column WHERE $other$column = 1234;\nSELECT\n -- Key id at /id\n $id,\n -- Partitioned field part at /part\n $part,\n -- Field nested/int at /nested/int\n $nested$int,\n -- Field value at /value\n $value\n-- Disable this statement, so that it emits no documents.\nWHERE FALSE;\n" + } +} diff --git a/crates/derive-sqlite/src/validate.rs b/crates/derive-sqlite/src/validate.rs index 0c4faae635..70a4c483fd 100644 --- a/crates/derive-sqlite/src/validate.rs +++ b/crates/derive-sqlite/src/validate.rs @@ -107,20 +107,20 @@ const MIGRATION_STUB: &str = r#" -- -- A second lambda that reads out and joins over the indexed state: -- --- SELECT $id, $other::value, j.joined_value FROM my_join_table WHERE id = $id; +-- SELECT $id, $other$value, j.joined_value FROM my_join_table WHERE id = $id; -create table my_join_table ( +CREATE TABLE my_join_table ( -- A common ID that's joined over. - id integer primary key not null, + id INTEGER PRIMARY KEY NOT NULL, -- A value that's updated by one lambda, and read by another. - joined_value text not null + joined_value TEXT NOT NULL ); "#; fn lambda_stub( Transform { - name, + name: _, source, block: _, params, @@ -129,11 +129,32 @@ fn lambda_stub( use std::fmt::Write; let mut w = String::with_capacity(4096); + let root = params.iter().find(|param| param.projection.ptr.is_empty()); + + if let Some(root) = root { + _ = write!( + w, + r#" +-- Example statement which passes-through source {source} documents without modification. +-- Use a WHERE clause to filter, for example: WHERE $my$column = 1234 +SELECT JSON({}); +"#, + root.canonical_encoding + ); + } + _ = write!( w, r#" --- Example select statement for transform {name} of source collection {source}. -select +-- Example statement demonstrating how to SELECT specific locations from documents of {source}. +-- This statement is effectively disabled by its WHERE FALSE clause and does not emit any documents. +-- +-- You can rename a location by using the SQL "AS" syntax, for example: +-- SELECT $some$column AS "my_new_column_name; +-- +-- You can also filter by using locations in a WHERE clause, for example: +-- SELECT $some$column WHERE $other$column = 1234; +SELECT "#, ); @@ -167,11 +188,62 @@ select let params: Vec = params .iter() - .map(|p| format!(" -- {}\n {}", comment(p), p.canonical_encoding)) + .filter_map(|p| { + if p.projection.ptr.is_empty() { + None // Skip projection of the document root. + } else { + Some(format!( + " -- {}\n {}", + comment(p), + p.canonical_encoding + )) + } + }) .collect(); w.push_str(¶ms.join(",\n")); - w.push_str("\n;"); + w.push_str( + r#" +-- Disable this statement, so that it emits no documents. +WHERE FALSE; +"#, + ); w } + +#[cfg(test)] +mod test { + use super::super::test_param; + use super::do_validate; + use crate::Transform; + + #[test] + fn test_stub_generation() { + let mut params = vec![ + test_param("id", "/id", false, false, false), + test_param("part", "/part", false, false, false), + test_param("flow_document", "", false, false, false), + test_param("nested/int", "/nested/int", false, true, false), + test_param("value", "/value", false, false, false), + ]; + params[0].projection.is_primary_key = true; + params[1].projection.is_partition_key = true; + + let migrations = vec![ + "file://path/to/migration.stub".to_string(), + "CREATE TABLE foo (one TEXT, two INTEGER);".to_string(), + ]; + + let transforms = vec![Transform { + block: "file://path/to/transform.stub.sql".to_string(), + name: "fromFoobar".to_string(), + source: "acmeCo/foo/bar".to_string(), + params, + }]; + + let validated = do_validate(&migrations, &transforms).unwrap(); + + insta::assert_json_snapshot!(validated); + } +}