Skip to content

Commit

Permalink
derive-sqlite: improve transform stub generation
Browse files Browse the repository at this point in the history
Lead with an identity-transform example SELECT JSON($flow_document)

Continue to emit an example selecting individual fields, but
semantically disable it and also remove root document projections,
because these are never desired.
  • Loading branch information
jgraettinger committed Nov 5, 2024
1 parent 3d77f86 commit acc461f
Show file tree
Hide file tree
Showing 2 changed files with 96 additions and 9 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
---
source: crates/derive-sqlite/src/validate.rs
expression: validated
---
{
"transforms": [
{
"readOnly": true
}
],
"generatedFiles": {
"file://path/to/migration.stub": "\n-- Use migrations to create or alter tables that your derivation will use.\n-- Each migration is run only once, and new migrations will be applied as needed.\n--\n-- For example, create the join table below, and then use it across multiple lambdas:\n--\n-- A first lambda that updates indexed state:\n--\n-- INSERT INTO my_join_table (id, joined_value) VALUES ($id, $my::value)\n-- ON CONFLICT REPLACE;\n--\n-- A second lambda that reads out and joins over the indexed state:\n--\n-- SELECT $id, $other$value, j.joined_value FROM my_join_table WHERE id = $id;\n\nCREATE TABLE my_join_table (\n -- A common ID that's joined over.\n id INTEGER PRIMARY KEY NOT NULL,\n -- A value that's updated by one lambda, and read by another.\n joined_value TEXT NOT NULL\n);\n\n",
"file://path/to/transform.stub.sql": "\n-- Example statement which passes-through source acmeCo/foo/bar documents without modification.\n-- Use a WHERE clause to filter, for example: WHERE $my$column = 1234\nSELECT JSON($flow_document);\n\n-- Example statement demonstrating how to SELECT specific locations from documents of acmeCo/foo/bar.\n-- This statement is effectively disabled by its WHERE FALSE clause and does not emit any documents.\n--\n-- You can rename a location by using the SQL \"AS\" syntax, for example:\n-- SELECT $some$column AS \"my_new_column_name;\n--\n-- You can also filter by using locations in a WHERE clause, for example:\n-- SELECT $some$column WHERE $other$column = 1234;\nSELECT\n -- Key id at /id\n $id,\n -- Partitioned field part at /part\n $part,\n -- Field nested/int at /nested/int\n $nested$int,\n -- Field value at /value\n $value\n-- Disable this statement, so that it emits no documents.\nWHERE FALSE;\n"
}
}
90 changes: 81 additions & 9 deletions crates/derive-sqlite/src/validate.rs
Original file line number Diff line number Diff line change
Expand Up @@ -107,20 +107,20 @@ const MIGRATION_STUB: &str = r#"
--
-- A second lambda that reads out and joins over the indexed state:
--
-- SELECT $id, $other::value, j.joined_value FROM my_join_table WHERE id = $id;
-- SELECT $id, $other$value, j.joined_value FROM my_join_table WHERE id = $id;
create table my_join_table (
CREATE TABLE my_join_table (
-- A common ID that's joined over.
id integer primary key not null,
id INTEGER PRIMARY KEY NOT NULL,
-- A value that's updated by one lambda, and read by another.
joined_value text not null
joined_value TEXT NOT NULL
);
"#;

fn lambda_stub(
Transform {
name,
name: _,
source,
block: _,
params,
Expand All @@ -129,11 +129,32 @@ fn lambda_stub(
use std::fmt::Write;
let mut w = String::with_capacity(4096);

let root = params.iter().find(|param| param.projection.ptr.is_empty());

if let Some(root) = root {
_ = write!(
w,
r#"
-- Example statement which passes-through source {source} documents without modification.
-- Use a WHERE clause to filter, for example: WHERE $my$column = 1234
SELECT JSON({});
"#,
root.canonical_encoding
);
}

_ = write!(
w,
r#"
-- Example select statement for transform {name} of source collection {source}.
select
-- Example statement demonstrating how to SELECT specific locations from documents of {source}.
-- This statement is effectively disabled by its WHERE FALSE clause and does not emit any documents.
--
-- You can rename a location by using the SQL "AS" syntax, for example:
-- SELECT $some$column AS "my_new_column_name;
--
-- You can also filter by using locations in a WHERE clause, for example:
-- SELECT $some$column WHERE $other$column = 1234;
SELECT
"#,
);

Expand Down Expand Up @@ -167,11 +188,62 @@ select

let params: Vec<String> = params
.iter()
.map(|p| format!(" -- {}\n {}", comment(p), p.canonical_encoding))
.filter_map(|p| {
if p.projection.ptr.is_empty() {
None // Skip projection of the document root.
} else {
Some(format!(
" -- {}\n {}",
comment(p),
p.canonical_encoding
))
}
})
.collect();

w.push_str(&params.join(",\n"));
w.push_str("\n;");
w.push_str(
r#"
-- Disable this statement, so that it emits no documents.
WHERE FALSE;
"#,
);

w
}

#[cfg(test)]
mod test {
use super::super::test_param;
use super::do_validate;
use crate::Transform;

#[test]
fn test_stub_generation() {
let mut params = vec![
test_param("id", "/id", false, false, false),
test_param("part", "/part", false, false, false),
test_param("flow_document", "", false, false, false),
test_param("nested/int", "/nested/int", false, true, false),
test_param("value", "/value", false, false, false),
];
params[0].projection.is_primary_key = true;
params[1].projection.is_partition_key = true;

let migrations = vec![
"file://path/to/migration.stub".to_string(),
"CREATE TABLE foo (one TEXT, two INTEGER);".to_string(),
];

let transforms = vec![Transform {
block: "file://path/to/transform.stub.sql".to_string(),
name: "fromFoobar".to_string(),
source: "acmeCo/foo/bar".to_string(),
params,
}];

let validated = do_validate(&migrations, &transforms).unwrap();

insta::assert_json_snapshot!(validated);
}
}

0 comments on commit acc461f

Please sign in to comment.