From 8ddb527467c204e4750ec6821591b1c6731daaa6 Mon Sep 17 00:00:00 2001 From: keppere <2keppere@gmail.com> Date: Fri, 17 Jan 2025 22:33:01 +0700 Subject: [PATCH] refactor: improve query placeholder --- src/components/gui/tabs/query-tab.tsx | 72 +++++++++++++++------------ src/lib/sql/tokenizer.test.ts | 37 ++++++++++++++ src/lib/sql/tokenizer.ts | 9 +++- 3 files changed, 84 insertions(+), 34 deletions(-) diff --git a/src/components/gui/tabs/query-tab.tsx b/src/components/gui/tabs/query-tab.tsx index e6ae6c30..583efa61 100644 --- a/src/components/gui/tabs/query-tab.tsx +++ b/src/components/gui/tabs/query-tab.tsx @@ -87,31 +87,30 @@ export default function QueryWindow({ initialNamespace ?? "Unsaved Query" ); const [savedKey, setSavedKey] = useState(initialSavedKey); - const [placeHolders, setPlaceHolders] = useState>({}); + const [placeholders, setPlaceholders] = useState>({}); useEffect(() => { const timer = setTimeout(() => { - const editorState = editorRef.current?.view?.state; - if (!editorState) return; - const finalStatements = splitSqlQuery(editorState).map((q) => q.text); - const newPlaceholders: Record = {}; - for (const statement of finalStatements) { - const token = tokenizeSql(statement, databaseDriver.getFlags().dialect); - const placeholders = token + setPlaceholders((prev) => { + const newPlaceholders: Record = {}; + const token = tokenizeSql(code, databaseDriver.getFlags().dialect); + const foundPlaceholders = token .filter((t) => t.type === "PLACEHOLDER") - .map((t) => t.value.split(":")[1]); - for (const placeholder of placeholders) { - newPlaceholders[placeholder] = ""; + .map((t) => t.value.slice(1)); + + for (const foundPlaceholder of foundPlaceholders) { + newPlaceholders[foundPlaceholder] = ""; } - } - for (const newKey of Object.keys(newPlaceholders)) { - newPlaceholders[newKey] = placeHolders[newKey] ?? ""; - } - setPlaceHolders(newPlaceholders); + // write old placeholders value into new placeholders + for (const newKey of Object.keys(newPlaceholders)) { + newPlaceholders[newKey] = prev[newKey] ?? ""; + } + + return { ...newPlaceholders }; + }); }, 1000); return () => clearTimeout(timer); - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [code]); + }, [code, databaseDriver]); const onFormatClicked = () => { try { @@ -166,26 +165,33 @@ export default function QueryWindow({ setQueryTabIndex(0); //inject placeholders - for (const statement of finalStatements) { - const token = tokenizeSql(statement, databaseDriver.getFlags().dialect); + for (let i = 0; i < finalStatements.length; i++) { + const token = tokenizeSql( + finalStatements[i], + databaseDriver.getFlags().dialect + ); + const variables = token .filter((t) => t.type === "PLACEHOLDER") - .map((t) => t.value.split(":")[1]); + .map((t) => t.value.slice(1)); if ( variables.length > 0 && - variables.some((p) => placeHolders[p] === "") + variables.some((p) => placeholders[p] === "") ) { toast.error("Please fill in all placeholders"); return; } - } - for (const key of Object.keys(placeHolders)) { - finalStatements = finalStatements.map((s) => - s.replace( - new RegExp(`:${key}`, "g"), - escapeSqlValue(extractInputValue(placeHolders[key])) - ) - ); + + finalStatements[i] = token + .map((t) => { + if (t.type === "PLACEHOLDER") { + return escapeSqlValue( + extractInputValue(placeholders[t.value.slice(1)]) + ); + } + return t.value; + }) + .join(""); } multipleQuery(databaseDriver, finalStatements, (currentProgress) => { @@ -400,10 +406,10 @@ export default function QueryWindow({
Col {columnNumber + 1}
- {Object.keys(placeHolders).length > 0 && ( + {Object.keys(placeholders).length > 0 && ( )}
diff --git a/src/lib/sql/tokenizer.test.ts b/src/lib/sql/tokenizer.test.ts index 71cd04e6..01adc37a 100644 --- a/src/lib/sql/tokenizer.test.ts +++ b/src/lib/sql/tokenizer.test.ts @@ -406,3 +406,40 @@ test("INNER JOIN", () => { ]); expect(tokens.map((t) => t.value).join("")).toBe(sql); }); + +test("Accumulate unknown tokens", () => { + const sql = `SELECT * FROM customers WHERE name = 'John Doe' ### this # is a comment ##`; + const tokens = tokenizeSql(sql, "sqlite"); + expect(tokens).toEqual([ + { type: "IDENTIFIER", value: "SELECT" }, + { type: "WHITESPACE", value: " " }, + { type: "OPERATOR", value: "*" }, + { type: "WHITESPACE", value: " " }, + { type: "IDENTIFIER", value: "FROM" }, + { type: "WHITESPACE", value: " " }, + { type: "IDENTIFIER", value: "customers" }, + { type: "WHITESPACE", value: " " }, + { type: "IDENTIFIER", value: "WHERE" }, + { type: "WHITESPACE", value: " " }, + { type: "IDENTIFIER", value: "name" }, + { type: "WHITESPACE", value: " " }, + { type: "OPERATOR", value: "=" }, + { type: "WHITESPACE", value: " " }, + { type: "STRING", value: "'John Doe'" }, + { type: "WHITESPACE", value: " " }, + { type: "UNKNOWN", value: "###" }, + { type: "WHITESPACE", value: " " }, + { type: "IDENTIFIER", value: "this" }, + { type: "WHITESPACE", value: " " }, + { type: "UNKNOWN", value: "#" }, + { type: "WHITESPACE", value: " " }, + { type: "IDENTIFIER", value: "is" }, + { type: "WHITESPACE", value: " " }, + { type: "IDENTIFIER", value: "a" }, + { type: "WHITESPACE", value: " " }, + { type: "IDENTIFIER", value: "comment" }, + { type: "WHITESPACE", value: " " }, + { type: "UNKNOWN", value: "##" }, + ]); + expect(tokens.map((t) => t.value).join("")).toBe(sql); +}); diff --git a/src/lib/sql/tokenizer.ts b/src/lib/sql/tokenizer.ts index 300ec94d..5d82473c 100644 --- a/src/lib/sql/tokenizer.ts +++ b/src/lib/sql/tokenizer.ts @@ -84,6 +84,7 @@ export function tokenizeSql(sql: string, dialect: SupportedDialect): Token[] { const tokens: Token[] = []; let cursor = 0; const length = sql.length; + let unknownAcc = ""; while (cursor < length) { let matched = false; @@ -91,6 +92,11 @@ export function tokenizeSql(sql: string, dialect: SupportedDialect): Token[] { for (const { type, findToken } of tokenTypes) { const match = findToken(subStr, dialect); if (match) { + if (unknownAcc !== "") { + tokens.push({ type: "UNKNOWN", value: unknownAcc }); + unknownAcc = ""; + } + tokens.push({ type, value: match }); cursor += match.length; matched = true; @@ -99,10 +105,11 @@ export function tokenizeSql(sql: string, dialect: SupportedDialect): Token[] { } if (!matched) { - tokens.push({ type: "UNKNOWN", value: subStr[0] }); + unknownAcc += subStr[0]; cursor++; } } + if (unknownAcc !== "") tokens.push({ type: "UNKNOWN", value: unknownAcc }); return tokens; } catch (e) { return [{ type: "SQL", value: sql }];