diff --git a/packages/modeldb-durable-objects/src/api.ts b/packages/modeldb-durable-objects/src/api.ts index 83cf46f51..58773c287 100644 --- a/packages/modeldb-durable-objects/src/api.ts +++ b/packages/modeldb-durable-objects/src/api.ts @@ -20,7 +20,13 @@ import { PropertyValue, } from "@canvas-js/modeldb" -import { decodePrimitiveValue, decodeReferenceValue, encodePrimitiveValue, encodeReferenceValue } from "./encoding.js" +import { + SqlitePrimitiveValue, + decodePrimitiveValue, + decodeReferenceValue, + encodePrimitiveValue, + encodeReferenceValue, +} from "./encoding.js" import { Method, Query } from "./utils.js" @@ -68,8 +74,8 @@ export class ModelAPI { string, { columns: string[] - encode: (value: PropertyValue) => SqlStorageValue[] - decode: (record: Record) => PropertyValue + encode: (value: PropertyValue) => SqlitePrimitiveValue[] + decode: (record: Record) => PropertyValue } > = {} @@ -275,8 +281,8 @@ export class ModelAPI { } } - private encodeProperties(properties: Property[], value: ModelValue): SqlStorageValue[] { - const result: SqlStorageValue[] = [] + private encodeProperties(properties: Property[], value: ModelValue): SqlitePrimitiveValue[] { + const result: SqlitePrimitiveValue[] = [] for (const property of properties) { if (property.kind === "primitive") { const { name, type, nullable } = property @@ -320,7 +326,7 @@ export class ModelAPI { public count(where?: WhereCondition): number { const sql: string[] = [] - const params: SqlStorageValue[] = [] + const params: SqlitePrimitiveValue[] = [] // SELECT sql.push(`SELECT COUNT(*) AS count FROM "${this.#table}"`) @@ -357,7 +363,11 @@ export class ModelAPI { } } - private parseRecord(row: Record, properties: string[], relations: Relation[]): ModelValue { + private parseRecord( + row: Record, + properties: string[], + relations: Relation[], + ): ModelValue { const record: ModelValue = {} for (const name of properties) { record[name] = this.codecs[name].decode(row) @@ -381,10 +391,10 @@ export class ModelAPI { private parseQuery( query: QueryParams, - ): [sql: string, properties: string[], relations: Relation[], params: SqlStorageValue[]] { + ): [sql: string, properties: string[], relations: Relation[], params: SqlitePrimitiveValue[]] { // See https://www.sqlite.org/lang_select.html for railroad diagram const sql: string[] = [] - const params: SqlStorageValue[] = [] + const params: SqlitePrimitiveValue[] = [] // SELECT const select = query.select ?? mapValues(this.#properties, () => true) @@ -471,8 +481,8 @@ export class ModelAPI { return [columns.join(", "), properties, relations] } - private getWhereExpression(where: WhereCondition = {}): [where: string | null, params: SqlStorageValue[]] { - const params: SqlStorageValue[] = [] + private getWhereExpression(where: WhereCondition = {}): [where: string | null, params: SqlitePrimitiveValue[]] { + const params: SqlitePrimitiveValue[] = [] const filters: string[] = [] for (const [name, expression] of Object.entries(where)) { @@ -671,18 +681,18 @@ export class RelationAPI { this.#select = new Query(this.db, `SELECT ${targetColumns} FROM "${this.table}" WHERE ${selectBySource}`) } - public get(sourceKey: SqlStorageValue[]): SqlStorageValue[][] { + public get(sourceKey: SqlitePrimitiveValue[]): SqlitePrimitiveValue[][] { const targets = this.#select.all(sourceKey) return targets.map((record) => this.sourceColumnNames.map((name) => record[name])) } - public add(sourceKey: SqlStorageValue[], targetKeys: SqlStorageValue[][]) { + public add(sourceKey: SqlitePrimitiveValue[], targetKeys: SqlitePrimitiveValue[][]) { for (const targetKey of targetKeys) { this.#insert.run([...sourceKey, ...targetKey]) } } - public delete(sourceKey: SqlStorageValue[]) { + public delete(sourceKey: SqlitePrimitiveValue[]) { this.#delete.run(sourceKey) } diff --git a/packages/modeldb-durable-objects/src/encoding.ts b/packages/modeldb-durable-objects/src/encoding.ts index 5f9b1f5f7..ee929522b 100644 --- a/packages/modeldb-durable-objects/src/encoding.ts +++ b/packages/modeldb-durable-objects/src/encoding.ts @@ -6,6 +6,8 @@ import { PrimaryKeyValue, PrimitiveProperty, PrimitiveType, PrimitiveValue, Prop import { assert, signalInvalidType } from "@canvas-js/utils" +export type SqlitePrimitiveValue = SqlStorageValue + export function toArrayBuffer(data: Uint8Array): ArrayBuffer { if (data.byteOffset === 0 && data.byteLength === data.buffer.byteLength) { return data.buffer @@ -25,7 +27,7 @@ export function encodePrimitiveValue( type: PrimitiveType, nullable: boolean, value: PropertyValue, -): SqlStorageValue { +): SqlitePrimitiveValue { if (value === null) { if (nullable) { return null @@ -80,7 +82,7 @@ export function encodeReferenceValue( target: PrimitiveProperty[], nullable: boolean, value: PropertyValue, -): SqlStorageValue[] { +): SqlitePrimitiveValue[] { if (value === null) { if (nullable) { return Array.from({ length: target.length }).fill(null) @@ -101,7 +103,7 @@ export function decodePrimitiveValue( propertyName: string, type: PrimitiveType, nullable: boolean, - value: SqlStorageValue, + value: SqlitePrimitiveValue, ): PrimitiveValue { if (value === null) { if (nullable) { @@ -161,7 +163,7 @@ export function decodeReferenceValue( propertyName: string, nullable: boolean, target: PrimitiveProperty[], - values: SqlStorageValue[], + values: SqlitePrimitiveValue[], ): PrimaryKeyValue | PrimaryKeyValue[] | null { if (values.every((value) => value === null)) { if (nullable) { diff --git a/packages/modeldb-sqlite-expo/src/ModelDB.ts b/packages/modeldb-sqlite-expo/src/ModelDB.ts index 903859bc4..9338015cc 100644 --- a/packages/modeldb-sqlite-expo/src/ModelDB.ts +++ b/packages/modeldb-sqlite-expo/src/ModelDB.ts @@ -6,12 +6,12 @@ import { assert, signalInvalidType } from "@canvas-js/utils" import { AbstractModelDB, ModelDBBackend, - parseConfig, Effect, ModelValue, ModelSchema, QueryParams, WhereCondition, + Config, } from "@canvas-js/modeldb" import { ModelAPI } from "./api.js" @@ -28,7 +28,7 @@ export class ModelDB extends AbstractModelDB { #transaction: (effects: Effect[]) => void constructor({ path, models, clear }: { clear?: boolean } & ModelDBOptions) { - super(parseConfig(models)) + super(Config.parse(models)) this.db = SQLite.openDatabaseSync(path ?? ":memory:") diff --git a/packages/modeldb-sqlite-expo/src/api.ts b/packages/modeldb-sqlite-expo/src/api.ts index 2c595f3b3..89c9a5933 100644 --- a/packages/modeldb-sqlite-expo/src/api.ts +++ b/packages/modeldb-sqlite-expo/src/api.ts @@ -1,4 +1,4 @@ -import { SQLiteDatabase, SQLiteBindValue } from "expo-sqlite" +import { SQLiteDatabase } from "expo-sqlite" import { assert, signalInvalidType, mapValues } from "@canvas-js/utils" @@ -20,9 +20,13 @@ import { PropertyValue, } from "@canvas-js/modeldb" -import { zip } from "@canvas-js/utils" - -import { decodePrimitiveValue, decodeReferenceValue, encodePrimitiveValue, encodeReferenceValue } from "./encoding.js" +import { + SqlitePrimitiveValue, + decodePrimitiveValue, + decodeReferenceValue, + encodePrimitiveValue, + encodeReferenceValue, +} from "./encoding.js" import { Method, Query } from "./utils.js" @@ -70,8 +74,8 @@ export class ModelAPI { string, { columns: string[] - encode: (value: PropertyValue) => SQLiteBindValue[] - decode: (record: Record) => PropertyValue + encode: (value: PropertyValue) => SqlitePrimitiveValue[] + decode: (record: Record) => PropertyValue } > = {} @@ -276,8 +280,8 @@ export class ModelAPI { } } - private encodeProperties(properties: Property[], value: ModelValue): SQLiteBindValue[] { - const result: SQLiteBindValue[] = [] + private encodeProperties(properties: Property[], value: ModelValue): SqlitePrimitiveValue[] { + const result: SqlitePrimitiveValue[] = [] for (const property of properties) { if (property.kind === "primitive") { const { name, type, nullable } = property @@ -358,7 +362,11 @@ export class ModelAPI { } } - private parseRecord(row: Record, properties: string[], relations: Relation[]): ModelValue { + private parseRecord( + row: Record, + properties: string[], + relations: Relation[], + ): ModelValue { const record: ModelValue = {} for (const name of properties) { record[name] = this.codecs[name].decode(row) @@ -382,10 +390,10 @@ export class ModelAPI { private parseQuery( query: QueryParams, - ): [sql: string, properties: string[], relations: Relation[], params: SQLiteBindValue[]] { + ): [sql: string, properties: string[], relations: Relation[], params: SqlitePrimitiveValue[]] { // See https://www.sqlite.org/lang_select.html for railroad diagram const sql: string[] = [] - const params: SQLiteBindValue[] = [] + const params: SqlitePrimitiveValue[] = [] // SELECT const select = query.select ?? mapValues(this.#properties, () => true) @@ -672,18 +680,18 @@ export class RelationAPI { this.#select = new Query(this.db, `SELECT ${targetColumns} FROM "${this.table}" WHERE ${selectBySource}`) } - public get(sourceKey: SQLiteBindValue[]): SQLiteBindValue[][] { + public get(sourceKey: SqlitePrimitiveValue[]): SqlitePrimitiveValue[][] { const targets = this.#select.all(sourceKey) return targets.map((record) => this.sourceColumnNames.map((name) => record[name])) } - public add(sourceKey: SQLiteBindValue[], targetKeys: SQLiteBindValue[][]) { + public add(sourceKey: SqlitePrimitiveValue[], targetKeys: SqlitePrimitiveValue[][]) { for (const targetKey of targetKeys) { this.#insert.run([...sourceKey, ...targetKey]) } } - public delete(sourceKey: SQLiteBindValue[]) { + public delete(sourceKey: SqlitePrimitiveValue[]) { this.#delete.run(sourceKey) } diff --git a/packages/modeldb-sqlite-expo/src/encoding.ts b/packages/modeldb-sqlite-expo/src/encoding.ts index b65a00b8c..79ebd697e 100644 --- a/packages/modeldb-sqlite-expo/src/encoding.ts +++ b/packages/modeldb-sqlite-expo/src/encoding.ts @@ -12,12 +12,14 @@ import type { import { assert, signalInvalidType } from "@canvas-js/utils" +export type SqlitePrimitiveValue = SQLiteBindValue + export function encodePrimitiveValue( propertyName: string, type: PrimitiveType, nullable: boolean, value: PropertyValue, -): SQLiteBindValue { +): SqlitePrimitiveValue { if (value === null) { if (nullable) { return null @@ -72,7 +74,7 @@ export function encodeReferenceValue( target: PrimitiveProperty[], nullable: boolean, value: PropertyValue, -): SQLiteBindValue[] { +): SqlitePrimitiveValue[] { if (value === null) { if (nullable) { return Array.from({ length: target.length }).fill(null) @@ -93,7 +95,7 @@ export function decodePrimitiveValue( propertyName: string, type: PrimitiveType, nullable: boolean, - value: SQLiteBindValue, + value: SqlitePrimitiveValue, ): PrimitiveValue { if (value === null) { if (nullable) { @@ -153,7 +155,7 @@ export function decodeReferenceValue( propertyName: string, nullable: boolean, target: PrimitiveProperty[], - values: SQLiteBindValue[], + values: SqlitePrimitiveValue[], ): PrimaryKeyValue | PrimaryKeyValue[] | null { if (values.every((value) => value === null)) { if (nullable) { diff --git a/packages/modeldb-sqlite-expo/src/utils.ts b/packages/modeldb-sqlite-expo/src/utils.ts index 1d9904ab1..a76c2b7e0 100644 --- a/packages/modeldb-sqlite-expo/src/utils.ts +++ b/packages/modeldb-sqlite-expo/src/utils.ts @@ -1,6 +1,9 @@ import { SQLiteBindValue, SQLiteDatabase, SQLiteStatement } from "expo-sqlite" -export class Query

> { +export class Query< + P extends SQLiteBindValue[] = SQLiteBindValue[], + R extends Record = Record, +> { private readonly statement: SQLiteStatement constructor(db: SQLiteDatabase, private readonly sql: string) { diff --git a/packages/modeldb-sqlite-wasm/src/InnerModelDB.ts b/packages/modeldb-sqlite-wasm/src/InnerModelDB.ts index 2d241d9a8..d54494b11 100644 --- a/packages/modeldb-sqlite-wasm/src/InnerModelDB.ts +++ b/packages/modeldb-sqlite-wasm/src/InnerModelDB.ts @@ -53,8 +53,29 @@ export class InnerModelDB { public iterate(modelName: string, query: QueryParams = {}): AsyncIterable { const api = this.#models[modelName] - assert(api !== undefined, `model ${modelName} not found`) - return Comlink.proxy(api.iterate(query)) + if (api === undefined) { + throw new Error(`model ${modelName} not found`) + } + + return Comlink.proxy({ + [Symbol.asyncIterator]() { + const iter = api.iterate(query) + + return { + async next() { + return iter.next() + }, + + async return(value) { + return iter.return?.(value) ?? { done: true, value: undefined } + }, + + async throw(err) { + return iter.throw?.(err) ?? { done: true, value: undefined } + }, + } + }, + }) } public count(modelName: string, where?: WhereCondition): number { diff --git a/packages/modeldb-sqlite-wasm/src/ModelAPI.ts b/packages/modeldb-sqlite-wasm/src/ModelAPI.ts index a87ec3478..9500e3586 100644 --- a/packages/modeldb-sqlite-wasm/src/ModelAPI.ts +++ b/packages/modeldb-sqlite-wasm/src/ModelAPI.ts @@ -1,5 +1,7 @@ import { OpfsDatabase } from "@sqlite.org/sqlite-wasm" +import { assert, signalInvalidType, mapValues } from "@canvas-js/utils" + import { Property, Relation, @@ -8,330 +10,401 @@ import { PrimitiveType, QueryParams, WhereCondition, - PrimitiveValue, - RangeExpression, isNotExpression, isLiteralExpression, isRangeExpression, - isPrimitiveValue, validateModelValue, PrimitiveProperty, Config, PrimaryKeyValue, - isPrimaryKey, + PropertyValue, } from "@canvas-js/modeldb" -import { assert, signalInvalidType, mapValues, zip } from "@canvas-js/utils" - import { - RecordValue, - RecordParams, + SqlitePrimitiveValue, decodePrimitiveValue, - decodeRecord, decodeReferenceValue, - encodeRecordParams, + encodePrimitiveValue, + encodeReferenceValue, } from "./encoding.js" import { Method, Query } from "./utils.js" -const primitiveColumnTypes = { +const columnTypes = { integer: "INTEGER", float: "FLOAT", - number: "NUMERIC", + number: "FLOAT", string: "TEXT", bytes: "BLOB", boolean: "INTEGER", json: "TEXT", } satisfies Record -function getPropertyColumnType(config: Config, model: Model, property: Property): string { - if (property.kind === "primitive") { - const type = primitiveColumnTypes[property.type] - - if (property.name === model.primaryKey) { - assert(property.nullable === false) - return `${type} PRIMARY KEY NOT NULL` - } - - return property.nullable ? type : `${type} NOT NULL` - } else if (property.kind === "reference") { - const target = config.models.find((model) => model.name === property.target) - assert(target !== undefined) - - const targetPrimaryKey = target.properties.find((property) => property.name === target.primaryKey) - assert(targetPrimaryKey !== undefined) - assert(targetPrimaryKey.kind === "primitive") - - const type = primitiveColumnTypes[targetPrimaryKey.type] - return property.nullable ? type : `${type} NOT NULL` - } else if (property.kind === "relation") { - throw new Error("internal error - relation properties don't map to columns") +function getColumn(name: string, type: PrimitiveType, nullable: boolean) { + if (nullable) { + return `"${name}" ${columnTypes[type]}` } else { - signalInvalidType(property) + return `"${name}" ${columnTypes[type]} NOT NULL` } } -const getPropertyColumn = (config: Config, model: Model, property: Property) => - `'${property.name}' ${getPropertyColumnType(config, model, property)}` +const quote = (name: string) => `"${name}"` export class ModelAPI { readonly #table: string - readonly #params: Record readonly #properties: Record // Methods - #insert: Method - #update: Method - #delete: Method> - #clear: Method<{}> + #insert: Method + #update: Method + #delete: Method + #clear: Method<[]> // Queries - #selectAll: Query<{}, RecordValue> - #select: Query, RecordValue> - #count: Query<{}, { count: number }> + #selectAll: Query<[]> + #select: Query + #count: Query<[], { count: number }> readonly #relations: Record = {} - readonly #primaryKeyName: string - readonly #primaryKeyParam: `p${string}` - columnNames: string[] + readonly primaryProperties: PrimitiveProperty[] + readonly mutableProperties: Property[] + + readonly codecs: Record< + string, + { + columns: string[] + encode: (value: PropertyValue) => SqlitePrimitiveValue[] + decode: (record: Record) => PropertyValue + } + > = {} public constructor(readonly db: OpfsDatabase, readonly config: Config, readonly model: Model) { this.#table = model.name - this.#params = {} this.#properties = Object.fromEntries(model.properties.map((property) => [property.name, property])) + /** SQL column declarations */ const columns: string[] = [] - this.columnNames = [] // quoted column names for non-relation properties - const columnParams: `:p${string}`[] = [] // query params for non-relation properties - let primaryKeyIndex: number | null = null - let primaryKey: PrimitiveProperty | null = null - for (const [i, property] of model.properties.entries()) { + + /** unquoted column names for non-relation properties */ + const columnNames: string[] = [] + + this.primaryProperties = config.primaryKeys[model.name] + this.mutableProperties = [] + + for (const property of model.properties) { if (property.kind === "primitive") { - columns.push(getPropertyColumn(config, model, property)) - this.columnNames.push(`"${property.name}"`) - columnParams.push(`:p${i}`) - this.#params[property.name] = `p${i}` - - if (model.primaryKey.includes(property.name)) { - primaryKeyIndex = i - primaryKey = property + const { name, type, nullable } = property + columns.push(getColumn(name, type, nullable)) + columnNames.push(name) + + const propertyName = `${model.name}/${name}` + this.codecs[property.name] = { + columns: [property.name], + encode: (value) => [encodePrimitiveValue(propertyName, type, nullable, value)], + decode: (record) => decodePrimitiveValue(propertyName, type, nullable, record[property.name]), + } + + if (!model.primaryKey.includes(property.name)) { + this.mutableProperties.push(property) } } else if (property.kind === "reference") { - columns.push(getPropertyColumn(config, model, property)) - this.columnNames.push(`"${property.name}"`) - columnParams.push(`:p${i}`) - this.#params[property.name] = `p${i}` + const propertyName = `${model.name}/${property.name}` + + const target = config.models.find((model) => model.name === property.target) + assert(target !== undefined) + + config.primaryKeys[target.name] + + if (target.primaryKey.length === 1) { + const [targetProperty] = config.primaryKeys[target.name] + columns.push(getColumn(property.name, targetProperty.type, false)) + columnNames.push(property.name) + + this.codecs[property.name] = { + columns: [property.name], + encode: (value) => encodeReferenceValue(propertyName, [targetProperty], property.nullable, value), + decode: (record) => + decodeReferenceValue(propertyName, property.nullable, [targetProperty], [record[property.name]]), + } + } else { + const refNames: string[] = [] + + for (const targetProperty of config.primaryKeys[target.name]) { + const refName = `${property.name}/${targetProperty.name}` + columns.push(getColumn(refName, targetProperty.type, false)) + columnNames.push(refName) + refNames.push(refName) + } + + this.codecs[property.name] = { + columns: refNames, + + encode: (value) => + encodeReferenceValue(propertyName, config.primaryKeys[target.name], property.nullable, value), + + decode: (record) => + decodeReferenceValue( + propertyName, + property.nullable, + config.primaryKeys[target.name], + refNames.map((name) => record[name]), + ), + } + } + + this.mutableProperties.push(property) } else if (property.kind === "relation") { const relation = config.relations.find( (relation) => relation.source === model.name && relation.sourceProperty === property.name, ) assert(relation !== undefined, "internal error - relation not found") - this.#relations[property.name] = new RelationAPI(db, relation) + this.#relations[property.name] = new RelationAPI(db, config, relation) + + this.mutableProperties.push(property) } else { signalInvalidType(property) } } - assert(primaryKey !== null, "expected primaryKey !== null") - assert(primaryKeyIndex !== null, "expected primaryKeyIndex !== null") - // this.#primaryKeyName = columnNames[primaryKeyIndex] - this.#primaryKeyName = primaryKey.name - this.#primaryKeyParam = `p${primaryKeyIndex}` - // Create record table - db.exec(`CREATE TABLE IF NOT EXISTS "${this.#table}" (${columns.join(", ")})`) + const primaryKeyConstraint = `PRIMARY KEY (${model.primaryKey.map(quote).join(", ")})` + const tableSchema = [...columns, primaryKeyConstraint].join(", ") + db.exec(`CREATE TABLE IF NOT EXISTS "${this.#table}" (${tableSchema})`) // Create indexes for (const index of model.indexes) { - const indexName = `${model.name}/${index.join("/")}` - const indexColumns = index.map((name) => `'${name}'`) - db.exec(`CREATE INDEX IF NOT EXISTS "${indexName}" ON "${this.#table}" (${indexColumns.join(", ")})`) + const indexName = [model.name, ...index].join("/") + const indexColumns = index.map(quote).join(", ") + db.exec(`CREATE INDEX IF NOT EXISTS "${indexName}" ON "${this.#table}" (${indexColumns})`) } // Prepare methods - const insertNames = this.columnNames.join(", ") - const insertParams = columnParams.join(", ") - this.#insert = new Method( + + const quotedColumnNames = columnNames.map(quote).join(", ") + + const insertParams = Array.from({ length: columnNames.length }).fill("?").join(", ") + this.#insert = new Method( db, - `INSERT OR IGNORE INTO "${this.#table}" (${insertNames}) VALUES (${insertParams})`, + `INSERT OR IGNORE INTO "${this.#table}" (${quotedColumnNames}) VALUES (${insertParams})`, ) - const where = `WHERE "${this.#primaryKeyName}" = :${this.#primaryKeyParam}` - const updateEntries = Array.from(zip(this.columnNames, columnParams)).map(([name, param]) => `${name} = ${param}`) + const primaryKeyEquals = model.primaryKey.map((name) => `"${name}" = ?`) + const wherePrimaryKeyEquals = `WHERE ${primaryKeyEquals.join(" AND ")}` - this.#update = new Method(db, `UPDATE "${this.#table}" SET ${updateEntries.join(", ")} ${where}`) + const updateNames = columnNames.filter((name) => !model.primaryKey.includes(name)) + const updateEntries = updateNames.map((name) => `"${name}" = ?`) - this.#delete = new Method>(db, `DELETE FROM "${this.#table}" ${where}`) - - this.#clear = new Method>(db, `DELETE FROM "${this.#table}"`) + this.#update = new Method(db, `UPDATE "${this.#table}" SET ${updateEntries.join(", ")} ${wherePrimaryKeyEquals}`) + this.#delete = new Method(db, `DELETE FROM "${this.#table}" ${wherePrimaryKeyEquals}`) + this.#clear = new Method(db, `DELETE FROM "${this.#table}"`) // Prepare queries - this.#count = new Query<{}, { count: number }>(this.db, `SELECT COUNT(*) AS count FROM "${this.#table}"`) + this.#count = new Query<[], { count: number }>(this.db, `SELECT COUNT(*) AS count FROM "${this.#table}"`) + this.#select = new Query(this.db, `SELECT ${quotedColumnNames} FROM "${this.#table}" ${wherePrimaryKeyEquals}`) + this.#selectAll = new Query(this.db, `SELECT ${quotedColumnNames} FROM "${this.#table}"`) + } + public get(key: PrimaryKeyValue | PrimaryKeyValue[]): ModelValue | null { + const wrappedKey = Array.isArray(key) ? key : [key] + if (wrappedKey.length !== this.primaryProperties.length) { + throw new TypeError(`${this.model.name}: expected primary key with ${this.primaryProperties.length} components`) + } - this.#select = new Query, RecordValue>( - this.db, - `SELECT ${this.columnNames.join(", ")} FROM "${this.#table}" ${where}`, + const encodedKey = this.primaryProperties.map(({ name, type, nullable }, i) => + encodePrimitiveValue(name, type, nullable, wrappedKey[i]), ) - this.#selectAll = new Query<{}, RecordValue>(this.db, `SELECT ${this.columnNames.join(", ")} FROM "${this.#table}"`) - } - - public get(key: PrimaryKeyValue): ModelValue | null { - const record = this.#select.get({ [this.#primaryKeyParam]: key }) + const record = this.#select.get(encodedKey) if (record === null) { return null } - return { - ...decodeRecord(this.model, record), - ...mapValues(this.#relations, (api) => api.get(key)), + const result: ModelValue = Object.fromEntries( + this.primaryProperties.map((property, i) => [property.name, wrappedKey[i]]), + ) + + for (const property of this.mutableProperties) { + const propertyName = `${this.model.name}/${property.name}` + if (property.kind === "primitive") { + const { name, type, nullable } = property + result[name] = decodePrimitiveValue(propertyName, type, nullable, record[name]) + } else if (property.kind === "reference") { + const { name, nullable } = property + const values = this.codecs[name].columns.map((name) => record[name]) + result[name] = decodeReferenceValue(propertyName, nullable, this.config.primaryKeys[name], values) + } else if (property.kind === "relation") { + const { name, target } = property + const targets = this.#relations[name].get(encodedKey) + result[name] = targets.map((key) => + decodeReferenceValue(propertyName, false, this.config.primaryKeys[target], key), + ) as PrimaryKeyValue[] | PrimaryKeyValue[][] + } else { + signalInvalidType(property) + } } + + return result } - public getMany(keys: PrimaryKeyValue[]): (ModelValue | null)[] { + public getMany(keys: PrimaryKeyValue[] | PrimaryKeyValue[][]): (ModelValue | null)[] { return keys.map((key) => this.get(key)) } public set(value: ModelValue) { validateModelValue(this.model, value) - const key = value[this.#primaryKeyName] as PrimaryKeyValue - const encodedParams = encodeRecordParams(this.model, value, this.#params) - const existingRecord = this.#select.get({ [this.#primaryKeyParam]: key }) + const encodedKey = this.primaryProperties.map(({ name, type, nullable }) => + encodePrimitiveValue(name, type, nullable, value[name]), + ) + + const existingRecord = this.#select.get(encodedKey) if (existingRecord === null) { - this.#insert.run(encodedParams) + const params = this.encodeProperties(this.model.properties, value) + this.#insert.run(params) } else { - this.#update.run(encodedParams) + const params = this.encodeProperties(this.mutableProperties, value) + this.#update.run([...params, ...encodedKey]) } for (const [name, relation] of Object.entries(this.#relations)) { if (existingRecord !== null) { - relation.delete(key) + relation.delete(encodedKey) } - assert(Array.isArray(value[name]) && value[name].every(isPrimaryKey)) - relation.add(key, value[name]) + assert(Array.isArray(value[name])) + const target = this.config.primaryKeys[relation.relation.target] + const encodedTargets = value[name].map((key) => encodeReferenceValue(name, target, false, key)) + + relation.add(encodedKey, encodedTargets) } } - public delete(key: PrimaryKeyValue) { - const existingRecord = this.#select.get({ [this.#primaryKeyParam]: key }) - if (existingRecord === null) { - return + private encodeProperties(properties: Property[], value: ModelValue): SqlitePrimitiveValue[] { + const result: SqlitePrimitiveValue[] = [] + for (const property of properties) { + if (property.kind === "primitive") { + const { name, type, nullable } = property + result.push(encodePrimitiveValue(name, type, nullable, value[name])) + } else if (property.kind === "reference") { + const { name, target, nullable } = property + const targetProperties = this.config.primaryKeys[target] + result.push(...encodeReferenceValue(name, targetProperties, nullable, value[property.name])) + } else if (property.kind === "relation") { + continue + } else { + signalInvalidType(property) + } + } + + return result + } + + public delete(key: PrimaryKeyValue | PrimaryKeyValue[]) { + const wrappedKey = Array.isArray(key) ? key : [key] + if (wrappedKey.length !== this.primaryProperties.length) { + throw new TypeError(`${this.model.name}: expected primary key with ${this.primaryProperties.length} components`) } - this.#delete.run({ [this.#primaryKeyParam]: key }) + const encodedKey = this.primaryProperties.map(({ name, type, nullable }, i) => + encodePrimitiveValue(name, type, nullable, wrappedKey[i]), + ) + + this.#delete.run(encodedKey) for (const relation of Object.values(this.#relations)) { - relation.delete(key) + relation.delete(encodedKey) } } public clear() { - const existingRecords = this.#select.all({}) - - this.#clear.run({}) - - for (const record of existingRecords) { - const key = record[this.#primaryKeyParam] - for (const relation of Object.values(this.#relations)) { - if (!key || typeof key !== "string") continue - relation.delete(key) - } + this.#clear.run([]) + for (const relation of Object.values(this.#relations)) { + relation.clear() } } public count(where?: WhereCondition): number { const sql: string[] = [] + const params: SqlitePrimitiveValue[] = [] // SELECT sql.push(`SELECT COUNT(*) AS count FROM "${this.#table}"`) // WHERE - const [whereExpression, params] = this.getWhereExpression(where) + const [whereExpression, whereParams] = this.getWhereExpression(where) if (whereExpression) { sql.push(`WHERE ${whereExpression}`) + params.push(...whereParams) } - const paramsWithColons: any = {} - for (const key of Object.keys(params)) { - paramsWithColons[`:${key}`] = params[key] - } - const results = this.db.selectObjects(sql.join(" "), paramsWithColons) - - const countResult = results[0].count - if (typeof countResult === "number") { - return countResult - } else { - throw new Error("internal error") - } + const { count } = new Query(this.db, sql.join(" ")).get(params) ?? {} + assert(typeof count === "number") + return count } - public async *iterate(query: QueryParams = {}): AsyncIterable { - // TODO: implement iterate (needs special handling over comlink) - if (Object.keys(query).length > 0) { - throw new Error("not implemented") - } - - for (const record of this.#selectAll.iterate({})) { - const key = record[this.#primaryKeyName] as PrimaryKeyValue - const value = { - ...decodeRecord(this.model, record), - ...mapValues(this.#relations, (api) => api.get(key)), - } + public query(query: QueryParams): ModelValue[] { + const [sql, properties, relations, params] = this.parseQuery(query) + const results: ModelValue[] = [] - yield value + for (const row of new Query(this.db, sql).iterate(params)) { + results.push(this.parseRecord(row, properties, relations)) } + + return results } - public query(query: QueryParams): ModelValue[] { - const [sql, relations, params] = this.parseQuery(query) + public *iterate(query: QueryParams): IterableIterator { + const [sql, properties, relations, params] = this.parseQuery(query) - const paramsWithColons: any = {} - for (const key of Object.keys(params)) { - paramsWithColons[`:${key}`] = params[key] + for (const row of new Query(this.db, sql).iterate(params)) { + yield this.parseRecord(row, properties, relations) } + } - const results = this.db.selectObjects(sql, paramsWithColons) - return results.map((record): ModelValue => { - const key = record[this.#primaryKeyName] as PrimaryKeyValue - - const value: ModelValue = {} - for (const [propertyName, propertyValue] of Object.entries(record)) { - const property = this.#properties[propertyName] - if (property.kind === "primitive") { - value[propertyName] = decodePrimitiveValue(this.model.name, property, propertyValue) - } else if (property.kind === "reference") { - value[propertyName] = decodeReferenceValue(this.model.name, property, propertyValue) - } else if (property.kind === "relation") { - throw new Error("internal error") - } else { - signalInvalidType(property) - } - } + private parseRecord( + row: Record, + properties: string[], + relations: Relation[], + ): ModelValue { + const record: ModelValue = {} + for (const name of properties) { + record[name] = this.codecs[name].decode(row) + } - for (const relation of relations) { - value[relation.sourceProperty] = this.#relations[relation.sourceProperty].get(key) - } + for (const relation of relations) { + const encodedKey = this.config.primaryKeys[this.model.name].map(({ name }) => { + assert(row[name] !== undefined, "cannot select relation properties without selecting the primary key") + return row[name] + }) + + const targetKeys = this.#relations[relation.sourceProperty].get(encodedKey) + const targetPrimaryKey = this.config.primaryKeys[relation.target] + record[relation.sourceProperty] = targetKeys.map((targetKey) => + decodeReferenceValue(relation.sourceProperty, false, targetPrimaryKey, targetKey), + ) as PrimaryKeyValue[] | PrimaryKeyValue[][] + } - return value - }) + return record } - private parseQuery(query: QueryParams): [sql: string, relations: Relation[], params: Record] { + private parseQuery( + query: QueryParams, + ): [sql: string, properties: string[], relations: Relation[], params: SqlitePrimitiveValue[]] { // See https://www.sqlite.org/lang_select.html for railroad diagram const sql: string[] = [] + const params: SqlitePrimitiveValue[] = [] // SELECT - const [select, relations] = this.getSelectExpression(query.select) - sql.push(`SELECT ${select} FROM "${this.#table}"`) + const select = query.select ?? mapValues(this.#properties, () => true) + const [selectExpression, selectProperties, selectRelations] = this.getSelectExpression(select) + sql.push(`SELECT ${selectExpression} FROM "${this.#table}"`) // WHERE - const [where, params] = this.getWhereExpression(query.where) + const [where, whereParams] = this.getWhereExpression(query.where) if (where !== null) { sql.push(`WHERE ${where}`) + params.push(...whereParams) } // ORDER BY @@ -341,14 +414,18 @@ export class ModelAPI { const [[indexName, direction]] = orders const index = indexName.split("/") - assert(!index.some((name) => this.#properties[name]?.kind === "relation"), "cannot order by relation properties") + for (const name of index) { + if (this.#properties[name].kind === "relation") { + throw new Error("cannot order by relation properties") + } + } if (direction === "asc") { - const orders = index.map((name) => `"${name}" ASC`).join(", ") - sql.push(`ORDER BY ${orders}`) + const columns = index.flatMap((name) => this.codecs[name].columns) + sql.push(`ORDER BY ${columns.map((name) => `"${name}" ASC`).join(", ")}`) } else if (direction === "desc") { - const orders = index.map((name) => `"${name}" DESC`).join(", ") - sql.push(`ORDER BY ${orders}`) + const columns = index.flatMap((name) => this.codecs[name].columns) + sql.push(`ORDER BY ${columns.map((name) => `"${name}" DESC`).join(", ")}`) } else { throw new Error("invalid orderBy direction") } @@ -356,14 +433,14 @@ export class ModelAPI { // LIMIT if (typeof query.limit === "number") { - sql.push(`LIMIT :limit`) - params.limit = query.limit + sql.push(`LIMIT ?`) + params.push(query.limit) } // OFFSET if (typeof query.offset === "number") { - sql.push(`LIMIT :offset`) - params.limit = query.offset + sql.push(`OFFSET ?`) + params.push(query.offset) } // JOIN (not supported) @@ -371,12 +448,13 @@ export class ModelAPI { throw new Error("cannot use 'include' in queries outside the browser/idb") } - return [sql.join(" "), relations, params] + return [sql.join(" "), selectProperties, selectRelations, params] } private getSelectExpression( - select: Record = mapValues(this.#properties, () => true), - ): [select: string, relations: Relation[]] { + select: Record, + ): [selectExpression: string, properties: string[], relations: Relation[]] { + const properties: string[] = [] const relations: Relation[] = [] const columns = [] @@ -388,7 +466,8 @@ export class ModelAPI { const property = this.#properties[name] assert(property !== undefined, "property not found") if (property.kind === "primitive" || property.kind === "reference") { - columns.push(`"${name}"`) + properties.push(property.name) + columns.push(...this.codecs[name].columns.map(quote)) } else if (property.kind === "relation") { relations.push(this.#relations[name].relation) } else { @@ -397,158 +476,125 @@ export class ModelAPI { } assert(columns.length > 0, "cannot query an empty select expression") - assert(columns.includes(`"${this.#primaryKeyName}"`), "select expression must include the primary key") - return [columns.join(", "), relations] + return [columns.join(", "), properties, relations] } - private getWhereExpression( - where: WhereCondition = {}, - ): [where: string | null, params: Record] { - const params: Record = {} - const filters = Object.entries(where).flatMap(([name, expression], i) => { + private getWhereExpression(where: WhereCondition = {}): [where: string | null, params: SqlitePrimitiveValue[]] { + const params: SqlitePrimitiveValue[] = [] + + const filters: string[] = [] + for (const [name, expression] of Object.entries(where)) { const property = this.#properties[name] assert(property !== undefined, "property not found") if (expression === undefined) { - return [] + continue } if (property.kind === "primitive") { - assert(property.type !== "json", "json properties are not supported in where clauses") + const { type, nullable } = property + assert(type !== "json", "json properties are not supported in where clauses") if (isLiteralExpression(expression)) { if (expression === null) { - return [`"${name}" ISNULL`] - } else if (Array.isArray(expression)) { - throw new Error("invalid primitive value (expected null | number | string | Uint8Array)") - } else { - assert(isPrimitiveValue(expression)) - const p = `p${i}` - params[p] = expression - return [`"${name}" = :${p}`] + filters.push(`"${name}" ISNULL`) + continue } + + const encodedValue = encodePrimitiveValue(name, type, false, expression) + params.push(encodedValue) + filters.push(`"${name}" = ?`) } else if (isNotExpression(expression)) { const { neq: value } = expression if (value === undefined) { - return [] + continue } else if (value === null) { - return [`"${name}" NOTNULL`] - } else if (Array.isArray(value)) { - throw new Error("invalid primitive value (expected null | number | string | Uint8Array)") + filters.push(`"${name}" NOTNULL`) + } + + const encodedValue = encodePrimitiveValue(name, type, false, value) + params.push(encodedValue) + if (nullable) { + filters.push(`("${name}" ISNULL OR "${name}" != ?)`) } else { - assert(isPrimitiveValue(value)) - const p = `p${i}` - params[p] = value - if (property.nullable) { - return [`("${name}" ISNULL OR "${name}" != :${p})`] - } else { - return [`"${name}" != :${p}`] - } + filters.push(`"${name}" != ?`) } } else if (isRangeExpression(expression)) { - const keys = Object.keys(expression) as (keyof RangeExpression)[] - - return keys - .filter((key) => expression[key] !== undefined) - .flatMap((key, j) => { - const value = expression[key] as PrimitiveValue - if (value === null) { - switch (key) { - case "gt": - return [`"${name}" NOTNULL`] - case "gte": - return [] - case "lt": - return ["0 = 1"] - case "lte": - return [] - } - } + for (const [key, value] of Object.entries(expression)) { + if (value === undefined) { + continue + } - const p = `p${i}q${j}` - params[p] = value - switch (key) { - case "gt": - return [`("${name}" NOTNULL) AND ("${name}" > :${p})`] - case "gte": - return [`("${name}" NOTNULL) AND ("${name}" >= :${p})`] - case "lt": - return [`("${name}" ISNULL) OR ("${name}" < :${p})`] - case "lte": - return [`("${name}" ISNULL) OR ("${name}" <= :${p})`] + if (value === null) { + if (key === "gt") { + filters.push(`"${name}" NOTNULL`) + } else if (key === "gte") { + continue + } else if (key === "lt") { + filters.push("0 = 1") + } else if (key === "lte") { + filters.push(`"${name}" ISNULL`) + } else { + throw new Error(`invalid range expression "${key}"`) } - }) + } else { + params.push(encodePrimitiveValue(name, type, nullable, value)) + if (key === "gt") { + filters.push(`("${name}" NOTNULL) AND ("${name}" > ?)`) + } else if (key === "gte") { + filters.push(`("${name}" NOTNULL) AND ("${name}" >= ?)`) + } else if (key === "lt") { + filters.push(`("${name}" ISNULL) OR ("${name}" < ?)`) + } else if (key === "lte") { + filters.push(`("${name}" ISNULL) OR ("${name}" <= ?)`) + } + } + } } else { signalInvalidType(expression) } } else if (property.kind === "reference") { + const target = this.config.primaryKeys[property.target] + const { columns } = this.codecs[name] + if (isLiteralExpression(expression)) { - const reference = expression - if (reference === null) { - return [`"${name}" ISNULL`] - } else if (typeof reference === "string") { - const p = `p${i}` - params[p] = reference - return [`"${name}" = :${p}`] + const encodedKey = encodeReferenceValue(name, target, true, expression) + if (encodedKey.every((key) => key === null)) { + filters.push(columns.map((c) => `"${c}" ISNULL`).join(" AND ")) } else { - throw new Error("invalid reference value (expected string | null)") + filters.push(columns.map((c) => `"${c}" = ?`).join(" AND ")) + params.push(...encodedKey) } } else if (isNotExpression(expression)) { - const reference = expression.neq - if (reference === null) { - return [`"${name}" NOTNULL`] - } else if (typeof reference === "string") { - const p = `p${i}` - params[p] = reference - return [`"${name}" != :${p}`] + if (expression.neq === undefined) { + continue + } + + const encodedKey = encodeReferenceValue(name, target, true, expression.neq) + + if (encodedKey.every((key) => key === null)) { + filters.push(columns.map((c) => `"${c}" NOTNULL`).join(" AND ")) } else { - throw new Error("invalid reference value (expected string | null)") + const isNull = columns.map((c) => `"${c}" ISNULL`).join(" AND ") + const isNotEq = columns.map((c) => `"${c}" != ?`).join(" AND ") + filters.push(`(${isNull}) OR (${isNotEq})`) + params.push(...encodedKey) } } else if (isRangeExpression(expression)) { + // TODO: support range queries on references throw new Error("cannot use range expressions on reference values") } else { signalInvalidType(expression) } } else if (property.kind === "relation") { - const relationTable = this.#relations[property.name].table - if (isLiteralExpression(expression)) { - const references = expression - assert(Array.isArray(references), "invalid relation value (expected string[])") - const targets: string[] = [] - for (const [j, reference] of references.entries()) { - assert(typeof reference === "string", "invalid relation value (expected string[])") - const p = `p${i}q${j}` - params[p] = reference - targets.push( - `"${this.#primaryKeyName}" IN (SELECT _source FROM "${relationTable}" WHERE (_target = :${p}))`, - ) - } - return targets.length > 0 ? [targets.join(" AND ")] : [] - } else if (isNotExpression(expression)) { - const references = expression.neq - assert(Array.isArray(references), "invalid relation value (expected string[])") - const targets: string[] = [] - for (const [j, reference] of references.entries()) { - assert(typeof reference === "string", "invalid relation value (expected string[])") - const p = `p${i}q${j}` - params[p] = reference - targets.push( - `"${this.#primaryKeyName}" NOT IN (SELECT _source FROM "${relationTable}" WHERE (_target = :${p}))`, - ) - } - return targets.length > 0 ? [targets.join(" AND ")] : [] - } else if (isRangeExpression(expression)) { - throw new Error("cannot use range expressions on relation values") - } else { - signalInvalidType(expression) - } + throw new Error("cannot query relation values") } else { signalInvalidType(property) } - }) + } if (filters.length === 0) { - return [null, {}] + return [null, []] } else { return [`${filters.map((filter) => `(${filter})`).join(" AND ")}`, params] } @@ -560,73 +606,95 @@ export class RelationAPI { public readonly sourceIndex: string public readonly targetIndex: string - readonly #select: Query<{ _source: PrimaryKeyValue }, { _target: PrimaryKeyValue }> - readonly #insert: Method<{ _source: PrimaryKeyValue; _target: PrimaryKeyValue }> - readonly #delete: Method<{ _source: PrimaryKeyValue }> + readonly sourceColumnNames: string[] + readonly targetColumnNames: string[] + + readonly #select: Query + readonly #insert: Method + readonly #delete: Method + readonly #clear: Method<[]> - public constructor(readonly db: OpfsDatabase, readonly relation: Relation) { + public constructor(readonly db: OpfsDatabase, readonly config: Config, readonly relation: Relation) { this.table = `${relation.source}/${relation.sourceProperty}` this.sourceIndex = `${relation.source}/${relation.sourceProperty}/source` this.targetIndex = `${relation.source}/${relation.sourceProperty}/target` + this.sourceColumnNames = [] + this.targetColumnNames = [] + + const sourcePrimaryKey = config.primaryKeys[relation.source] + const targetPrimaryKey = config.primaryKeys[relation.target] + { const columns: string[] = [] - if (relation.sourcePrimaryKey.length === 1) { - const [{ type }] = relation.sourcePrimaryKey - columns.push(`_source ${primitiveColumnTypes[type]} NOT NULL`) + if (sourcePrimaryKey.length === 1) { + const [{ type }] = sourcePrimaryKey + columns.push(`_source ${columnTypes[type]} NOT NULL`) + this.sourceColumnNames.push("_source") } else { - for (const [i, { type }] of relation.sourcePrimaryKey.entries()) { - columns.push(`"_source/${i}" ${primitiveColumnTypes[type]} NOT NULL`) + for (const [i, { type }] of sourcePrimaryKey.entries()) { + const columnName = `_source/${i}` + columns.push(`"${columnName}" ${columnTypes[type]} NOT NULL`) + this.sourceColumnNames.push(columnName) } } - if (relation.targetPrimaryKey.length === 1) { - const [{ type }] = relation.targetPrimaryKey - columns.push(`_target ${primitiveColumnTypes[type]} NOT NULL`) + if (targetPrimaryKey.length === 1) { + const [{ type }] = targetPrimaryKey + columns.push(`_target ${columnTypes[type]} NOT NULL`) + this.targetColumnNames.push("_target") } else { - for (const [i, { type }] of relation.targetPrimaryKey.entries()) { - columns.push(`"_target/${i}" ${primitiveColumnTypes[type]} NOT NULL`) + for (const [i, { type }] of targetPrimaryKey.entries()) { + const columnName = `_target/${i}` + columns.push(`"${columnName}" ${columnTypes[type]} NOT NULL`) + this.targetColumnNames.push(columnName) } } db.exec(`CREATE TABLE IF NOT EXISTS "${this.table}" (${columns.join(", ")})`) } - db.exec(`CREATE INDEX IF NOT EXISTS "${this.sourceIndex}" ON "${this.table}" (_source)`) + const sourceColumns = this.sourceColumnNames.map(quote).join(", ") + const targetColumns = this.targetColumnNames.map(quote).join(", ") + + db.exec(`CREATE INDEX IF NOT EXISTS "${this.sourceIndex}" ON "${this.table}" (${sourceColumns})`) if (relation.indexed) { - db.exec(`CREATE INDEX IF NOT EXISTS "${this.targetIndex}" ON "${this.table}" (_target)`) + db.exec(`CREATE INDEX IF NOT EXISTS "${this.targetIndex}" ON "${this.table}" (${targetColumns})`) } // Prepare methods - this.#insert = new Method<{ _source: string; _target: string }>( - this.db, - `INSERT INTO "${this.table}" (_source, _target) VALUES (:_source, :_target)`, - ) + const insertColumns = [...this.sourceColumnNames, ...this.targetColumnNames].map(quote).join(", ") + const insertParamCount = this.sourceColumnNames.length + this.targetColumnNames.length + const insertParams = Array.from({ length: insertParamCount }).fill("?").join(", ") + this.#insert = new Method(this.db, `INSERT INTO "${this.table}" (${insertColumns}) VALUES (${insertParams})`) - this.#delete = new Method<{ _source: string }>(this.db, `DELETE FROM "${this.table}" WHERE _source = :_source`) + const selectBySource = this.sourceColumnNames.map((name) => `"${name}" = ?`).join(" AND ") + + this.#delete = new Method(this.db, `DELETE FROM "${this.table}" WHERE ${selectBySource}`) + this.#clear = new Method(this.db, `DELETE FROM "${this.table}"`) // Prepare queries - this.#select = new Query<{ _source: string }, { _target: string }>( - this.db, - `SELECT _target FROM "${this.table}" WHERE _source = :_source`, - ) + this.#select = new Query(this.db, `SELECT ${targetColumns} FROM "${this.table}" WHERE ${selectBySource}`) } - public get(source: PrimaryKeyValue): PrimaryKeyValue[] { - const targets = this.#select.all({ _source: source }) - return targets.map(({ _target: target }) => target) + public get(sourceKey: SqlitePrimitiveValue[]): SqlitePrimitiveValue[][] { + const targets = this.#select.all(sourceKey) + return targets.map((record) => this.sourceColumnNames.map((name) => record[name])) } - public add(source: PrimaryKeyValue, targets: PrimaryKeyValue[]) { - assert(Array.isArray(targets), "expected PrimaryKey[]") - for (const target of targets) { - this.#insert.run({ _source: source, _target: target }) + public add(sourceKey: SqlitePrimitiveValue[], targetKeys: SqlitePrimitiveValue[][]) { + for (const targetKey of targetKeys) { + this.#insert.run([...sourceKey, ...targetKey]) } } - public delete(source: PrimaryKeyValue) { - this.#delete.run({ _source: source }) + public delete(sourceKey: SqlitePrimitiveValue[]) { + this.#delete.run(sourceKey) + } + + public clear() { + this.#clear.run([]) } } diff --git a/packages/modeldb-sqlite-wasm/src/ModelDB.ts b/packages/modeldb-sqlite-wasm/src/ModelDB.ts index 776c108c6..1ad4e17da 100644 --- a/packages/modeldb-sqlite-wasm/src/ModelDB.ts +++ b/packages/modeldb-sqlite-wasm/src/ModelDB.ts @@ -2,14 +2,13 @@ import { logger } from "@libp2p/logger" import * as Comlink from "comlink" import sqlite3InitModule from "@sqlite.org/sqlite-wasm" import { + Config, AbstractModelDB, ModelDBBackend, - parseConfig, Effect, ModelValue, ModelSchema, QueryParams, - Config, WhereCondition, } from "@canvas-js/modeldb" import { InnerModelDB } from "./InnerModelDB.js" @@ -25,7 +24,7 @@ export class ModelDB extends AbstractModelDB { private readonly wrappedDB: Comlink.Remote | InnerModelDB public static async initialize({ path, models }: ModelDBOptions) { - const config = parseConfig(models) + const config = Config.parse(models) let worker, wrappedDB if (path) { worker = new Worker(new URL("./worker.js", import.meta.url), { type: "module" }) diff --git a/packages/modeldb-sqlite-wasm/src/encoding.ts b/packages/modeldb-sqlite-wasm/src/encoding.ts index e44113d56..de030c772 100644 --- a/packages/modeldb-sqlite-wasm/src/encoding.ts +++ b/packages/modeldb-sqlite-wasm/src/encoding.ts @@ -1,208 +1,175 @@ import * as json from "@ipld/dag-json" -import { - isPrimaryKey, - type Model, - type ModelValue, - type PrimaryKeyValue, - type PrimitiveProperty, - type PrimitiveValue, - type PropertyValue, - type ReferenceProperty, +import type { + PrimitiveType, + PrimaryKeyValue, + PrimitiveProperty, + PrimitiveValue, + PropertyValue, } from "@canvas-js/modeldb" -import { assert, mapValues, signalInvalidType } from "@canvas-js/utils" -import { SqlValue } from "@sqlite.org/sqlite-wasm" +import { assert, signalInvalidType } from "@canvas-js/utils" -export type RecordValue = Record -export type RecordParams = Record<`p${string}`, SqlValue> +export type SqlitePrimitiveValue = null | number | string | Uint8Array -export function encodeRecordParams( - model: Model, - value: ModelValue, - params: Record, -): RecordParams { - const values: RecordParams = {} - - for (const property of model.properties) { - const propertyValue = value[property.name] - if (propertyValue === undefined) { - throw new Error(`missing value for property ${model.name}/${property.name}`) - } - - const param = params[property.name] - if (property.kind === "primitive") { - values[param] = encodePrimitiveValue(model.name, property, value[property.name]) - } else if (property.kind === "reference") { - values[param] = encodeReferenceValue(model.name, property, value[property.name]) - } else if (property.kind === "relation") { - assert(Array.isArray(value[property.name])) - continue - } else { - signalInvalidType(property) - } - } - - return values -} - -function encodePrimitiveValue(modelName: string, property: PrimitiveProperty, value: PropertyValue): SqlValue { +export function encodePrimitiveValue( + propertyName: string, + type: PrimitiveType, + nullable: boolean, + value: PropertyValue, +): SqlitePrimitiveValue { if (value === null) { - if (property.nullable) { + if (nullable) { return null - } else if (property.type === "json") { + } else if (type === "json") { return "null" } else { - throw new TypeError(`${modelName}/${property.name} cannot be null`) + throw new TypeError(`${propertyName} cannot be null`) } - } else if (property.type === "integer") { + } else if (type === "integer") { if (typeof value === "number" && Number.isSafeInteger(value)) { return value } else { - throw new TypeError(`${modelName}/${property.name} must be a safely representable integer`) + throw new TypeError(`${propertyName} must be a safely representable integer`) } - } else if (property.type === "number" || property.type === "float") { + } else if (type === "number" || type === "float") { if (typeof value === "number") { return value } else { - throw new TypeError(`${modelName}/${property.name} must be a number`) + throw new TypeError(`${propertyName} must be a number`) } - } else if (property.type === "string") { + } else if (type === "string") { if (typeof value === "string") { return value } else { - throw new TypeError(`${modelName}/${property.name} must be a string`) + throw new TypeError(`${propertyName} must be a string`) } - } else if (property.type === "bytes") { + } else if (type === "bytes") { if (value instanceof Uint8Array) { return value } else { - throw new TypeError(`${modelName}/${property.name} must be a Uint8Array`) + throw new TypeError(`${propertyName} must be a Uint8Array`) } - } else if (property.type === "boolean") { + } else if (type === "boolean") { if (typeof value === "boolean") { return value ? 1 : 0 } else { - throw new TypeError(`${modelName}/${property.name} must be a boolean`) + throw new TypeError(`${propertyName} must be a boolean`) } - } else if (property.type === "json") { + } else if (type === "json") { try { return json.stringify(value) } catch (e) { - throw new TypeError(`${modelName}/${property.name} must be IPLD-encodable`) + throw new TypeError(`${propertyName} must be IPLD-encodable`) } } else { - const _: never = property.type - throw new Error(`internal error - unknown primitive type ${JSON.stringify(property.type)}`) + signalInvalidType(type) } } -function encodeReferenceValue(modelName: string, property: ReferenceProperty, value: PropertyValue): SqlValue { +export function encodeReferenceValue( + propertyName: string, + target: PrimitiveProperty[], + nullable: boolean, + value: PropertyValue, +): SqlitePrimitiveValue[] { if (value === null) { - if (property.nullable) { - return null + if (nullable) { + return Array.from({ length: target.length }).fill(null) } else { - throw new TypeError(`${modelName}/${property.name} cannot be null`) + throw new TypeError(`${propertyName} cannot be null`) } - } else if (isPrimaryKey(value)) { - return value - } else { - throw new TypeError(`${modelName}/${property.name} must be a primary key`) } -} - -export function decodeRecord(model: Model, record: Record): ModelValue { - const value: ModelValue = {} - for (const property of model.properties) { - if (property.kind === "primitive") { - value[property.name] = decodePrimitiveValue(model.name, property, record[property.name]) - } else if (property.kind === "reference") { - value[property.name] = decodeReferenceValue(model.name, property, record[property.name]) - } else if (property.kind === "relation") { - continue - } else { - signalInvalidType(property) - } + const wrappedValue = Array.isArray(value) ? value : [value] + if (wrappedValue.length !== target.length) { + throw new TypeError(`${propertyName} - expected primary key with ${target.length} components`) } - return value + return target.map(({ name, type }) => encodePrimitiveValue(name, type, false, value)) } -export function decodePrimitiveValue(modelName: string, property: PrimitiveProperty, value: SqlValue): PrimitiveValue { +export function decodePrimitiveValue( + propertyName: string, + type: PrimitiveType, + nullable: boolean, + value: SqlitePrimitiveValue, +): PrimitiveValue { if (value === null) { - if (property.nullable) { + if (nullable) { return null } else { - throw new Error(`internal error - missing ${modelName}/${property.name} value`) + throw new Error(`internal error - missing ${propertyName} value`) } } - if (property.type === "integer") { + if (type === "integer") { if (typeof value === "number" && Number.isSafeInteger(value)) { return value } else { console.error("expected integer, got", value) - throw new Error(`internal error - invalid ${modelName}/${property.name} value (expected integer)`) + throw new Error(`internal error - invalid ${propertyName} value (expected integer)`) } - } else if (property.type === "number" || property.type === "float") { + } else if (type === "number" || type === "float") { if (typeof value === "number") { return value } else { console.error("expected float, got", value) - throw new Error(`internal error - invalid ${modelName}/${property.name} value (expected float)`) + throw new Error(`internal error - invalid ${propertyName} value (expected float)`) } - } else if (property.type === "string") { + } else if (type === "string") { if (typeof value === "string") { return value } else { console.error("expected string, got", value) - throw new Error(`internal error - invalid ${modelName}/${property.name} value (expected string)`) + throw new Error(`internal error - invalid ${propertyName} value (expected string)`) } - } else if (property.type === "bytes") { - if (Buffer.isBuffer(value)) { - return new Uint8Array(value.buffer, value.byteOffset, value.byteLength) - } else if (value instanceof Uint8Array) { + } else if (type === "bytes") { + if (value instanceof Uint8Array) { return value } else { - console.error("expected Uint8Array, got", value) - throw new Error(`internal error - invalid ${modelName}/${property.name} value (expected Uint8Array)`) + throw new Error(`internal error - invalid ${propertyName} value (expected bytes)`) } - } else if (property.type === "boolean") { + } else if (type === "boolean") { if (typeof value === "number") { return value === 1 } else { - console.error("expected boolean, got", value) - throw new Error(`internal error - invalid ${modelName}/${property.name} value (expected boolean)`) + throw new Error(`internal error - invalid ${propertyName} value (expected 0 or 1)`) } - } else if (property.type === "json") { + } else if (type === "json") { assert(typeof value === "string", 'internal error - expected typeof value === "string"') try { return json.parse(value) } catch (e) { console.error("internal error - invalid dag-json", value) - throw new Error(`internal error - invalid ${modelName}/${property.name} value (expected dag-json)`) + throw new Error(`internal error - invalid ${propertyName} value (expected dag-json)`) } } else { - const _: never = property.type - throw new Error(`internal error - unknown primitive type ${JSON.stringify(property.type)}`) + signalInvalidType(type) } } export function decodeReferenceValue( - modelName: string, - property: ReferenceProperty, - value: SqlValue, -): PrimaryKeyValue | null { - if (value === null) { - if (property.nullable) { + propertyName: string, + nullable: boolean, + target: PrimitiveProperty[], + values: SqlitePrimitiveValue[], +): PrimaryKeyValue | PrimaryKeyValue[] | null { + if (values.every((value) => value === null)) { + if (nullable) { return null } else { - throw new TypeError(`internal error - missing ${modelName}/${property.name} value`) + throw new Error(`internal error - missing ${propertyName} value`) } - } else if (isPrimaryKey(value)) { - return value + } + + const result = target.map( + ({ name, type }, i) => decodePrimitiveValue(name, type, false, values[i]) as PrimaryKeyValue, + ) + + if (result.length === 1) { + return result[0] } else { - throw new Error(`internal error - invalid ${modelName}/${property.name} value (expected primary key)`) + return result } } diff --git a/packages/modeldb-sqlite-wasm/src/utils.ts b/packages/modeldb-sqlite-wasm/src/utils.ts index 66f5cfbc8..5b04370df 100644 --- a/packages/modeldb-sqlite-wasm/src/utils.ts +++ b/packages/modeldb-sqlite-wasm/src/utils.ts @@ -1,82 +1,121 @@ -import { OpfsDatabase, PreparedStatement, SqlValue } from "@sqlite.org/sqlite-wasm" +import { OpfsDatabase, PreparedStatement } from "@sqlite.org/sqlite-wasm" -export class Query

{ +import { SqlitePrimitiveValue } from "./encoding.js" + +export class Query< + P extends SqlitePrimitiveValue[] = SqlitePrimitiveValue[], + R extends Record = Record, +> { private readonly statement: PreparedStatement constructor(db: OpfsDatabase, private readonly sql: string) { this.statement = db.prepare(sql) } - public get(params: P): R | null { - const statement = this.statement - - const paramsWithColons = Object.fromEntries(Object.entries(params).map(([key, value]) => [":" + key, value])) + public finalize() { + this.statement.finalize() + } + public get(params: P): R | null { + const stmt = this.statement try { - if (Object.keys(paramsWithColons).length > 0) statement.bind(paramsWithColons) - if (!statement.step()) { + if (params.length > 0) { + stmt.bind(params) + } + + if (!stmt.step()) { return null } - return statement.get({}) as R + + return stmt.get({}) as R } finally { - statement.reset(true) + stmt.reset(true) } } public all(params: P): R[] { - const statement = this.statement - const paramsWithColons = Object.fromEntries(Object.entries(params).map(([key, value]) => [":" + key, value])) + const stmt = this.statement + try { - if (Object.keys(paramsWithColons).length > 0) statement.bind(paramsWithColons) - const result = [] - while (statement.step()) { - result.push(statement.get({}) as R) + if (params.length > 0) { + stmt.bind(params) + } + + const rows: R[] = [] + while (stmt.step()) { + rows.push(stmt.get({}) as R) } - return result + + return rows } finally { - statement.reset(true) + stmt.reset(true) } } - public *iterate(params: P): IterableIterator { - const statement = this.statement - const paramsWithColons = Object.fromEntries(Object.entries(params).map(([key, value]) => [":" + key, value])) - if (Object.keys(paramsWithColons).length > 0) statement.bind(paramsWithColons) + public iterate(params: P): IterableIterator { + const stmt = this.statement + if (params.length > 0) { + stmt.bind(params) + } + + let finished = false - const iter: IterableIterator = { + return { [Symbol.iterator]() { return this }, + next() { - const done = statement.step() + if (finished) { + return { done: true, value: undefined } + } + + const done = stmt.step() if (done) { + finished = true + stmt.reset(true) return { done: true, value: undefined } } else { - return { done: false, value: statement.get({}) as R } + return { done: false, value: stmt.get({}) as R } } }, - } - try { - yield* iter - } finally { - statement.reset(true) + return() { + finished = true + stmt.reset(true) + return { done: true, value: undefined } + }, + + throw(err: any) { + finished = true + stmt.reset(true) + throw err + }, } } } -export class Method

{ +export class Method

{ private readonly statement: PreparedStatement constructor(db: OpfsDatabase, private readonly sql: string) { this.statement = db.prepare(sql) } + public finalize() { + this.statement.finalize() + } + public run(params: P) { - const statement = this.statement - const paramsWithColons = Object.fromEntries(Object.entries(params).map(([key, value]) => [":" + key, value])) - if (Object.keys(paramsWithColons).length > 0) statement.bind(paramsWithColons) - statement.step() - statement.reset(true) + const stmt = this.statement + if (params.length > 0) { + stmt.bind(params) + } + + try { + stmt.step() + } finally { + stmt.reset(true) + } } } diff --git a/packages/modeldb-sqlite/src/encoding.ts b/packages/modeldb-sqlite/src/encoding.ts index 9dd006bd2..491380ac5 100644 --- a/packages/modeldb-sqlite/src/encoding.ts +++ b/packages/modeldb-sqlite/src/encoding.ts @@ -1,6 +1,12 @@ import * as json from "@ipld/dag-json" -import { PrimaryKeyValue, PrimitiveProperty, PrimitiveValue, PropertyValue, PrimitiveType } from "@canvas-js/modeldb" +import type { + PrimaryKeyValue, + PrimitiveProperty, + PrimitiveValue, + PropertyValue, + PrimitiveType, +} from "@canvas-js/modeldb" import { assert, signalInvalidType } from "@canvas-js/utils" @@ -8,7 +14,7 @@ import { assert, signalInvalidType } from "@canvas-js/utils" // this may not match onto the types in the model // because sqlite does not natively support all of the types we might want // for example, sqlite does not have a boolean or a json type -export type SqlitePrimitiveValue = string | number | Buffer | null +export type SqlitePrimitiveValue = null | number | string | Buffer const fromBuffer = (data: Buffer): Uint8Array => new Uint8Array(data.buffer, data.byteOffset, data.byteLength) const toBuffer = (data: Uint8Array) => Buffer.from(data.buffer, data.byteOffset, data.byteLength)