diff --git a/.changeset/lovely-impalas-do.md b/.changeset/lovely-impalas-do.md new file mode 100644 index 000000000..f2d561224 --- /dev/null +++ b/.changeset/lovely-impalas-do.md @@ -0,0 +1,5 @@ +--- +'@powersync/web': minor +--- + +Add cacheSizeKb option, defaulting to 50MB. diff --git a/.changeset/slow-spiders-smash.md b/.changeset/slow-spiders-smash.md new file mode 100644 index 000000000..258550a69 --- /dev/null +++ b/.changeset/slow-spiders-smash.md @@ -0,0 +1,5 @@ +--- +'@powersync/diagnostics-app': minor +--- + +Switch diagnostics app to OPFS. diff --git a/.changeset/spotty-students-serve.md b/.changeset/spotty-students-serve.md new file mode 100644 index 000000000..857ea4011 --- /dev/null +++ b/.changeset/spotty-students-serve.md @@ -0,0 +1,5 @@ +--- +'@powersync/op-sqlite': minor +--- + +Default to using memory for temp store, and 50MB cache size. diff --git a/packages/powersync-op-sqlite/README.md b/packages/powersync-op-sqlite/README.md index e348b7f76..3b9e31556 100644 --- a/packages/powersync-op-sqlite/README.md +++ b/packages/powersync-op-sqlite/README.md @@ -74,9 +74,7 @@ To load additional SQLite extensions include the `extensions` option in `sqliteO ```js sqliteOptions: { - extensions: [ - { path: libPath, entryPoint: 'sqlite3_powersync_init' } - ] + extensions: [{ path: libPath, entryPoint: 'sqlite3_powersync_init' }]; } ``` @@ -87,9 +85,9 @@ Example usage: ```ts import { getDylibPath } from '@op-engineering/op-sqlite'; -let libPath: string +let libPath: string; if (Platform.OS === 'ios') { - libPath = getDylibPath('co.powersync.sqlitecore', 'powersync-sqlite-core') + libPath = getDylibPath('co.powersync.sqlitecore', 'powersync-sqlite-core'); } else { libPath = 'libpowersync'; } @@ -97,28 +95,11 @@ if (Platform.OS === 'ios') { const factory = new OPSqliteOpenFactory({ dbFilename: 'sqlite.db', sqliteOptions: { - extensions: [ - { path: libPath, entryPoint: 'sqlite3_powersync_init' } - ] + extensions: [{ path: libPath, entryPoint: 'sqlite3_powersync_init' }] } }); ``` -## Using the Memory Temporary Store - -For some targets like Android 12/API 31, syncing of large datasets may cause disk IO errors due to the default temporary store option (file) used. -To resolve this you can use the `memory` option, by adding the following configuration option to your application's `package.json` - -```json -{ - // your normal package.json - // ... - "op-sqlite": { - "sqliteFlags": "-DSQLITE_TEMP_STORE=2" - } -} -``` - ## Native Projects This package uses native libraries. Create native Android and iOS projects (if not created already) by running: diff --git a/packages/powersync-op-sqlite/src/db/OPSqliteAdapter.ts b/packages/powersync-op-sqlite/src/db/OPSqliteAdapter.ts index 7b2423656..6bc035b46 100644 --- a/packages/powersync-op-sqlite/src/db/OPSqliteAdapter.ts +++ b/packages/powersync-op-sqlite/src/db/OPSqliteAdapter.ts @@ -44,19 +44,28 @@ export class OPSQLiteDBAdapter extends BaseObserver implement } protected async init() { - const { lockTimeoutMs, journalMode, journalSizeLimit, synchronous } = this.options.sqliteOptions!; + const { lockTimeoutMs, journalMode, journalSizeLimit, synchronous, cacheSizeKb, temporaryStorage } = + this.options.sqliteOptions!; const dbFilename = this.options.name; this.writeConnection = await this.openConnection(dbFilename); - const statements: string[] = [ + const baseStatements = [ `PRAGMA busy_timeout = ${lockTimeoutMs}`, + `PRAGMA cache_size = -${cacheSizeKb}`, + `PRAGMA temp_store = ${temporaryStorage}` + ]; + + const writeConnectionStatements = [ + ...baseStatements, `PRAGMA journal_mode = ${journalMode}`, `PRAGMA journal_size_limit = ${journalSizeLimit}`, `PRAGMA synchronous = ${synchronous}` ]; - for (const statement of statements) { + const readConnectionStatements = [...baseStatements, 'PRAGMA query_only = true']; + + for (const statement of writeConnectionStatements) { for (let tries = 0; tries < 30; tries++) { try { await this.writeConnection!.execute(statement); @@ -79,7 +88,9 @@ export class OPSQLiteDBAdapter extends BaseObserver implement this.readConnections = []; for (let i = 0; i < READ_CONNECTIONS; i++) { const conn = await this.openConnection(dbFilename); - await conn.execute('PRAGMA query_only = true'); + for (let statement of readConnectionStatements) { + await conn.execute(statement); + } this.readConnections.push({ busy: false, connection: conn }); } } diff --git a/packages/powersync-op-sqlite/src/db/SqliteOptions.ts b/packages/powersync-op-sqlite/src/db/SqliteOptions.ts index 9b5961bf2..9b962ef0e 100644 --- a/packages/powersync-op-sqlite/src/db/SqliteOptions.ts +++ b/packages/powersync-op-sqlite/src/db/SqliteOptions.ts @@ -30,6 +30,21 @@ export interface SqliteOptions { */ encryptionKey?: string | null; + /** + * Where to store SQLite temporary files. Defaults to 'MEMORY'. + * Setting this to `FILESYSTEM` can cause issues with larger queries or datasets. + * + * For details, see: https://www.sqlite.org/pragma.html#pragma_temp_store + */ + temporaryStorage?: TemporaryStorageOption; + + /** + * Maximum SQLite cache size. Defaults to 50MB. + * + * For details, see: https://www.sqlite.org/pragma.html#pragma_cache_size + */ + cacheSizeKb?: number; + /** * Load extensions using the path and entryPoint. * More info can be found here https://op-engineering.github.io/op-sqlite/docs/api#loading-extensions. @@ -40,6 +55,11 @@ export interface SqliteOptions { }>; } +export enum TemporaryStorageOption { + MEMORY = 'memory', + FILESYSTEM = 'file' +} + // SQLite journal mode. Set on the primary connection. // This library is written with WAL mode in mind - other modes may cause // unexpected locking behavior. @@ -65,6 +85,8 @@ export const DEFAULT_SQLITE_OPTIONS: Required = { journalMode: SqliteJournalMode.wal, synchronous: SqliteSynchronous.normal, journalSizeLimit: 6 * 1024 * 1024, + cacheSizeKb: 50 * 1024, + temporaryStorage: TemporaryStorageOption.MEMORY, lockTimeoutMs: 30000, encryptionKey: null, extensions: [] diff --git a/packages/web/src/db/adapters/wa-sqlite/WASQLiteConnection.ts b/packages/web/src/db/adapters/wa-sqlite/WASQLiteConnection.ts index d73800d96..3f8336b12 100644 --- a/packages/web/src/db/adapters/wa-sqlite/WASQLiteConnection.ts +++ b/packages/web/src/db/adapters/wa-sqlite/WASQLiteConnection.ts @@ -235,6 +235,7 @@ export class WASqliteConnection await this.openDB(); this.registerBroadcastListeners(); await this.executeSingleStatement(`PRAGMA temp_store = ${this.options.temporaryStorage};`); + await this.executeSingleStatement(`PRAGMA cache_size = -${this.options.cacheSizeKb};`); await this.executeEncryptionPragma(); this.sqliteAPI.update_hook(this.dbP, (updateType: number, dbName: string | null, tableName: string | null) => { diff --git a/packages/web/src/db/adapters/wa-sqlite/WASQLiteDBAdapter.ts b/packages/web/src/db/adapters/wa-sqlite/WASQLiteDBAdapter.ts index 2c96a00f6..01ffef3b0 100644 --- a/packages/web/src/db/adapters/wa-sqlite/WASQLiteDBAdapter.ts +++ b/packages/web/src/db/adapters/wa-sqlite/WASQLiteDBAdapter.ts @@ -3,7 +3,12 @@ import * as Comlink from 'comlink'; import { resolveWebPowerSyncFlags } from '../../PowerSyncDatabase'; import { OpenAsyncDatabaseConnection } from '../AsyncDatabaseConnection'; import { LockedAsyncDatabaseAdapter } from '../LockedAsyncDatabaseAdapter'; -import { ResolvedWebSQLOpenOptions, TemporaryStorageOption, WebSQLFlags } from '../web-sql-flags'; +import { + DEFAULT_CACHE_SIZE_KB, + ResolvedWebSQLOpenOptions, + TemporaryStorageOption, + WebSQLFlags +} from '../web-sql-flags'; import { WorkerWrappedAsyncDatabaseConnection } from '../WorkerWrappedAsyncDatabaseConnection'; import { WASQLiteVFS } from './WASQLiteConnection'; import { WASQLiteOpenFactory } from './WASQLiteOpenFactory'; @@ -27,6 +32,7 @@ export interface WASQLiteDBAdapterOptions extends Omit { - const { workerPort, temporaryStorage } = options; + const { workerPort, temporaryStorage, cacheSizeKb } = options; if (workerPort) { const remote = Comlink.wrap(workerPort); return new WorkerWrappedAsyncDatabaseConnection({ @@ -52,6 +58,7 @@ export class WASQLiteDBAdapter extends LockedAsyncDatabaseAdapter { baseConnection: await remote({ ...options, temporaryStorage: temporaryStorage ?? TemporaryStorageOption.MEMORY, + cacheSizeKb: cacheSizeKb ?? DEFAULT_CACHE_SIZE_KB, flags: resolveWebPowerSyncFlags(options.flags), encryptionKey: options.encryptionKey }) @@ -63,6 +70,7 @@ export class WASQLiteDBAdapter extends LockedAsyncDatabaseAdapter { debugMode: options.debugMode, flags: options.flags, temporaryStorage, + cacheSizeKb, logger: options.logger, vfs: options.vfs, encryptionKey: options.encryptionKey, diff --git a/packages/web/src/db/adapters/wa-sqlite/WASQLiteOpenFactory.ts b/packages/web/src/db/adapters/wa-sqlite/WASQLiteOpenFactory.ts index 816039b63..6217b35dc 100644 --- a/packages/web/src/db/adapters/wa-sqlite/WASQLiteOpenFactory.ts +++ b/packages/web/src/db/adapters/wa-sqlite/WASQLiteOpenFactory.ts @@ -4,7 +4,12 @@ import { openWorkerDatabasePort, resolveWorkerDatabasePortFactory } from '../../ import { AbstractWebSQLOpenFactory } from '../AbstractWebSQLOpenFactory'; import { AsyncDatabaseConnection, OpenAsyncDatabaseConnection } from '../AsyncDatabaseConnection'; import { LockedAsyncDatabaseAdapter } from '../LockedAsyncDatabaseAdapter'; -import { ResolvedWebSQLOpenOptions, TemporaryStorageOption, WebSQLOpenFactoryOptions } from '../web-sql-flags'; +import { + DEFAULT_CACHE_SIZE_KB, + ResolvedWebSQLOpenOptions, + TemporaryStorageOption, + WebSQLOpenFactoryOptions +} from '../web-sql-flags'; import { WorkerWrappedAsyncDatabaseConnection } from '../WorkerWrappedAsyncDatabaseConnection'; import { WASqliteConnection, WASQLiteVFS } from './WASQLiteConnection'; @@ -44,6 +49,7 @@ export class WASQLiteOpenFactory extends AbstractWebSQLOpenFactory { const { vfs = WASQLiteVFS.IDBBatchAtomicVFS, temporaryStorage = TemporaryStorageOption.MEMORY, + cacheSizeKb = DEFAULT_CACHE_SIZE_KB, encryptionKey } = this.waOptions; @@ -60,6 +66,7 @@ export class WASQLiteOpenFactory extends AbstractWebSQLOpenFactory { optionsDbWorker({ ...this.options, temporaryStorage, + cacheSizeKb, flags: this.resolvedFlags, encryptionKey }) @@ -74,6 +81,7 @@ export class WASQLiteOpenFactory extends AbstractWebSQLOpenFactory { dbFilename: this.options.dbFilename, vfs, temporaryStorage, + cacheSizeKb, flags: this.resolvedFlags, encryptionKey: encryptionKey }), @@ -94,6 +102,7 @@ export class WASQLiteOpenFactory extends AbstractWebSQLOpenFactory { debugMode: this.options.debugMode, vfs, temporaryStorage, + cacheSizeKb, flags: this.resolvedFlags, encryptionKey: encryptionKey }); diff --git a/packages/web/src/db/adapters/web-sql-flags.ts b/packages/web/src/db/adapters/web-sql-flags.ts index f2036f0eb..b2fdb4e4d 100644 --- a/packages/web/src/db/adapters/web-sql-flags.ts +++ b/packages/web/src/db/adapters/web-sql-flags.ts @@ -47,6 +47,8 @@ export interface ResolvedWebSQLOpenOptions extends SQLOpenOptions { */ temporaryStorage: TemporaryStorageOption; + cacheSizeKb: number; + /** * Encryption key for the database. * If set, the database will be encrypted using ChaCha20. @@ -59,6 +61,8 @@ export enum TemporaryStorageOption { FILESYSTEM = 'file' } +export const DEFAULT_CACHE_SIZE_KB = 50 * 1024; + /** * Options for opening a Web SQL connection */ @@ -74,12 +78,22 @@ export interface WebSQLOpenFactoryOptions extends SQLOpenOptions { worker?: string | URL | ((options: ResolvedWebSQLOpenOptions) => Worker | SharedWorker); logger?: ILogger; + /** * Where to store SQLite temporary files. Defaults to 'MEMORY'. * Setting this to `FILESYSTEM` can cause issues with larger queries or datasets. + * + * For details, see: https://www.sqlite.org/pragma.html#pragma_temp_store */ temporaryStorage?: TemporaryStorageOption; + /** + * Maximum SQLite cache size. Defaults to 50MB. + * + * For details, see: https://www.sqlite.org/pragma.html#pragma_cache_size + */ + cacheSizeKb?: number; + /** * Encryption key for the database. * If set, the database will be encrypted using ChaCha20. diff --git a/packages/web/src/db/sync/SharedWebStreamingSyncImplementation.ts b/packages/web/src/db/sync/SharedWebStreamingSyncImplementation.ts index b184fed53..916541d9f 100644 --- a/packages/web/src/db/sync/SharedWebStreamingSyncImplementation.ts +++ b/packages/web/src/db/sync/SharedWebStreamingSyncImplementation.ts @@ -6,7 +6,7 @@ import { SharedSyncClientEvent, SharedSyncImplementation } from '../../worker/sync/SharedSyncImplementation'; -import { resolveWebSQLFlags, TemporaryStorageOption } from '../adapters/web-sql-flags'; +import { DEFAULT_CACHE_SIZE_KB, resolveWebSQLFlags, TemporaryStorageOption } from '../adapters/web-sql-flags'; import { WebDBAdapter } from '../adapters/WebDBAdapter'; import { WebStreamingSyncImplementation, @@ -106,10 +106,10 @@ export class SharedWebStreamingSyncImplementation extends WebStreamingSyncImplem * This worker will manage all syncing operations remotely. */ const resolvedWorkerOptions = { - ...options, dbFilename: this.options.identifier!, - // TODO temporaryStorage: TemporaryStorageOption.MEMORY, + cacheSizeKb: DEFAULT_CACHE_SIZE_KB, + ...options, flags: resolveWebSQLFlags(options.flags) }; diff --git a/tools/diagnostics-app/src/library/powersync/ConnectionManager.ts b/tools/diagnostics-app/src/library/powersync/ConnectionManager.ts index 525952143..ad5da6aee 100644 --- a/tools/diagnostics-app/src/library/powersync/ConnectionManager.ts +++ b/tools/diagnostics-app/src/library/powersync/ConnectionManager.ts @@ -4,7 +4,10 @@ import { PowerSyncDatabase, WebRemote, WebStreamingSyncImplementation, - WebStreamingSyncImplementationOptions + WebStreamingSyncImplementationOptions, + WASQLiteOpenFactory, + TemporaryStorageOption, + WASQLiteVFS } from '@powersync/web'; import Logger from 'js-logger'; import { DynamicSchemaManager } from './DynamicSchemaManager'; @@ -25,14 +28,18 @@ export const getParams = () => { export const schemaManager = new DynamicSchemaManager(); +const openFactory = new WASQLiteOpenFactory({ + dbFilename: 'diagnostics.db', + debugMode: true, + cacheSizeKb: 500 * 1024, + temporaryStorage: TemporaryStorageOption.MEMORY, + vfs: WASQLiteVFS.OPFSCoopSyncVFS +}); + export const db = new PowerSyncDatabase({ - database: { - dbFilename: 'example.db', - debugMode: true - }, + database: openFactory, schema: schemaManager.buildSchema() }); -db.execute('PRAGMA cache_size=-500000'); export const connector = new TokenConnector();