From 52318fb4bd4e78dc6caddda696b5d10c4e08d63c Mon Sep 17 00:00:00 2001 From: Oskar Dudycz Date: Fri, 12 Jul 2024 10:32:47 +0200 Subject: [PATCH] Added Dumbo - utilities for PostgreSQL handling That will help to keep the shared code between Pongo and Emmett in one place. Dumbo will be versioned separately, which should give possibility to use different versions in Pongo and Emmett. And have this peer dependency more stable. Dumbo is treated for now as internal Pongo and Emmett dependency, so there are no plans (for now) to provide docs for it. --- src/package-lock.json | 27 ++- src/package.json | 3 +- src/packages/dumbo/package.json | 60 +++++++ src/packages/dumbo/src/connections/client.ts | 19 +++ src/packages/dumbo/src/connections/index.ts | 2 + .../src/connections}/pool.ts | 0 src/packages/dumbo/src/execute/index.ts | 158 ++++++++++++++++++ src/packages/dumbo/src/index.ts | 3 + .../src/postgres => dumbo/src}/sql/index.ts | 0 src/packages/dumbo/src/sql/schema.ts | 36 ++++ src/packages/dumbo/tsconfig.build.json | 6 + src/packages/dumbo/tsconfig.eslint.json | 3 + src/packages/dumbo/tsconfig.json | 11 ++ src/packages/dumbo/tsup.config.ts | 20 +++ src/packages/pongo/package.json | 3 +- .../src/e2e/compatibilityTest.e2e.spec.ts | 3 +- src/packages/pongo/src/postgres/client.ts | 2 +- .../pongo/src/postgres/execute/index.ts | 22 --- src/packages/pongo/src/postgres/index.ts | 3 - .../pongo/src/postgres/postgresCollection.ts | 3 +- .../pongo/src/postgres/update/index.ts | 2 +- src/packages/pongo/tsconfig.json | 10 +- src/tsconfig.json | 4 + 23 files changed, 363 insertions(+), 37 deletions(-) create mode 100644 src/packages/dumbo/package.json create mode 100644 src/packages/dumbo/src/connections/client.ts create mode 100644 src/packages/dumbo/src/connections/index.ts rename src/packages/{pongo/src/postgres => dumbo/src/connections}/pool.ts (100%) create mode 100644 src/packages/dumbo/src/execute/index.ts create mode 100644 src/packages/dumbo/src/index.ts rename src/packages/{pongo/src/postgres => dumbo/src}/sql/index.ts (100%) create mode 100644 src/packages/dumbo/src/sql/schema.ts create mode 100644 src/packages/dumbo/tsconfig.build.json create mode 100644 src/packages/dumbo/tsconfig.eslint.json create mode 100644 src/packages/dumbo/tsconfig.json create mode 100644 src/packages/dumbo/tsup.config.ts delete mode 100644 src/packages/pongo/src/postgres/execute/index.ts diff --git a/src/package-lock.json b/src/package-lock.json index a1cbc7a..5d6ebb0 100644 --- a/src/package-lock.json +++ b/src/package-lock.json @@ -1,13 +1,14 @@ { "name": "@event-driven-io/pongo-core", - "version": "0.2.4", + "version": "0.3.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@event-driven-io/pongo-core", - "version": "0.2.4", + "version": "0.3.0", "workspaces": [ + "packages/dumbo", "packages/pongo" ], "dependencies": { @@ -765,6 +766,10 @@ "node": "^12.22.0 || ^14.17.0 || >=16.0.0" } }, + "node_modules/@event-driven-io/dumbo": { + "resolved": "packages/dumbo", + "link": true + }, "node_modules/@event-driven-io/pongo": { "resolved": "packages/pongo", "link": true @@ -6618,13 +6623,29 @@ "node": "*" } }, + "packages/dumbo": { + "name": "@event-driven-io/dumbo", + "version": "0.1.0", + "devDependencies": { + "@types/node": "20.11.30" + }, + "peerDependencies": { + "@types/pg": "^8.11.6", + "@types/pg-format": "^1.0.5", + "@types/uuid": "^9.0.8", + "pg": "^8.12.0", + "pg-format": "^1.0.4", + "uuid": "^9.0.1" + } + }, "packages/pongo": { "name": "@event-driven-io/pongo", - "version": "0.2.4", + "version": "0.3.0", "devDependencies": { "@types/node": "20.11.30" }, "peerDependencies": { + "@event-driven-io/dumbo": "^0.1.0", "@types/mongodb": "^4.0.7", "@types/pg": "^8.11.6", "@types/pg-format": "^1.0.5", diff --git a/src/package.json b/src/package.json index f897633..8ddc0c3 100644 --- a/src/package.json +++ b/src/package.json @@ -1,6 +1,6 @@ { "name": "@event-driven-io/pongo-core", - "version": "0.2.4", + "version": "0.3.0", "description": "Pongo - Mongo with strong consistency on top of Postgres", "type": "module", "engines": { @@ -98,6 +98,7 @@ "pg-format": "^1.0.4" }, "workspaces": [ + "packages/dumbo", "packages/pongo" ] } diff --git a/src/packages/dumbo/package.json b/src/packages/dumbo/package.json new file mode 100644 index 0000000..0b0576f --- /dev/null +++ b/src/packages/dumbo/package.json @@ -0,0 +1,60 @@ +{ + "name": "@event-driven-io/dumbo", + "version": "0.1.0", + "description": "Dumbo - tools for dealing with PostgreSQL", + "type": "module", + "scripts": { + "build": "tsup", + "build:ts": "tsc", + "build:ts:watch": "tsc --watch", + "test": "run-s test:unit test:int test:e2e", + "test:unit": "glob -c \"node --import tsx --test\" **/*.unit.spec.ts", + "test:int": "glob -c \"node --import tsx --test\" **/*.int.spec.ts", + "test:e2e": "glob -c \"node --import tsx --test\" **/*.e2e.spec.ts", + "test:watch": "node --import tsx --test --watch", + "test:unit:watch": "glob -c \"node --import tsx --test --watch\" **/*.unit.spec.ts", + "test:int:watch": "glob -c \"node --import tsx --test --watch\" **/*.int.spec.ts", + "test:e2e:watch": "glob -c \"node --import tsx --test --watch\" **/*.e2e.spec.ts" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/event-driven-io/Pongo.git" + }, + "keywords": [ + "Event Sourcing" + ], + "author": "Oskar Dudycz", + "bugs": { + "url": "https://github.com/event-driven-io/Pongo/issues" + }, + "homepage": "https://event-driven-io.github.io/Pongo/", + "exports": { + ".": { + "import": { + "types": "./dist/index.d.ts", + "default": "./dist/index.js" + }, + "require": { + "types": "./dist/index.d.cts", + "default": "./dist/index.cjs" + } + } + }, + "main": "./dist/index.cjs", + "module": "./dist/index.js", + "types": "./dist/index.d.ts", + "files": [ + "dist" + ], + "peerDependencies": { + "@types/uuid": "^9.0.8", + "@types/pg": "^8.11.6", + "@types/pg-format": "^1.0.5", + "pg": "^8.12.0", + "pg-format": "^1.0.4", + "uuid": "^9.0.1" + }, + "devDependencies": { + "@types/node": "20.11.30" + } +} diff --git a/src/packages/dumbo/src/connections/client.ts b/src/packages/dumbo/src/connections/client.ts new file mode 100644 index 0000000..8b9fcf0 --- /dev/null +++ b/src/packages/dumbo/src/connections/client.ts @@ -0,0 +1,19 @@ +import pg from 'pg'; +import { endPool, getPool } from './pool'; + +export interface PostgresClient { + connect(): Promise; + close(): Promise; +} + +export const postgresClient = ( + connectionString: string, + database?: string, +): PostgresClient => { + const pool = getPool({ connectionString, database }); + + return { + connect: () => pool.connect(), + close: () => endPool(connectionString), + }; +}; diff --git a/src/packages/dumbo/src/connections/index.ts b/src/packages/dumbo/src/connections/index.ts new file mode 100644 index 0000000..b9936bb --- /dev/null +++ b/src/packages/dumbo/src/connections/index.ts @@ -0,0 +1,2 @@ +export * from './client'; +export * from './pool'; diff --git a/src/packages/pongo/src/postgres/pool.ts b/src/packages/dumbo/src/connections/pool.ts similarity index 100% rename from src/packages/pongo/src/postgres/pool.ts rename to src/packages/dumbo/src/connections/pool.ts diff --git a/src/packages/dumbo/src/execute/index.ts b/src/packages/dumbo/src/execute/index.ts new file mode 100644 index 0000000..f4cf280 --- /dev/null +++ b/src/packages/dumbo/src/execute/index.ts @@ -0,0 +1,158 @@ +import type pg from 'pg'; +import type { SQL } from '../sql'; + +export const execute = async ( + pool: pg.Pool, + handle: (client: pg.PoolClient) => Promise, +) => { + const client = await pool.connect(); + try { + return await handle(client); + } finally { + client.release(); + } +}; + +export const executeInTransaction = async ( + pool: pg.Pool, + handle: ( + client: pg.PoolClient, + ) => Promise<{ success: boolean; result: Result }>, +): Promise => + execute(pool, async (client) => { + try { + await client.query('BEGIN'); + + const { success, result } = await handle(client); + + if (success) await client.query('COMMIT'); + else await client.query('ROLLBACK'); + + return result; + } catch (e) { + await client.query('ROLLBACK'); + throw e; + } + }); + +export const executeSQL = async < + Result extends pg.QueryResultRow = pg.QueryResultRow, +>( + poolOrClient: pg.Pool | pg.PoolClient, + sql: SQL, +): Promise> => + 'totalCount' in poolOrClient + ? execute(poolOrClient, (client) => client.query(sql)) + : poolOrClient.query(sql); + +export const executeSQLInTransaction = async < + Result extends pg.QueryResultRow = pg.QueryResultRow, +>( + pool: pg.Pool, + sql: SQL, +) => { + console.log(sql); + return executeInTransaction(pool, async (client) => ({ + success: true, + result: await client.query(sql), + })); +}; + +export const executeSQLBatchInTransaction = async < + Result extends pg.QueryResultRow = pg.QueryResultRow, +>( + pool: pg.Pool, + ...sqls: SQL[] +) => + executeInTransaction(pool, async (client) => { + for (const sql of sqls) { + await client.query(sql); + } + + return { success: true, result: undefined }; + }); + +export const firstOrNull = async < + Result extends pg.QueryResultRow = pg.QueryResultRow, +>( + getResult: Promise>, +): Promise => { + const result = await getResult; + + return result.rows.length > 0 ? result.rows[0] ?? null : null; +}; + +export const first = async < + Result extends pg.QueryResultRow = pg.QueryResultRow, +>( + getResult: Promise>, +): Promise => { + const result = await getResult; + + if (result.rows.length === 0) + throw new Error("Query didn't return any result"); + + return result.rows[0]!; +}; + +export const singleOrNull = async < + Result extends pg.QueryResultRow = pg.QueryResultRow, +>( + getResult: Promise>, +): Promise => { + const result = await getResult; + + if (result.rows.length > 1) throw new Error('Query had more than one result'); + + return result.rows.length > 0 ? result.rows[0] ?? null : null; +}; + +export const single = async < + Result extends pg.QueryResultRow = pg.QueryResultRow, +>( + getResult: Promise>, +): Promise => { + const result = await getResult; + + if (result.rows.length === 0) + throw new Error("Query didn't return any result"); + + if (result.rows.length > 1) throw new Error('Query had more than one result'); + + return result.rows[0]!; +}; + +export const mapRows = async < + Result extends pg.QueryResultRow = pg.QueryResultRow, + Mapped = unknown, +>( + getResult: Promise>, + map: (row: Result) => Mapped, +): Promise => { + const result = await getResult; + + return result.rows.map(map); +}; + +export const toCamelCase = (snakeStr: string): string => + snakeStr.replace(/_([a-z])/g, (g) => g[1]?.toUpperCase() ?? ''); + +export const mapToCamelCase = >( + obj: T, +): T => { + const newObj: Record = {}; + for (const key in obj) { + if (Object.prototype.hasOwnProperty.call(obj, key)) { + newObj[toCamelCase(key)] = obj[key]; + } + } + return newObj as T; +}; + +export type ExistsSQLQueryResult = { exists: boolean }; + +export const exists = async (pool: pg.Pool, sql: SQL): Promise => { + const result = await single(executeSQL(pool, sql)); + + return result.exists === true; +}; diff --git a/src/packages/dumbo/src/index.ts b/src/packages/dumbo/src/index.ts new file mode 100644 index 0000000..9a0a1d2 --- /dev/null +++ b/src/packages/dumbo/src/index.ts @@ -0,0 +1,3 @@ +export * from './connections'; +export * from './execute'; +export * from './sql'; diff --git a/src/packages/pongo/src/postgres/sql/index.ts b/src/packages/dumbo/src/sql/index.ts similarity index 100% rename from src/packages/pongo/src/postgres/sql/index.ts rename to src/packages/dumbo/src/sql/index.ts diff --git a/src/packages/dumbo/src/sql/schema.ts b/src/packages/dumbo/src/sql/schema.ts new file mode 100644 index 0000000..8c6960f --- /dev/null +++ b/src/packages/dumbo/src/sql/schema.ts @@ -0,0 +1,36 @@ +import pg from 'pg'; +import { sql, type SQL } from '.'; +import { exists } from '../execute'; +export * from './schema'; + +export const tableExistsSQL = (tableName: string): SQL => + sql( + ` + SELECT EXISTS ( + SELECT FROM pg_tables + WHERE tablename = %L + ) AS exists;`, + tableName, + ); + +export const tableExists = async ( + pool: pg.Pool, + tableName: string, +): Promise => exists(pool, tableExistsSQL(tableName)); + +export const functionExistsSQL = (functionName: string): SQL => + sql( + ` + SELECT EXISTS ( + SELECT FROM pg_proc + WHERE + proname = %L + ) AS exists; + `, + functionName, + ); + +export const functionExists = async ( + pool: pg.Pool, + functionName: string, +): Promise => exists(pool, functionExistsSQL(functionName)); diff --git a/src/packages/dumbo/tsconfig.build.json b/src/packages/dumbo/tsconfig.build.json new file mode 100644 index 0000000..80b6a0a --- /dev/null +++ b/src/packages/dumbo/tsconfig.build.json @@ -0,0 +1,6 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "composite": false + } +} diff --git a/src/packages/dumbo/tsconfig.eslint.json b/src/packages/dumbo/tsconfig.eslint.json new file mode 100644 index 0000000..fc8520e --- /dev/null +++ b/src/packages/dumbo/tsconfig.eslint.json @@ -0,0 +1,3 @@ +{ + "extends": "./tsconfig.json" +} diff --git a/src/packages/dumbo/tsconfig.json b/src/packages/dumbo/tsconfig.json new file mode 100644 index 0000000..55f4a2c --- /dev/null +++ b/src/packages/dumbo/tsconfig.json @@ -0,0 +1,11 @@ +{ + "extends": "../../tsconfig.shared.json", + "include": ["./src/**/*"], + "compilerOptions": { + "composite": true, + "outDir": "./dist" /* Redirect output structure to the directory. */, + "rootDir": "./src", + "paths": {} + }, + "references": [] +} diff --git a/src/packages/dumbo/tsup.config.ts b/src/packages/dumbo/tsup.config.ts new file mode 100644 index 0000000..b4eb24e --- /dev/null +++ b/src/packages/dumbo/tsup.config.ts @@ -0,0 +1,20 @@ +import { defineConfig } from 'tsup'; + +const env = process.env.NODE_ENV; + +export default defineConfig({ + splitting: true, + clean: true, // clean up the dist folder + dts: true, // generate dts files + format: ['esm', 'cjs'], // generate cjs and esm files + minify: true, //env === 'production', + bundle: true, //env === 'production', + skipNodeModulesBundle: true, + watch: env === 'development', + target: 'esnext', + outDir: 'dist', //env === 'production' ? 'dist' : 'lib', + entry: ['src/index.ts'], + //entry: ['src/**/*.ts', '!src/**/*.spec.ts', '!src/**/*.internal.ts'], //include all files under src but not specs + sourcemap: true, + tsconfig: 'tsconfig.build.json', // workaround for https://github.com/egoist/tsup/issues/571#issuecomment-1760052931 +}); diff --git a/src/packages/pongo/package.json b/src/packages/pongo/package.json index a024b99..62bb022 100644 --- a/src/packages/pongo/package.json +++ b/src/packages/pongo/package.json @@ -1,6 +1,6 @@ { "name": "@event-driven-io/pongo", - "version": "0.2.4", + "version": "0.3.0", "description": "Pongo - Mongo with strong consistency on top of Postgres", "type": "module", "scripts": { @@ -51,6 +51,7 @@ "@types/pg": "^8.11.6", "@types/pg-format": "^1.0.5", "@types/mongodb": "^4.0.7", + "@event-driven-io/dumbo": "^0.1.0", "pg": "^8.12.0", "pg-format": "^1.0.4", "uuid": "^9.0.1" diff --git a/src/packages/pongo/src/e2e/compatibilityTest.e2e.spec.ts b/src/packages/pongo/src/e2e/compatibilityTest.e2e.spec.ts index 45f7ce3..7fb861b 100644 --- a/src/packages/pongo/src/e2e/compatibilityTest.e2e.spec.ts +++ b/src/packages/pongo/src/e2e/compatibilityTest.e2e.spec.ts @@ -1,3 +1,4 @@ +import { endAllPools } from '@event-driven-io/dumbo'; import { MongoDBContainer, type StartedMongoDBContainer, @@ -9,7 +10,7 @@ import { import assert from 'assert'; import { Db as MongoDb, MongoClient as OriginalMongoClient } from 'mongodb'; import { after, before, describe, it } from 'node:test'; -import { MongoClient, endAllPools, type Db } from '../'; +import { MongoClient, type Db } from '../'; type History = { street: string }; type Address = { diff --git a/src/packages/pongo/src/postgres/client.ts b/src/packages/pongo/src/postgres/client.ts index c64c5da..d30223e 100644 --- a/src/packages/pongo/src/postgres/client.ts +++ b/src/packages/pongo/src/postgres/client.ts @@ -1,5 +1,5 @@ +import { endPool, getPool } from '@event-driven-io/dumbo'; import { type DbClient } from '../main'; -import { endPool, getPool } from './pool'; import { postgresCollection } from './postgresCollection'; export const postgresClient = ( diff --git a/src/packages/pongo/src/postgres/execute/index.ts b/src/packages/pongo/src/postgres/execute/index.ts deleted file mode 100644 index a7c9132..0000000 --- a/src/packages/pongo/src/postgres/execute/index.ts +++ /dev/null @@ -1,22 +0,0 @@ -import type pg from 'pg'; -import type { SQL } from '../sql'; - -export const execute = async ( - pool: pg.Pool, - handle: (client: pg.PoolClient) => Promise, -) => { - const client = await pool.connect(); - try { - return await handle(client); - } finally { - client.release(); - } -}; - -export const executeSQL = async < - Result extends pg.QueryResultRow = pg.QueryResultRow, ->( - pool: pg.Pool, - sql: SQL, -): Promise> => - execute(pool, (client) => client.query(sql)); diff --git a/src/packages/pongo/src/postgres/index.ts b/src/packages/pongo/src/postgres/index.ts index 380bbab..17b4d87 100644 --- a/src/packages/pongo/src/postgres/index.ts +++ b/src/packages/pongo/src/postgres/index.ts @@ -1,7 +1,4 @@ export * from './client'; -export * from './execute'; export * from './filter'; -export * from './pool'; export * from './postgresCollection'; -export * from './sql'; export * from './update'; diff --git a/src/packages/pongo/src/postgres/postgresCollection.ts b/src/packages/pongo/src/postgres/postgresCollection.ts index c461e60..a93c76f 100644 --- a/src/packages/pongo/src/postgres/postgresCollection.ts +++ b/src/packages/pongo/src/postgres/postgresCollection.ts @@ -1,3 +1,4 @@ +import { executeSQL, sql, type SQL } from '@event-driven-io/dumbo'; import pg from 'pg'; import format from 'pg-format'; import { v4 as uuid } from 'uuid'; @@ -11,9 +12,7 @@ import { type PongoUpdateResult, type WithId, } from '../main'; -import { executeSQL } from './execute'; import { constructFilterQuery } from './filter'; -import { sql, type SQL } from './sql'; import { buildUpdateQuery } from './update'; export const postgresCollection = ( diff --git a/src/packages/pongo/src/postgres/update/index.ts b/src/packages/pongo/src/postgres/update/index.ts index b0efc69..c213e94 100644 --- a/src/packages/pongo/src/postgres/update/index.ts +++ b/src/packages/pongo/src/postgres/update/index.ts @@ -1,6 +1,6 @@ +import { sql, type SQL } from '@event-driven-io/dumbo'; import type { $inc, $push, $set, $unset, PongoUpdate } from '../../main'; import { entries } from '../../main/typing'; -import { sql, type SQL } from '../sql'; export const buildUpdateQuery = (update: PongoUpdate): SQL => entries(update).reduce((currentUpdateQuery, [op, value]) => { diff --git a/src/packages/pongo/tsconfig.json b/src/packages/pongo/tsconfig.json index 55f4a2c..09a81dc 100644 --- a/src/packages/pongo/tsconfig.json +++ b/src/packages/pongo/tsconfig.json @@ -5,7 +5,13 @@ "composite": true, "outDir": "./dist" /* Redirect output structure to the directory. */, "rootDir": "./src", - "paths": {} + "paths": { + "@event-driven-io/emmett": ["../packages/dumbo"] + } }, - "references": [] + "references": [ + { + "path": "../dumbo/" + } + ] } diff --git a/src/tsconfig.json b/src/tsconfig.json index 8405473..c89399e 100644 --- a/src/tsconfig.json +++ b/src/tsconfig.json @@ -6,10 +6,14 @@ "compilerOptions": { "noEmit": true /* Do not emit outputs. */, "paths": { + "@event-driven-io/dumbo": ["./packages/dumbo/src"], "@event-driven-io/pongo": ["./packages/pongo/src"] } }, "references": [ + { + "path": "./packages/dumbo/" + }, { "path": "./packages/pongo/" }