diff --git a/packages/cubejs-postgres-driver/src/PostgresDriver.ts b/packages/cubejs-postgres-driver/src/PostgresDriver.ts index fddd6de21f5ab..c3b5e1c4ad93c 100644 --- a/packages/cubejs-postgres-driver/src/PostgresDriver.ts +++ b/packages/cubejs-postgres-driver/src/PostgresDriver.ts @@ -296,6 +296,8 @@ export class PostgresDriver { + PostgresDriver.checkValuesLimit(values); + const conn = await this.pool.connect(); try { @@ -324,7 +326,22 @@ export class PostgresDriver= 65536) { + throw new Error(`PostgreSQL protocol does not support more than 65535 parameters, but ${length} passed`); + } + } + protected async queryResponse(query: string, values: unknown[]) { + PostgresDriver.checkValuesLimit(values); + const conn = await this.pool.connect(); try { diff --git a/packages/cubejs-postgres-driver/test/PostgresDriver.test.ts b/packages/cubejs-postgres-driver/test/PostgresDriver.test.ts index dc4ec3e20c46c..d7034c80d6e94 100644 --- a/packages/cubejs-postgres-driver/test/PostgresDriver.test.ts +++ b/packages/cubejs-postgres-driver/test/PostgresDriver.test.ts @@ -4,6 +4,10 @@ import { PostgresDriver } from '../src'; const streamToArray = require('stream-to-array'); +function largeParams(): Array { + return new Array(65536).fill('foo'); +} + describe('PostgresDriver', () => { let container: StartedTestContainer; let driver: PostgresDriver; @@ -56,6 +60,14 @@ describe('PostgresDriver', () => { ]); }); + test('too many params', async () => { + await expect( + driver.query(`SELECT 'foo'::TEXT;`, largeParams()) + ) + .rejects + .toThrow('PostgreSQL protocol does not support more than 65535 parameters, but 65536 passed'); + }); + test('stream', async () => { await driver.uploadTable( 'test.streaming_test', @@ -116,6 +128,20 @@ describe('PostgresDriver', () => { } }); + test('stream (too many params)', async () => { + try { + await driver.stream('select * from test.streaming_test', largeParams(), { + highWaterMark: 1000, + }); + + throw new Error('stream must throw an exception'); + } catch (e: any) { + expect(e.message).toEqual( + 'PostgreSQL protocol does not support more than 65535 parameters, but 65536 passed' + ); + } + }); + // Note: This test MUST be the last in the list. test('release', async () => { expect(async () => { diff --git a/packages/cubejs-redshift-driver/src/RedshiftDriver.ts b/packages/cubejs-redshift-driver/src/RedshiftDriver.ts index a572b5f35fb7e..cb3ced34ac94f 100644 --- a/packages/cubejs-redshift-driver/src/RedshiftDriver.ts +++ b/packages/cubejs-redshift-driver/src/RedshiftDriver.ts @@ -6,7 +6,13 @@ import { getEnv } from '@cubejs-backend/shared'; import { PostgresDriver, PostgresDriverConfiguration } from '@cubejs-backend/postgres-driver'; -import { DownloadTableCSVData, DriverCapabilities, UnloadOptions } from '@cubejs-backend/base-driver'; +import { + DownloadTableCSVData, + DriverCapabilities, + StreamOptions, + StreamTableDataWithTypes, + UnloadOptions +} from '@cubejs-backend/base-driver'; import crypto from 'crypto'; interface RedshiftDriverExportRequiredAWS { @@ -91,6 +97,32 @@ export class RedshiftDriver extends PostgresDriver }; } + protected static checkValuesLimit(values?: unknown[]) { + // Redshift server is not exactly compatible with PostgreSQL protocol + // And breaks after 32767 parameter values with `there is no parameter $-32768` + // This is a bug/misbehaviour on server side, nothing we can do besides generate a more meaningful error + const length = (values?.length ?? 0); + if (length >= 32768) { + throw new Error(`Redshift server does not support more than 32767 parameters, but ${length} passed`); + } + } + + public override async stream( + query: string, + values: unknown[], + options: StreamOptions + ): Promise { + RedshiftDriver.checkValuesLimit(values); + + return super.stream(query, values, options); + } + + protected override async queryResponse(query: string, values: unknown[]) { + RedshiftDriver.checkValuesLimit(values); + + return super.queryResponse(query, values); + } + protected getExportBucket( dataSource: string, ): RedshiftDriverExportAWS | undefined {