From 1731abd9694323a2115775b488663afc6089293c Mon Sep 17 00:00:00 2001 From: Levko Kravets Date: Thu, 3 Aug 2023 17:15:57 +0300 Subject: [PATCH] [PECO-238] Initial CloudFetch implementation Signed-off-by: Levko Kravets --- lib/DBSQLOperation/FetchResultsHelper.ts | 51 ++++-- lib/DBSQLOperation/SchemaHelper.ts | 31 +++- lib/DBSQLOperation/index.ts | 2 +- lib/DBSQLSession.ts | 1 + lib/contracts/IDBSQLSession.ts | 1 + lib/result/ArrowResult.ts | 10 +- lib/result/CloudFetchResult.ts | 43 +++++ lib/result/IOperationResult.ts | 2 +- lib/result/JsonResult.ts | 2 +- package-lock.json | 196 +++++++++++++++++++++++ package.json | 2 + 11 files changed, 316 insertions(+), 25 deletions(-) create mode 100644 lib/result/CloudFetchResult.ts diff --git a/lib/DBSQLOperation/FetchResultsHelper.ts b/lib/DBSQLOperation/FetchResultsHelper.ts index e1d638ad..833b8894 100644 --- a/lib/DBSQLOperation/FetchResultsHelper.ts +++ b/lib/DBSQLOperation/FetchResultsHelper.ts @@ -42,7 +42,7 @@ export default class FetchResultsHelper { private fetchOrientation: TFetchOrientation = TFetchOrientation.FETCH_FIRST; - private prefetchedResults: TFetchResultsResp[] = []; + private pendingResults: TFetchResultsResp[] = []; private readonly returnOnlyPrefetchedResults: boolean; @@ -58,7 +58,7 @@ export default class FetchResultsHelper { this.operationHandle = operationHandle; prefetchedResults.forEach((item) => { if (item) { - this.prefetchedResults.push(item); + this.prepareCloudFetchChunks(item); } }); this.returnOnlyPrefetchedResults = returnOnlyPrefetchedResults; @@ -68,7 +68,7 @@ export default class FetchResultsHelper { Status.assert(response.status); this.fetchOrientation = TFetchOrientation.FETCH_NEXT; - if (this.prefetchedResults.length > 0) { + if (this.pendingResults.length > 0) { this.hasMoreRows = true; } else if (this.returnOnlyPrefetchedResults) { this.hasMoreRows = false; @@ -80,18 +80,45 @@ export default class FetchResultsHelper { } public async fetch(maxRows: number) { - const prefetchedResponse = this.prefetchedResults.shift(); - if (prefetchedResponse) { - return this.processFetchResponse(prefetchedResponse); + if (this.pendingResults.length == 0) { + const results = await this.driver.fetchResults({ + operationHandle: this.operationHandle, + orientation: this.fetchOrientation, + maxRows: new Int64(maxRows), + fetchType: FetchType.Data, + }); + + this.prepareCloudFetchChunks(results); } - const response = await this.driver.fetchResults({ - operationHandle: this.operationHandle, - orientation: this.fetchOrientation, - maxRows: new Int64(maxRows), - fetchType: FetchType.Data, - }); + const response = this.pendingResults.shift(); + // This check is rather for safety and to make TS happy. In practice, such a case should not happen + if (!response) { + throw new Error('Unexpected error: no more data'); + } return this.processFetchResponse(response); } + + private prepareCloudFetchChunks(response: TFetchResultsResp) { + // TODO: Make it configurable. Effectively, this is a concurrent downloads limit for an operation + const maxLinkCount = 1; + + if (response.results && response.results.resultLinks && response.results.resultLinks.length > 0) { + const allLinks = [...response.results.resultLinks]; + while (allLinks.length > 0) { + // Shallow clone the original response object, but rewrite cloud fetch links array + // to contain the only entry + const responseFragment = { + ...response, + results: { + ...response.results, + resultLinks: allLinks.splice(0, maxLinkCount), + }, + }; + + this.pendingResults.push(responseFragment); + } + } + } } diff --git a/lib/DBSQLOperation/SchemaHelper.ts b/lib/DBSQLOperation/SchemaHelper.ts index d8368cb8..c3ce8576 100644 --- a/lib/DBSQLOperation/SchemaHelper.ts +++ b/lib/DBSQLOperation/SchemaHelper.ts @@ -4,6 +4,7 @@ import Status from '../dto/Status'; import IOperationResult from '../result/IOperationResult'; import JsonResult from '../result/JsonResult'; import ArrowResult from '../result/ArrowResult'; +import CloudFetchResult from '../result/CloudFetchResult'; import HiveDriverError from '../errors/HiveDriverError'; import { definedOrError } from '../utils'; @@ -14,6 +15,8 @@ export default class SchemaHelper { private metadata?: TGetResultSetMetadataResp; + private resultHandler?: IOperationResult; + constructor(driver: HiveDriver, operationHandle: TOperationHandle, metadata?: TGetResultSetMetadataResp) { this.driver = driver; this.operationHandle = operationHandle; @@ -41,13 +44,27 @@ export default class SchemaHelper { const metadata = await this.fetchMetadata(); const resultFormat = definedOrError(metadata.resultFormat); - switch (resultFormat) { - case TSparkRowSetType.COLUMN_BASED_SET: - return new JsonResult(metadata.schema); - case TSparkRowSetType.ARROW_BASED_SET: - return new ArrowResult(metadata.schema, metadata.arrowSchema); - default: - throw new HiveDriverError(`Unsupported result format: ${TSparkRowSetType[resultFormat]}`); + if (!this.resultHandler) { + switch (resultFormat) { + case TSparkRowSetType.COLUMN_BASED_SET: + this.resultHandler = new JsonResult(metadata.schema); + break; + case TSparkRowSetType.ARROW_BASED_SET: + this.resultHandler = new ArrowResult(metadata.schema, metadata.arrowSchema); + break; + case TSparkRowSetType.URL_BASED_SET: + this.resultHandler = new CloudFetchResult(metadata.schema, metadata.arrowSchema); + break; + default: + this.resultHandler = undefined; + break; + } + } + + if (!this.resultHandler) { + throw new HiveDriverError(`Unsupported result format: ${TSparkRowSetType[resultFormat]}`); } + + return this.resultHandler; } } diff --git a/lib/DBSQLOperation/index.ts b/lib/DBSQLOperation/index.ts index 79c74505..e2cb5dd0 100644 --- a/lib/DBSQLOperation/index.ts +++ b/lib/DBSQLOperation/index.ts @@ -106,7 +106,7 @@ export default class DBSQLOperation implements IOperation { this._data.fetch(options?.maxRows || defaultMaxRows), ]); - const result = resultHandler.getValue(data ? [data] : []); + const result = await resultHandler.getValue(data ? [data] : []); this.logger?.log( LogLevel.debug, `Fetched chunk of size: ${options?.maxRows || defaultMaxRows} from operation with id: ${this.getId()}`, diff --git a/lib/DBSQLSession.ts b/lib/DBSQLSession.ts index 82b8cdf1..6eda1faf 100644 --- a/lib/DBSQLSession.ts +++ b/lib/DBSQLSession.ts @@ -125,6 +125,7 @@ export default class DBSQLSession implements IDBSQLSession { runAsync: options.runAsync || false, ...getDirectResultsOptions(options.maxRows), ...getArrowOptions(), + canDownloadResult: options.useCloudFetch ?? false, }); return this.createOperation(response); diff --git a/lib/contracts/IDBSQLSession.ts b/lib/contracts/IDBSQLSession.ts index 4dcdad1c..b80a6666 100644 --- a/lib/contracts/IDBSQLSession.ts +++ b/lib/contracts/IDBSQLSession.ts @@ -7,6 +7,7 @@ export type ExecuteStatementOptions = { queryTimeout?: Int64; runAsync?: boolean; maxRows?: number | null; + useCloudFetch?: boolean; }; export type TypeInfoRequest = { diff --git a/lib/result/ArrowResult.ts b/lib/result/ArrowResult.ts index 1f497226..b76169d2 100644 --- a/lib/result/ArrowResult.ts +++ b/lib/result/ArrowResult.ts @@ -30,12 +30,16 @@ export default class ArrowResult implements IOperationResult { this.arrowSchema = arrowSchema; } - getValue(data?: Array) { + async getValue(data?: Array) { if (this.schema.length === 0 || !this.arrowSchema || !data) { return []; } - const batches = this.getBatches(data); + const batches = await this.getBatches(data); + return this.batchesToRows(batches); + } + + protected batchesToRows(batches: Array) { if (batches.length === 0) { return []; } @@ -44,7 +48,7 @@ export default class ArrowResult implements IOperationResult { return this.getRows(table.schema, table.toArray()); } - private getBatches(data: Array): Array { + protected async getBatches(data: Array): Promise> { const result: Array = []; data.forEach((rowSet) => { diff --git a/lib/result/CloudFetchResult.ts b/lib/result/CloudFetchResult.ts new file mode 100644 index 00000000..406d29ae --- /dev/null +++ b/lib/result/CloudFetchResult.ts @@ -0,0 +1,43 @@ +import { Buffer } from 'buffer'; +import fetch from 'node-fetch'; +import { TRowSet, TSparkArrowResultLink } from '../../thrift/TCLIService_types'; +import ArrowResult from './ArrowResult'; + +export default class CloudFetchResult extends ArrowResult { + protected batchesToRows(batches: Array) { + if (batches.length === 1) { + return super.batchesToRows(batches); + } + + const results: Array> = []; + + for (const batch of batches) { + results.push(super.batchesToRows([batch])); + } + + return results.flat(1); + } + + protected async getBatches(data: Array): Promise> { + const tasks: Array> = []; + + data?.forEach((item) => { + item.resultLinks?.forEach((link) => { + tasks.push(this.downloadLink(link)); + }); + }); + + return await Promise.all(tasks); + } + + private async downloadLink(link: TSparkArrowResultLink): Promise { + // TODO: Process expired links + const response = await fetch(link.fileLink); + if (!response.ok) { + throw new Error(`CloudFetch HTTP error ${response.status} ${response.statusText}`); + } + + const result = await response.arrayBuffer(); + return Buffer.from(result); + } +} diff --git a/lib/result/IOperationResult.ts b/lib/result/IOperationResult.ts index 4e871c3a..b35dc823 100644 --- a/lib/result/IOperationResult.ts +++ b/lib/result/IOperationResult.ts @@ -1,5 +1,5 @@ import { TRowSet } from '../../thrift/TCLIService_types'; export default interface IOperationResult { - getValue(data?: Array): any; + getValue(data?: Array): Promise; } diff --git a/lib/result/JsonResult.ts b/lib/result/JsonResult.ts index 17cdbedc..5439ff80 100644 --- a/lib/result/JsonResult.ts +++ b/lib/result/JsonResult.ts @@ -10,7 +10,7 @@ export default class JsonResult implements IOperationResult { this.schema = getSchemaColumns(schema); } - getValue(data?: Array): Array { + async getValue(data?: Array): Promise> { if (this.schema.length === 0 || !data) { return []; } diff --git a/package-lock.json b/package-lock.json index 686a6657..68b39763 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,6 +12,7 @@ "dependencies": { "apache-arrow": "^10.0.1", "commander": "^9.3.0", + "node-fetch": "^2.6.12", "node-int64": "^0.4.0", "open": "^8.4.2", "openid-client": "^5.4.2", @@ -22,6 +23,7 @@ }, "devDependencies": { "@types/node": "^18.11.9", + "@types/node-fetch": "^2.6.4", "@types/node-int64": "^0.4.29", "@types/thrift": "^0.10.11", "@types/uuid": "^8.3.4", @@ -810,6 +812,16 @@ "integrity": "sha512-DHQpWGjyQKSHj3ebjFI/wRKcqQcdR+MoFBygntYOZytCqNfkd2ZC4ARDJ2DQqhjH5p85Nnd3jhUJIXrszFX/JA==", "dev": true }, + "node_modules/@types/node-fetch": { + "version": "2.6.4", + "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.4.tgz", + "integrity": "sha512-1ZX9fcN4Rvkvgv4E6PAY5WXUFWFcRWxZa3EW83UjycOB9ljJCedb2CupIP4RZMEwF/M3eTcCihbBRgwtGbg5Rg==", + "dev": true, + "dependencies": { + "@types/node": "*", + "form-data": "^3.0.0" + } + }, "node_modules/@types/node-int64": { "version": "0.4.29", "resolved": "https://registry.npmjs.org/@types/node-int64/-/node-int64-0.4.29.tgz", @@ -1305,6 +1317,12 @@ "resolved": "https://registry.npmjs.org/async-limiter/-/async-limiter-1.0.1.tgz", "integrity": "sha512-csOlWGAcRFJaI6m+F2WKdnMKr4HhdhFVBk0H/QbJFMCr+uO2kwohwXQPxw/9OCxp05r5ghVBFSyioixx3gfkNQ==" }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", + "dev": true + }, "node_modules/at-least-node": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/at-least-node/-/at-least-node-1.0.0.tgz", @@ -1634,6 +1652,18 @@ "text-hex": "1.0.x" } }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "dev": true, + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, "node_modules/command-line-args": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/command-line-args/-/command-line-args-5.2.1.tgz", @@ -1894,6 +1924,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "dev": true, + "engines": { + "node": ">=0.4.0" + } + }, "node_modules/diff": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/diff/-/diff-5.0.0.tgz", @@ -2708,6 +2747,20 @@ "node": ">=8.0.0" } }, + "node_modules/form-data": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz", + "integrity": "sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==", + "dev": true, + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/fromentries": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/fromentries/-/fromentries-1.3.2.tgz", @@ -3828,6 +3881,27 @@ "node": ">=8.6" } }, + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "dev": true, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "dev": true, + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/minimatch": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", @@ -3968,6 +4042,25 @@ "path-to-regexp": "^1.7.0" } }, + "node_modules/node-fetch": { + "version": "2.6.12", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.12.tgz", + "integrity": "sha512-C/fGU2E8ToujUivIO0H+tpQ6HWo4eEmchoPIoXtxCrVghxdKq+QOHqEZW7tuP3KlV3bC8FRMO5nMCC7Zm1VP6g==", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, "node_modules/node-int64": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz", @@ -5345,6 +5438,11 @@ "node": ">=8.0" } }, + "node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" + }, "node_modules/triple-beam": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/triple-beam/-/triple-beam-1.3.0.tgz", @@ -5544,6 +5642,20 @@ "integrity": "sha512-l8lCEmLcLYZh4nbunNZvQCJc5pv7+RCwa8q/LdUx8u7lsWvPDKmpodJAJNwkAhJC//dFY48KuIEmjtd4RViDrA==", "dev": true }, + "node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" + }, + "node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, "node_modules/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", @@ -6402,6 +6514,16 @@ "integrity": "sha512-DHQpWGjyQKSHj3ebjFI/wRKcqQcdR+MoFBygntYOZytCqNfkd2ZC4ARDJ2DQqhjH5p85Nnd3jhUJIXrszFX/JA==", "dev": true }, + "@types/node-fetch": { + "version": "2.6.4", + "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.4.tgz", + "integrity": "sha512-1ZX9fcN4Rvkvgv4E6PAY5WXUFWFcRWxZa3EW83UjycOB9ljJCedb2CupIP4RZMEwF/M3eTcCihbBRgwtGbg5Rg==", + "dev": true, + "requires": { + "@types/node": "*", + "form-data": "^3.0.0" + } + }, "@types/node-int64": { "version": "0.4.29", "resolved": "https://registry.npmjs.org/@types/node-int64/-/node-int64-0.4.29.tgz", @@ -6744,6 +6866,12 @@ "resolved": "https://registry.npmjs.org/async-limiter/-/async-limiter-1.0.1.tgz", "integrity": "sha512-csOlWGAcRFJaI6m+F2WKdnMKr4HhdhFVBk0H/QbJFMCr+uO2kwohwXQPxw/9OCxp05r5ghVBFSyioixx3gfkNQ==" }, + "asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", + "dev": true + }, "at-least-node": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/at-least-node/-/at-least-node-1.0.0.tgz", @@ -6986,6 +7114,15 @@ "text-hex": "1.0.x" } }, + "combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "dev": true, + "requires": { + "delayed-stream": "~1.0.0" + } + }, "command-line-args": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/command-line-args/-/command-line-args-5.2.1.tgz", @@ -7181,6 +7318,12 @@ "object-keys": "^1.1.1" } }, + "delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "dev": true + }, "diff": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/diff/-/diff-5.0.0.tgz", @@ -7811,6 +7954,17 @@ "signal-exit": "^3.0.2" } }, + "form-data": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz", + "integrity": "sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==", + "dev": true, + "requires": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + } + }, "fromentries": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/fromentries/-/fromentries-1.3.2.tgz", @@ -8614,6 +8768,21 @@ "picomatch": "^2.3.1" } }, + "mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "dev": true + }, + "mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "dev": true, + "requires": { + "mime-db": "1.52.0" + } + }, "minimatch": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", @@ -8727,6 +8896,14 @@ "path-to-regexp": "^1.7.0" } }, + "node-fetch": { + "version": "2.6.12", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.12.tgz", + "integrity": "sha512-C/fGU2E8ToujUivIO0H+tpQ6HWo4eEmchoPIoXtxCrVghxdKq+QOHqEZW7tuP3KlV3bC8FRMO5nMCC7Zm1VP6g==", + "requires": { + "whatwg-url": "^5.0.0" + } + }, "node-int64": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz", @@ -9731,6 +9908,11 @@ "is-number": "^7.0.0" } }, + "tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" + }, "triple-beam": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/triple-beam/-/triple-beam-1.3.0.tgz", @@ -9873,6 +10055,20 @@ "integrity": "sha512-l8lCEmLcLYZh4nbunNZvQCJc5pv7+RCwa8q/LdUx8u7lsWvPDKmpodJAJNwkAhJC//dFY48KuIEmjtd4RViDrA==", "dev": true }, + "webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" + }, + "whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "requires": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, "which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", diff --git a/package.json b/package.json index 0c7949b1..aa82c095 100644 --- a/package.json +++ b/package.json @@ -50,6 +50,7 @@ "license": "Apache 2.0", "devDependencies": { "@types/node": "^18.11.9", + "@types/node-fetch": "^2.6.4", "@types/node-int64": "^0.4.29", "@types/thrift": "^0.10.11", "@types/uuid": "^8.3.4", @@ -73,6 +74,7 @@ "dependencies": { "apache-arrow": "^10.0.1", "commander": "^9.3.0", + "node-fetch": "^2.6.12", "node-int64": "^0.4.0", "open": "^8.4.2", "openid-client": "^5.4.2",