From 07a2b3d2f5edb111656bba588f3c84962186e843 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Fri, 17 May 2024 10:06:24 -0400 Subject: [PATCH] Add support for pagination at table level. (#1244) (#1538) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(postgres): ✨ add transfer pagination support * test(postgres): ✅ add transfer pagination test * feat(comlink): ✨ add pagination for getTransfers * test(ender): ✅ update transfer handler test * feat(postgres): ✨ add fill pagination support * test(postgres): ✅ add fill pagination test * feat(comlink): ✨ add pagination for getTrades and getFills * test: :white_check_mark: update pagination tests updated pagination tests to also evaluate total pages. * feat(postgres): :label: add explicit types added explicit types for fill and transfer tables. * feat(postgres): :sparkles: add order pagination support * test(postgres): :white_check_mark: add order pagination test * feat: :recycle: update order findAll usage * revert: :rewind: revert add pagination for getTransfers * revert: :rewind: revert add pagination for getTrades and getFills * feat: :label: add explicit ts types * docs: :bulb: add pagination todo comment * docs(postgres): :bulb: improve pagination comment improve pagination comment thanks to coderabbitai. --------- Co-authored-by: Davide Segullo (cherry picked from commit e12ebf24d0c167b2ff33fdbe784d9d3c793b4aa0) Co-authored-by: Giorgio Nocera --- .../__tests__/stores/fill-table.test.ts | 67 +++++++++++++-- .../__tests__/stores/order-table.test.ts | 82 +++++++++++++++++-- .../__tests__/stores/transfer-table.test.ts | 57 ++++++++++++- .../postgres/src/stores/fill-table.ts | 38 ++++++++- .../postgres/src/stores/order-table.ts | 38 ++++++++- .../postgres/src/stores/transfer-table.ts | 38 ++++++++- indexer/packages/postgres/src/types/index.ts | 1 + .../postgres/src/types/pagination-types.ts | 6 ++ .../postgres/src/types/query-types.ts | 2 + .../controllers/api/v4/fills-controller.ts | 4 +- .../controllers/api/v4/orders-controller.ts | 9 +- .../controllers/api/v4/trades-controller.ts | 2 +- .../api/v4/transfers-controller.ts | 12 +-- .../handlers/transfer-handler.test.ts | 6 +- .../tasks/cancel-stale-orders.test.ts | 39 +++++---- .../src/tasks/cancel-stale-orders.ts | 28 ++++--- 16 files changed, 357 insertions(+), 72 deletions(-) create mode 100644 indexer/packages/postgres/src/types/pagination-types.ts diff --git a/indexer/packages/postgres/__tests__/stores/fill-table.test.ts b/indexer/packages/postgres/__tests__/stores/fill-table.test.ts index d31ed9e177..e641c0da36 100644 --- a/indexer/packages/postgres/__tests__/stores/fill-table.test.ts +++ b/indexer/packages/postgres/__tests__/stores/fill-table.test.ts @@ -71,7 +71,7 @@ describe('Fill store', () => { FillTable.create(defaultFill), ]); - const fills: FillFromDatabase[] = await FillTable.findAll({}, [], {}); + const { results: fills } = await FillTable.findAll({}, [], {}); expect(fills.length).toEqual(2); expect(fills[0]).toEqual(expect.objectContaining(defaultFill)); @@ -91,7 +91,7 @@ describe('Fill store', () => { }), ]); - const fills: FillFromDatabase[] = await FillTable.findAll({}, [], { + const { results: fills } = await FillTable.findAll({}, [], { orderBy: [[FillColumns.eventId, Ordering.DESC]], }); @@ -103,6 +103,59 @@ describe('Fill store', () => { expect(fills[1]).toEqual(expect.objectContaining(defaultFill)); }); + it('Successfully finds fills using pagination', async () => { + await Promise.all([ + FillTable.create(defaultFill), + FillTable.create({ + ...defaultFill, + eventId: defaultTendermintEventId2, + }), + ]); + + const responsePageOne = await FillTable.findAll({ + page: 1, + limit: 1, + }, [], { + orderBy: [[FillColumns.eventId, Ordering.DESC]], + }); + + expect(responsePageOne.results.length).toEqual(1); + expect(responsePageOne.results[0]).toEqual(expect.objectContaining({ + ...defaultFill, + eventId: defaultTendermintEventId2, + })); + expect(responsePageOne.offset).toEqual(0); + expect(responsePageOne.total).toEqual(2); + + const responsePageTwo = await FillTable.findAll({ + page: 2, + limit: 1, + }, [], { + orderBy: [[FillColumns.eventId, Ordering.DESC]], + }); + + expect(responsePageTwo.results.length).toEqual(1); + expect(responsePageTwo.results[0]).toEqual(expect.objectContaining(defaultFill)); + expect(responsePageTwo.offset).toEqual(1); + expect(responsePageTwo.total).toEqual(2); + + const responsePageAllPages = await FillTable.findAll({ + page: 1, + limit: 2, + }, [], { + orderBy: [[FillColumns.eventId, Ordering.DESC]], + }); + + expect(responsePageAllPages.results.length).toEqual(2); + expect(responsePageAllPages.results[0]).toEqual(expect.objectContaining({ + ...defaultFill, + eventId: defaultTendermintEventId2, + })); + expect(responsePageAllPages.results[1]).toEqual(expect.objectContaining(defaultFill)); + expect(responsePageAllPages.offset).toEqual(0); + expect(responsePageAllPages.total).toEqual(2); + }); + it('Successfully finds Fill with eventId', async () => { await Promise.all([ FillTable.create(defaultFill), @@ -112,7 +165,7 @@ describe('Fill store', () => { }), ]); - const fills: FillFromDatabase[] = await FillTable.findAll( + const { results: fills } = await FillTable.findAll( { eventId: defaultFill.eventId, }, @@ -134,7 +187,7 @@ describe('Fill store', () => { ) => { await FillTable.create(defaultFill); - const fills: FillFromDatabase[] = await FillTable.findAll( + const { results: fills } = await FillTable.findAll( { createdBeforeOrAt: createdDateTime.plus({ seconds: deltaSeconds }).toISO(), }, @@ -155,7 +208,7 @@ describe('Fill store', () => { ) => { await FillTable.create(defaultFill); - const fills: FillFromDatabase[] = await FillTable.findAll( + const { results: fills } = await FillTable.findAll( { createdBeforeOrAtHeight: Big(createdHeight).plus(deltaBlocks).toFixed(), }, @@ -177,7 +230,7 @@ describe('Fill store', () => { ) => { await FillTable.create(defaultFill); - const fills: FillFromDatabase[] = await FillTable.findAll( + const { results: fills } = await FillTable.findAll( { createdOnOrAfter: createdDateTime.minus({ seconds: deltaSeconds }).toISO(), }, @@ -199,7 +252,7 @@ describe('Fill store', () => { ) => { await FillTable.create(defaultFill); - const fills: FillFromDatabase[] = await FillTable.findAll( + const { results: fills } = await FillTable.findAll( { createdOnOrAfterHeight: Big(createdHeight).minus(deltaBlocks).toFixed(), }, diff --git a/indexer/packages/postgres/__tests__/stores/order-table.test.ts b/indexer/packages/postgres/__tests__/stores/order-table.test.ts index 1cd9e26a60..a5449405df 100644 --- a/indexer/packages/postgres/__tests__/stores/order-table.test.ts +++ b/indexer/packages/postgres/__tests__/stores/order-table.test.ts @@ -4,6 +4,7 @@ import { OrderFromDatabase, Ordering, OrderStatus, + PaginationFromDatabase, TimeInForce, } from '../../src/types'; import * as OrderTable from '../../src/stores/order-table'; @@ -49,7 +50,9 @@ describe('Order store', () => { it('Successfully creates an Order with goodTilBlockTime', async () => { await OrderTable.create(defaultOrderGoodTilBlockTime); - const orders: OrderFromDatabase[] = await OrderTable.findAll({}, [], {}); + const { + results: orders, + }: PaginationFromDatabase = await OrderTable.findAll({}, [], {}); expect(orders).toHaveLength(1); expect(orders[0]).toEqual(expect.objectContaining({ @@ -67,7 +70,9 @@ describe('Order store', () => { }), ]); - const orders: OrderFromDatabase[] = await OrderTable.findAll({}, [], { + const { + results: orders, + }: PaginationFromDatabase = await OrderTable.findAll({}, [], { orderBy: [[OrderColumns.clientId, Ordering.ASC]], }); @@ -79,6 +84,66 @@ describe('Order store', () => { })); }); + it('Successfully finds all Orders using pagination', async () => { + await Promise.all([ + OrderTable.create(defaultOrder), + OrderTable.create({ + ...defaultOrder, + clientId: '2', + }), + ]); + + const responsePageOne: PaginationFromDatabase = await OrderTable.findAll({ + page: 1, + limit: 1, + }, + [], + { + orderBy: [[OrderColumns.clientId, Ordering.ASC]], + }); + + expect(responsePageOne.results.length).toEqual(1); + expect(responsePageOne.results[0]).toEqual(expect.objectContaining(defaultOrder)); + expect(responsePageOne.offset).toEqual(0); + expect(responsePageOne.total).toEqual(2); + + const responsePageTwo: PaginationFromDatabase = await OrderTable.findAll({ + page: 2, + limit: 1, + }, + [], + { + orderBy: [[OrderColumns.clientId, Ordering.ASC]], + }); + + expect(responsePageTwo.results.length).toEqual(1); + expect(responsePageTwo.results[0]).toEqual(expect.objectContaining({ + ...defaultOrder, + clientId: '2', + })); + expect(responsePageTwo.offset).toEqual(1); + expect(responsePageTwo.total).toEqual(2); + + const responsePageAllPages: PaginationFromDatabase = await OrderTable + .findAll({ + page: 1, + limit: 2, + }, + [], + { + orderBy: [[OrderColumns.clientId, Ordering.ASC]], + }); + + expect(responsePageAllPages.results.length).toEqual(2); + expect(responsePageAllPages.results[0]).toEqual(expect.objectContaining(defaultOrder)); + expect(responsePageAllPages.results[1]).toEqual(expect.objectContaining({ + ...defaultOrder, + clientId: '2', + })); + expect(responsePageAllPages.offset).toEqual(0); + expect(responsePageAllPages.total).toEqual(2); + }); + it('findOpenLongTermOrConditionalOrders', async () => { await Promise.all([ OrderTable.create(defaultOrder), @@ -106,7 +171,7 @@ describe('Order store', () => { }), ]); - const orders: OrderFromDatabase[] = await OrderTable.findAll( + const { results: orders }: PaginationFromDatabase = await OrderTable.findAll( { clientId: '1', }, @@ -202,11 +267,12 @@ describe('Order store', () => { OrderTable.create(defaultOrderGoodTilBlockTime), ]); - const orders: OrderFromDatabase[] = await OrderTable.findAll( - filter, - [], - { readReplica: true }, - ); + const { results: orders }: PaginationFromDatabase = await OrderTable + .findAll( + filter, + [], + { readReplica: true }, + ); expect(orders).toHaveLength(1); expect(orders[0]).toEqual(expect.objectContaining(expectedOrder)); diff --git a/indexer/packages/postgres/__tests__/stores/transfer-table.test.ts b/indexer/packages/postgres/__tests__/stores/transfer-table.test.ts index a2c52739ce..c3293b12d6 100644 --- a/indexer/packages/postgres/__tests__/stores/transfer-table.test.ts +++ b/indexer/packages/postgres/__tests__/stores/transfer-table.test.ts @@ -115,7 +115,7 @@ describe('Transfer store', () => { TransferTable.create(transfer2), ]); - const transfers: TransferFromDatabase[] = await TransferTable.findAllToOrFromSubaccountId( + const { results: transfers } = await TransferTable.findAllToOrFromSubaccountId( { subaccountId: [defaultSubaccountId] }, [], { orderBy: [[TransferColumns.id, Ordering.ASC]], @@ -142,7 +142,7 @@ describe('Transfer store', () => { TransferTable.create(transfer2), ]); - const transfers: TransferFromDatabase[] = await TransferTable.findAllToOrFromSubaccountId( + const { results: transfers } = await TransferTable.findAllToOrFromSubaccountId( { subaccountId: [defaultSubaccountId], eventId: [defaultTendermintEventId], @@ -155,6 +155,57 @@ describe('Transfer store', () => { expect(transfers[0]).toEqual(expect.objectContaining(defaultTransfer)); }); + it('Successfully finds all transfers to and from subaccount using pagination', async () => { + const transfer2: TransferCreateObject = { + senderSubaccountId: defaultSubaccountId2, + recipientSubaccountId: defaultSubaccountId, + assetId: defaultAsset2.id, + size: '5', + eventId: defaultTendermintEventId2, + transactionHash: '', // TODO: Add a real transaction Hash + createdAt: createdDateTime.toISO(), + createdAtHeight: createdHeight, + }; + await Promise.all([ + TransferTable.create(defaultTransfer), + TransferTable.create(transfer2), + ]); + + const responsePageOne = await TransferTable.findAllToOrFromSubaccountId( + { subaccountId: [defaultSubaccountId], page: 1, limit: 1 }, + [], { + orderBy: [[TransferColumns.id, Ordering.ASC]], + }); + + expect(responsePageOne.results.length).toEqual(1); + expect(responsePageOne.results[0]).toEqual(expect.objectContaining(defaultTransfer)); + expect(responsePageOne.offset).toEqual(0); + expect(responsePageOne.total).toEqual(2); + + const responsePageTwo = await TransferTable.findAllToOrFromSubaccountId( + { subaccountId: [defaultSubaccountId], page: 2, limit: 1 }, + [], { + orderBy: [[TransferColumns.id, Ordering.ASC]], + }); + + expect(responsePageTwo.results.length).toEqual(1); + expect(responsePageTwo.results[0]).toEqual(expect.objectContaining(transfer2)); + expect(responsePageTwo.offset).toEqual(1); + expect(responsePageTwo.total).toEqual(2); + + const responsePageAllPages = await TransferTable.findAllToOrFromSubaccountId( + { subaccountId: [defaultSubaccountId], page: 1, limit: 2 }, + [], { + orderBy: [[TransferColumns.id, Ordering.ASC]], + }); + + expect(responsePageAllPages.results.length).toEqual(2); + expect(responsePageAllPages.results[0]).toEqual(expect.objectContaining(defaultTransfer)); + expect(responsePageAllPages.results[1]).toEqual(expect.objectContaining(transfer2)); + expect(responsePageAllPages.offset).toEqual(0); + expect(responsePageAllPages.total).toEqual(2); + }); + it('Successfully finds Transfer with eventId', async () => { await Promise.all([ TransferTable.create(defaultTransfer), @@ -234,7 +285,7 @@ describe('Transfer store', () => { TransferTable.create(transfer2), ]); - const transfers: TransferFromDatabase[] = await TransferTable.findAllToOrFromSubaccountId( + const { results: transfers } = await TransferTable.findAllToOrFromSubaccountId( { subaccountId: [defaultSubaccountId], createdBeforeOrAt: '2000-05-25T00:00:00.000Z', diff --git a/indexer/packages/postgres/src/stores/fill-table.ts b/indexer/packages/postgres/src/stores/fill-table.ts index 186f645b6a..0ece392045 100644 --- a/indexer/packages/postgres/src/stores/fill-table.ts +++ b/indexer/packages/postgres/src/stores/fill-table.ts @@ -25,6 +25,7 @@ import { CostOfFills, QueryableField, QueryConfig, + PaginationFromDatabase, } from '../types'; export function uuid(eventId: Buffer, liquidity: Liquidity): string { @@ -49,10 +50,11 @@ export async function findAll( createdOnOrAfter, clientMetadata, fee, + page, }: FillQueryConfig, requiredFields: QueryableField[], options: Options = DEFAULT_POSTGRES_OPTIONS, -): Promise { +): Promise> { verifyAllRequiredFields( { limit, @@ -156,11 +158,41 @@ export async function findAll( Ordering.DESC, ); - if (limit !== undefined) { + if (limit !== undefined && page === undefined) { baseQuery = baseQuery.limit(limit); } - return baseQuery.returning('*'); + /** + * If a query is made using a page number, then the limit property is used as 'page limit' + * TODO: Improve pagination by adding a required eventId for orderBy clause + */ + if (page !== undefined && limit !== undefined) { + /** + * We make sure that the page number is always >= 1 + */ + const currentPage: number = Math.max(1, page); + const offset: number = (currentPage - 1) * limit; + + /** + * Ensure sorting is applied to maintain consistent pagination results. + * Also a casting of the ts type is required since the infer of the type + * obtained from the count is not performed. + */ + const count: { count?: string } = await baseQuery.clone().clearOrder().count({ count: '*' }).first() as unknown as { count?: string }; + + baseQuery = baseQuery.offset(offset).limit(limit); + + return { + results: await baseQuery.returning('*'), + limit, + offset, + total: parseInt(count.count ?? '0', 10), + }; + } + + return { + results: await baseQuery.returning('*'), + }; } export async function create( diff --git a/indexer/packages/postgres/src/stores/order-table.ts b/indexer/packages/postgres/src/stores/order-table.ts index df59c4e27c..a197dee1a9 100644 --- a/indexer/packages/postgres/src/stores/order-table.ts +++ b/indexer/packages/postgres/src/stores/order-table.ts @@ -15,6 +15,7 @@ import { OrderQueryConfig, OrderStatus, OrderUpdateObject, + PaginationFromDatabase, QueryableField, QueryConfig, } from '../types'; @@ -67,10 +68,11 @@ export async function findAll( goodTilBlockTimeBeforeOrAt, clientMetadata, triggerPrice, + page, }: OrderQueryConfig, requiredFields: QueryableField[], options: Options = DEFAULT_POSTGRES_OPTIONS, -): Promise { +): Promise> { verifyAllRequiredFields( { limit, @@ -185,11 +187,41 @@ export async function findAll( } } - if (limit !== undefined) { + if (limit !== undefined && page === undefined) { baseQuery = baseQuery.limit(limit); } - return baseQuery.returning('*'); + /** + * If a query is made using a page number, then the limit property is used as 'page limit' + * TODO: Improve pagination by adding a required eventId for orderBy clause + */ + if (page !== undefined && limit !== undefined) { + /** + * We make sure that the page number is always >= 1 + */ + const currentPage: number = Math.max(1, page); + const offset: number = (currentPage - 1) * limit; + + /** + * Ensure sorting is applied to maintain consistent pagination results. + * Also a casting of the ts type is required since the infer of the type + * obtained from the count is not performed. + */ + const count: { count?: string } = await baseQuery.clone().clearOrder().count({ count: '*' }).first() as unknown as { count?: string }; + + baseQuery = baseQuery.offset(offset).limit(limit); + + return { + results: await baseQuery.returning('*'), + limit, + offset, + total: parseInt(count.count ?? '0', 10), + }; + } + + return { + results: await baseQuery.returning('*'), + }; } export async function create( diff --git a/indexer/packages/postgres/src/stores/transfer-table.ts b/indexer/packages/postgres/src/stores/transfer-table.ts index 1a14bb174b..01611b9183 100644 --- a/indexer/packages/postgres/src/stores/transfer-table.ts +++ b/indexer/packages/postgres/src/stores/transfer-table.ts @@ -21,6 +21,7 @@ import { QueryableField, ToAndFromSubaccountTransferQueryConfig, SubaccountAssetNetTransferMap, + PaginationFromDatabase, } from '../types'; export function uuid( @@ -194,10 +195,11 @@ export async function findAllToOrFromSubaccountId( createdBeforeOrAt, createdAfterHeight, createdAfter, + page, }: ToAndFromSubaccountTransferQueryConfig, requiredFields: QueryableField[], options: Options = DEFAULT_POSTGRES_OPTIONS, -): Promise { +): Promise> { verifyAllRequiredFields( { limit, @@ -291,11 +293,41 @@ export async function findAllToOrFromSubaccountId( } } - if (limit !== undefined) { + if (limit !== undefined && page === undefined) { baseQuery = baseQuery.limit(limit); } - return baseQuery.returning('*'); + /** + * If a query is made using a page number, then the limit property is used as 'page limit' + * TODO: Improve pagination by adding a required eventId for orderBy clause + */ + if (page !== undefined && limit !== undefined) { + /** + * We make sure that the page number is always >= 1 + */ + const currentPage: number = Math.max(1, page); + const offset: number = (currentPage - 1) * limit; + + /** + * Ensure sorting is applied to maintain consistent pagination results. + * Also a casting of the ts type is required since the infer of the type + * obtained from the count is not performed. + */ + const count: { count?: string } = await baseQuery.clone().clearOrder().count({ count: '*' }).first() as unknown as { count?: string }; + + baseQuery = baseQuery.offset(offset).limit(limit); + + return { + results: await baseQuery.returning('*'), + limit, + offset, + total: parseInt(count.count ?? '0', 10), + }; + } + + return { + results: await baseQuery.returning('*'), + }; } function convertToSubaccountAssetMap( diff --git a/indexer/packages/postgres/src/types/index.ts b/indexer/packages/postgres/src/types/index.ts index 455c67d7ff..ca52d40443 100644 --- a/indexer/packages/postgres/src/types/index.ts +++ b/indexer/packages/postgres/src/types/index.ts @@ -25,4 +25,5 @@ export * from './compliance-data-types'; export * from './compliance-status-types'; export * from './trading-reward-types'; export * from './trading-reward-aggregation-types'; +export * from './pagination-types'; export { PositionSide } from './position-types'; diff --git a/indexer/packages/postgres/src/types/pagination-types.ts b/indexer/packages/postgres/src/types/pagination-types.ts new file mode 100644 index 0000000000..1566e2eef3 --- /dev/null +++ b/indexer/packages/postgres/src/types/pagination-types.ts @@ -0,0 +1,6 @@ +export interface PaginationFromDatabase { + results: T[]; + total?: number, + offset?: number, + limit?: number +} diff --git a/indexer/packages/postgres/src/types/query-types.ts b/indexer/packages/postgres/src/types/query-types.ts index b4e2541059..34cfc3b7dc 100644 --- a/indexer/packages/postgres/src/types/query-types.ts +++ b/indexer/packages/postgres/src/types/query-types.ts @@ -10,6 +10,7 @@ import { IsoString } from './utility-types'; export enum QueryableField { LIMIT = 'limit', + PAGE = 'page', ID = 'id', ADDRESS = 'address', SUBACCOUNT_NUMBER = 'subaccountNumber', @@ -88,6 +89,7 @@ export enum QueryableField { export interface QueryConfig { [QueryableField.LIMIT]?: number; + [QueryableField.PAGE]?: number; } export interface SubaccountQueryConfig extends QueryConfig { diff --git a/indexer/services/comlink/src/controllers/api/v4/fills-controller.ts b/indexer/services/comlink/src/controllers/api/v4/fills-controller.ts index f9ae639b92..0df1441c11 100644 --- a/indexer/services/comlink/src/controllers/api/v4/fills-controller.ts +++ b/indexer/services/comlink/src/controllers/api/v4/fills-controller.ts @@ -68,7 +68,7 @@ class FillsController extends Controller { } const subaccountId: string = SubaccountTable.uuid(address, subaccountNumber); - const fills: FillFromDatabase[] = await FillTable.findAll( + const { results: fills } = await FillTable.findAll( { subaccountId: [subaccountId], clobPairId, @@ -132,7 +132,7 @@ class FillsController extends Controller { ); const subaccountIds: string[] = Object.keys(childIdtoSubaccountNumber); - const fills: FillFromDatabase[] = await FillTable.findAll( + const { results: fills } = await FillTable.findAll( { subaccountId: subaccountIds, clobPairId, diff --git a/indexer/services/comlink/src/controllers/api/v4/orders-controller.ts b/indexer/services/comlink/src/controllers/api/v4/orders-controller.ts index e8833af121..90be17657f 100644 --- a/indexer/services/comlink/src/controllers/api/v4/orders-controller.ts +++ b/indexer/services/comlink/src/controllers/api/v4/orders-controller.ts @@ -12,6 +12,7 @@ import { OrderStatus, OrderTable, OrderType, + PaginationFromDatabase, perpetualMarketRefresher, protocolTranslations, SubaccountTable, @@ -112,10 +113,10 @@ async function listOrdersCommon( : Ordering.DESC; const [ redisOrderMap, - postgresOrders, + { results: postgresOrders }, ]: [ RedisOrderMap, - OrderFromDatabase[], + PaginationFromDatabase, ] = await Promise.all([ getRedisOrderMapForSubaccountIds( subaccountIds, @@ -157,11 +158,13 @@ async function listOrdersCommon( // then we do not want to return this order to the user as 'BEST_EFFORT_OPENED'. let additionalPostgresOrders: OrderFromDatabase[] = []; if (!_.isEmpty(postgresOrderIdsToFetch)) { - additionalPostgresOrders = await OrderTable.findAll({ + const { results }: PaginationFromDatabase = await OrderTable.findAll({ id: postgresOrderIdsToFetch, }, [], { ...DEFAULT_POSTGRES_OPTIONS, }); + + additionalPostgresOrders = results; } const postgresOrderMap: PostgresOrderMap = _.keyBy( diff --git a/indexer/services/comlink/src/controllers/api/v4/trades-controller.ts b/indexer/services/comlink/src/controllers/api/v4/trades-controller.ts index e94e514c56..f9059b69f4 100644 --- a/indexer/services/comlink/src/controllers/api/v4/trades-controller.ts +++ b/indexer/services/comlink/src/controllers/api/v4/trades-controller.ts @@ -53,7 +53,7 @@ class TradesController extends Controller { throw new NotFoundError(`${ticker} not found in tickers of type ${MarketType.PERPETUAL}`); } - const fills: FillFromDatabase[] = await FillTable.findAll( + const { results: fills } = await FillTable.findAll( { clobPairId, liquidity: Liquidity.TAKER, diff --git a/indexer/services/comlink/src/controllers/api/v4/transfers-controller.ts b/indexer/services/comlink/src/controllers/api/v4/transfers-controller.ts index 1ba50af669..354e02ba3e 100644 --- a/indexer/services/comlink/src/controllers/api/v4/transfers-controller.ts +++ b/indexer/services/comlink/src/controllers/api/v4/transfers-controller.ts @@ -6,6 +6,7 @@ import { DEFAULT_POSTGRES_OPTIONS, IsoString, Ordering, + PaginationFromDatabase, QueryableField, SubaccountColumns, SubaccountFromDatabase, @@ -64,12 +65,7 @@ class TransfersController extends Controller { const subaccountId: string = SubaccountTable.uuid(address, subaccountNumber); // TODO(DEC-656): Change to a cache in Redis similar to Librarian instead of querying DB. - const [subaccount, transfers, assets]: [ - SubaccountFromDatabase | undefined, - TransferFromDatabase[], - AssetFromDatabase[] - ] = await - Promise.all([ + const [subaccount, { results: transfers }, assets] = await Promise.all([ SubaccountTable.findById( subaccountId, ), @@ -151,9 +147,9 @@ class TransfersController extends Controller { ); // TODO(DEC-656): Change to a cache in Redis similar to Librarian instead of querying DB. - const [subaccounts, transfers, assets]: [ + const [subaccounts, { results: transfers }, assets]: [ SubaccountFromDatabase[] | undefined, - TransferFromDatabase[], + PaginationFromDatabase, AssetFromDatabase[] ] = await Promise.all([ diff --git a/indexer/services/ender/__tests__/handlers/transfer-handler.test.ts b/indexer/services/ender/__tests__/handlers/transfer-handler.test.ts index 040a261d8f..1b6370dd65 100644 --- a/indexer/services/ender/__tests__/handlers/transfer-handler.test.ts +++ b/indexer/services/ender/__tests__/handlers/transfer-handler.test.ts @@ -545,7 +545,7 @@ async function expectNoExistingTransfers( subaccountIds: string[], ) { // Confirm there is no existing transfer to or from the subaccounts - const transfers: TransferFromDatabase[] = await TransferTable.findAllToOrFromSubaccountId( + const { results: transfers } = await TransferTable.findAllToOrFromSubaccountId( { subaccountId: subaccountIds, }, @@ -572,7 +572,7 @@ async function expectAndReturnNewTransfer( ): Promise { // Confirm there is now a transfer to or from the recipient subaccount if (recipientSubaccountId) { - const newTransfersRelatedToRecipient: TransferFromDatabase[] = await + const { results: newTransfersRelatedToRecipient } = await TransferTable.findAllToOrFromSubaccountId( { subaccountId: [ @@ -588,7 +588,7 @@ async function expectAndReturnNewTransfer( if (senderSubaccountId) { // Confirm there is now a transfer to or from the sender subaccount - const newTransfersRelatedToSender: TransferFromDatabase[] = await + const { results: newTransfersRelatedToSender } = await TransferTable.findAllToOrFromSubaccountId( { subaccountId: [ diff --git a/indexer/services/roundtable/__tests__/tasks/cancel-stale-orders.test.ts b/indexer/services/roundtable/__tests__/tasks/cancel-stale-orders.test.ts index 2628ab4150..77b1ff96b0 100644 --- a/indexer/services/roundtable/__tests__/tasks/cancel-stale-orders.test.ts +++ b/indexer/services/roundtable/__tests__/tasks/cancel-stale-orders.test.ts @@ -1,5 +1,12 @@ import { - BlockTable, OrderFromDatabase, OrderTable, OrderStatus, dbHelpers, testConstants, testMocks, + BlockTable, + OrderFromDatabase, + OrderTable, + OrderStatus, + dbHelpers, + testConstants, + testMocks, + PaginationFromDatabase, } from '@dydxprotocol-indexer/postgres'; import cancelStaleOrdersTask from '../../src/tasks/cancel-stale-orders'; import { defaultOrderGoodTilBlockTime } from '@dydxprotocol-indexer/postgres/build/__tests__/helpers/constants'; @@ -83,13 +90,14 @@ describe('cancel-stale-orders', () => { await cancelStaleOrdersTask(); - const ordersAfterTask: OrderFromDatabase[] = await OrderTable.findAll( - { - id: createdOrderIds, - }, - [], - {}, - ); + const { results: ordersAfterTask }: PaginationFromDatabase = await OrderTable + .findAll( + { + id: createdOrderIds, + }, + [], + {}, + ); expect(_.sortBy(ordersAfterTask, ['id'])).toEqual(_.sortBy(expectedOrders, ['id'])); expect(stats.gauge).toHaveBeenCalledWith('roundtable.num_stale_orders.count', 2); expect(stats.gauge).toHaveBeenCalledWith('roundtable.num_stale_orders_canceled.count', 2); @@ -145,13 +153,14 @@ describe('cancel-stale-orders', () => { expect(stats.gauge).toHaveBeenCalledWith('roundtable.num_stale_orders.count', 1); expect(stats.gauge).toHaveBeenCalledWith('roundtable.num_stale_orders_canceled.count', 1); - const ordersAfterTask: OrderFromDatabase[] = await OrderTable.findAll( - { - id: createdOrderIds, - }, - [], - {}, - ); + const { results: ordersAfterTask }: PaginationFromDatabase = await OrderTable + .findAll( + { + id: createdOrderIds, + }, + [], + {}, + ); expect(_.sortBy(ordersAfterTask, ['id'])).toEqual(_.sortBy(expectedOrders, ['id'])); config.CANCEL_STALE_ORDERS_QUERY_BATCH_SIZE = oldLimit; diff --git a/indexer/services/roundtable/src/tasks/cancel-stale-orders.ts b/indexer/services/roundtable/src/tasks/cancel-stale-orders.ts index 43f315c21a..505202a6ee 100644 --- a/indexer/services/roundtable/src/tasks/cancel-stale-orders.ts +++ b/indexer/services/roundtable/src/tasks/cancel-stale-orders.ts @@ -7,6 +7,7 @@ import { OrderFromDatabase, OrderStatus, OrderTable, + PaginationFromDatabase, } from '@dydxprotocol-indexer/postgres'; import { ORDER_FLAG_SHORT_TERM } from '@dydxprotocol-indexer/v4-proto-parser'; @@ -22,19 +23,20 @@ export default async function runTask(): Promise { const latestBlockHeight: number = parseInt(latestBlock.blockHeight, 10); - const staleOpenOrders: OrderFromDatabase[] = await OrderTable.findAll( - { - statuses: [OrderStatus.OPEN], - orderFlags: ORDER_FLAG_SHORT_TERM.toString(), - // goodTilBlock needs to be < latest block height to be guaranteed to be CANCELED - goodTilBlockBeforeOrAt: (latestBlockHeight - 1).toString(), - limit: config.CANCEL_STALE_ORDERS_QUERY_BATCH_SIZE, - }, - [], - { - readReplica: true, - }, - ); + const { results: staleOpenOrders }: PaginationFromDatabase = await OrderTable + .findAll( + { + statuses: [OrderStatus.OPEN], + orderFlags: ORDER_FLAG_SHORT_TERM.toString(), + // goodTilBlock needs to be < latest block height to be guaranteed to be CANCELED + goodTilBlockBeforeOrAt: (latestBlockHeight - 1).toString(), + limit: config.CANCEL_STALE_ORDERS_QUERY_BATCH_SIZE, + }, + [], + { + readReplica: true, + }, + ); stats.timing(`${config.SERVICE_NAME}.cancel_stale_orders.query.timing`, Date.now() - queryStart); const updateStart: number = Date.now();