Skip to content

Commit

Permalink
Add support for pagination at table level. (#1244) (#1538)
Browse files Browse the repository at this point in the history 
* feat(postgres): ✨ add transfer pagination support

* test(postgres): ✅ add transfer pagination test

* feat(comlink): ✨ add pagination for getTransfers

* test(ender): ✅ update transfer handler test

* feat(postgres): ✨ add fill pagination support

* test(postgres): ✅ add fill pagination test

* feat(comlink): ✨ add pagination for getTrades and getFills

* test: ✅ update pagination tests

updated pagination tests to also evaluate total pages.

* feat(postgres): 🏷️ add explicit types

added explicit types for fill and transfer tables.

* feat(postgres): ✨ add order pagination support

* test(postgres): ✅ add order pagination test

* feat: ♻️ update order findAll usage

* revert: ⏪ revert add pagination for getTransfers

* revert: ⏪ revert add pagination for getTrades and getFills

* feat: 🏷️ add explicit ts types

* docs: 💡 add pagination todo comment

* docs(postgres): 💡 improve pagination comment

improve pagination comment thanks to coderabbitai.

---------

Co-authored-by: Davide Segullo <[email protected]>
(cherry picked from commit e12ebf2)

Co-authored-by: Giorgio Nocera <[email protected]>
  • Loading branch information
@mergify @giorgionocera
mergify[bot] and giorgionocera authored May 17, 2024
1 parent c26b459 commit 07a2b3d
Show file tree
Hide file tree
Showing 16 changed files with 357 additions and 72 deletions.
67 changes: 60 additions & 7 deletions indexer/packages/postgres/__tests__/stores/fill-table.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ describe('Fill store', () => {
FillTable.create(defaultFill),
]);

const fills: FillFromDatabase[] = await FillTable.findAll({}, [], {});
const { results: fills } = await FillTable.findAll({}, [], {});

expect(fills.length).toEqual(2);
expect(fills[0]).toEqual(expect.objectContaining(defaultFill));
Expand All @@ -91,7 +91,7 @@ describe('Fill store', () => {
}),
]);

const fills: FillFromDatabase[] = await FillTable.findAll({}, [], {
const { results: fills } = await FillTable.findAll({}, [], {
orderBy: [[FillColumns.eventId, Ordering.DESC]],
});

Expand All @@ -103,6 +103,59 @@ describe('Fill store', () => {
expect(fills[1]).toEqual(expect.objectContaining(defaultFill));
});

it('Successfully finds fills using pagination', async () => {
await Promise.all([
FillTable.create(defaultFill),
FillTable.create({
...defaultFill,
eventId: defaultTendermintEventId2,
}),
]);

const responsePageOne = await FillTable.findAll({
page: 1,
limit: 1,
}, [], {
orderBy: [[FillColumns.eventId, Ordering.DESC]],
});

expect(responsePageOne.results.length).toEqual(1);
expect(responsePageOne.results[0]).toEqual(expect.objectContaining({
...defaultFill,
eventId: defaultTendermintEventId2,
}));
expect(responsePageOne.offset).toEqual(0);
expect(responsePageOne.total).toEqual(2);

const responsePageTwo = await FillTable.findAll({
page: 2,
limit: 1,
}, [], {
orderBy: [[FillColumns.eventId, Ordering.DESC]],
});

expect(responsePageTwo.results.length).toEqual(1);
expect(responsePageTwo.results[0]).toEqual(expect.objectContaining(defaultFill));
expect(responsePageTwo.offset).toEqual(1);
expect(responsePageTwo.total).toEqual(2);

const responsePageAllPages = await FillTable.findAll({
page: 1,
limit: 2,
}, [], {
orderBy: [[FillColumns.eventId, Ordering.DESC]],
});

expect(responsePageAllPages.results.length).toEqual(2);
expect(responsePageAllPages.results[0]).toEqual(expect.objectContaining({
...defaultFill,
eventId: defaultTendermintEventId2,
}));
expect(responsePageAllPages.results[1]).toEqual(expect.objectContaining(defaultFill));
expect(responsePageAllPages.offset).toEqual(0);
expect(responsePageAllPages.total).toEqual(2);
});

it('Successfully finds Fill with eventId', async () => {
await Promise.all([
FillTable.create(defaultFill),
Expand All @@ -112,7 +165,7 @@ describe('Fill store', () => {
}),
]);

const fills: FillFromDatabase[] = await FillTable.findAll(
const { results: fills } = await FillTable.findAll(
{
eventId: defaultFill.eventId,
},
Expand All @@ -134,7 +187,7 @@ describe('Fill store', () => {
) => {
await FillTable.create(defaultFill);

const fills: FillFromDatabase[] = await FillTable.findAll(
const { results: fills } = await FillTable.findAll(
{
createdBeforeOrAt: createdDateTime.plus({ seconds: deltaSeconds }).toISO(),
},
Expand All @@ -155,7 +208,7 @@ describe('Fill store', () => {
) => {
await FillTable.create(defaultFill);

const fills: FillFromDatabase[] = await FillTable.findAll(
const { results: fills } = await FillTable.findAll(
{
createdBeforeOrAtHeight: Big(createdHeight).plus(deltaBlocks).toFixed(),
},
Expand All @@ -177,7 +230,7 @@ describe('Fill store', () => {
) => {
await FillTable.create(defaultFill);

const fills: FillFromDatabase[] = await FillTable.findAll(
const { results: fills } = await FillTable.findAll(
{
createdOnOrAfter: createdDateTime.minus({ seconds: deltaSeconds }).toISO(),
},
Expand All @@ -199,7 +252,7 @@ describe('Fill store', () => {
) => {
await FillTable.create(defaultFill);

const fills: FillFromDatabase[] = await FillTable.findAll(
const { results: fills } = await FillTable.findAll(
{
createdOnOrAfterHeight: Big(createdHeight).minus(deltaBlocks).toFixed(),
},
Expand Down
82 changes: 74 additions & 8 deletions indexer/packages/postgres/__tests__/stores/order-table.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import {
OrderFromDatabase,
Ordering,
OrderStatus,
PaginationFromDatabase,
TimeInForce,
} from '../../src/types';
import * as OrderTable from '../../src/stores/order-table';
Expand Down Expand Up @@ -49,7 +50,9 @@ describe('Order store', () => {
it('Successfully creates an Order with goodTilBlockTime', async () => {
await OrderTable.create(defaultOrderGoodTilBlockTime);

const orders: OrderFromDatabase[] = await OrderTable.findAll({}, [], {});
const {
results: orders,
}: PaginationFromDatabase<OrderFromDatabase> = await OrderTable.findAll({}, [], {});

expect(orders).toHaveLength(1);
expect(orders[0]).toEqual(expect.objectContaining({
Expand All @@ -67,7 +70,9 @@ describe('Order store', () => {
}),
]);

const orders: OrderFromDatabase[] = await OrderTable.findAll({}, [], {
const {
results: orders,
}: PaginationFromDatabase<OrderFromDatabase> = await OrderTable.findAll({}, [], {
orderBy: [[OrderColumns.clientId, Ordering.ASC]],
});

Expand All @@ -79,6 +84,66 @@ describe('Order store', () => {
}));
});

it('Successfully finds all Orders using pagination', async () => {
await Promise.all([
OrderTable.create(defaultOrder),
OrderTable.create({
...defaultOrder,
clientId: '2',
}),
]);

const responsePageOne: PaginationFromDatabase<OrderFromDatabase> = await OrderTable.findAll({
page: 1,
limit: 1,
},
[],
{
orderBy: [[OrderColumns.clientId, Ordering.ASC]],
});

expect(responsePageOne.results.length).toEqual(1);
expect(responsePageOne.results[0]).toEqual(expect.objectContaining(defaultOrder));
expect(responsePageOne.offset).toEqual(0);
expect(responsePageOne.total).toEqual(2);

const responsePageTwo: PaginationFromDatabase<OrderFromDatabase> = await OrderTable.findAll({
page: 2,
limit: 1,
},
[],
{
orderBy: [[OrderColumns.clientId, Ordering.ASC]],
});

expect(responsePageTwo.results.length).toEqual(1);
expect(responsePageTwo.results[0]).toEqual(expect.objectContaining({
...defaultOrder,
clientId: '2',
}));
expect(responsePageTwo.offset).toEqual(1);
expect(responsePageTwo.total).toEqual(2);

const responsePageAllPages: PaginationFromDatabase<OrderFromDatabase> = await OrderTable
.findAll({
page: 1,
limit: 2,
},
[],
{
orderBy: [[OrderColumns.clientId, Ordering.ASC]],
});

expect(responsePageAllPages.results.length).toEqual(2);
expect(responsePageAllPages.results[0]).toEqual(expect.objectContaining(defaultOrder));
expect(responsePageAllPages.results[1]).toEqual(expect.objectContaining({
...defaultOrder,
clientId: '2',
}));
expect(responsePageAllPages.offset).toEqual(0);
expect(responsePageAllPages.total).toEqual(2);
});

it('findOpenLongTermOrConditionalOrders', async () => {
await Promise.all([
OrderTable.create(defaultOrder),
Expand Down Expand Up @@ -106,7 +171,7 @@ describe('Order store', () => {
}),
]);

const orders: OrderFromDatabase[] = await OrderTable.findAll(
const { results: orders }: PaginationFromDatabase<OrderFromDatabase> = await OrderTable.findAll(
{
clientId: '1',
},
Expand Down Expand Up @@ -202,11 +267,12 @@ describe('Order store', () => {
OrderTable.create(defaultOrderGoodTilBlockTime),
]);

const orders: OrderFromDatabase[] = await OrderTable.findAll(
filter,
[],
{ readReplica: true },
);
const { results: orders }: PaginationFromDatabase<OrderFromDatabase> = await OrderTable
.findAll(
filter,
[],
{ readReplica: true },
);

expect(orders).toHaveLength(1);
expect(orders[0]).toEqual(expect.objectContaining(expectedOrder));
Expand Down
57 changes: 54 additions & 3 deletions indexer/packages/postgres/__tests__/stores/transfer-table.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@ describe('Transfer store', () => {
TransferTable.create(transfer2),
]);

const transfers: TransferFromDatabase[] = await TransferTable.findAllToOrFromSubaccountId(
const { results: transfers } = await TransferTable.findAllToOrFromSubaccountId(
{ subaccountId: [defaultSubaccountId] },
[], {
orderBy: [[TransferColumns.id, Ordering.ASC]],
Expand All @@ -142,7 +142,7 @@ describe('Transfer store', () => {
TransferTable.create(transfer2),
]);

const transfers: TransferFromDatabase[] = await TransferTable.findAllToOrFromSubaccountId(
const { results: transfers } = await TransferTable.findAllToOrFromSubaccountId(
{
subaccountId: [defaultSubaccountId],
eventId: [defaultTendermintEventId],
Expand All @@ -155,6 +155,57 @@ describe('Transfer store', () => {
expect(transfers[0]).toEqual(expect.objectContaining(defaultTransfer));
});

it('Successfully finds all transfers to and from subaccount using pagination', async () => {
const transfer2: TransferCreateObject = {
senderSubaccountId: defaultSubaccountId2,
recipientSubaccountId: defaultSubaccountId,
assetId: defaultAsset2.id,
size: '5',
eventId: defaultTendermintEventId2,
transactionHash: '', // TODO: Add a real transaction Hash
createdAt: createdDateTime.toISO(),
createdAtHeight: createdHeight,
};
await Promise.all([
TransferTable.create(defaultTransfer),
TransferTable.create(transfer2),
]);

const responsePageOne = await TransferTable.findAllToOrFromSubaccountId(
{ subaccountId: [defaultSubaccountId], page: 1, limit: 1 },
[], {
orderBy: [[TransferColumns.id, Ordering.ASC]],
});

expect(responsePageOne.results.length).toEqual(1);
expect(responsePageOne.results[0]).toEqual(expect.objectContaining(defaultTransfer));
expect(responsePageOne.offset).toEqual(0);
expect(responsePageOne.total).toEqual(2);

const responsePageTwo = await TransferTable.findAllToOrFromSubaccountId(
{ subaccountId: [defaultSubaccountId], page: 2, limit: 1 },
[], {
orderBy: [[TransferColumns.id, Ordering.ASC]],
});

expect(responsePageTwo.results.length).toEqual(1);
expect(responsePageTwo.results[0]).toEqual(expect.objectContaining(transfer2));
expect(responsePageTwo.offset).toEqual(1);
expect(responsePageTwo.total).toEqual(2);

const responsePageAllPages = await TransferTable.findAllToOrFromSubaccountId(
{ subaccountId: [defaultSubaccountId], page: 1, limit: 2 },
[], {
orderBy: [[TransferColumns.id, Ordering.ASC]],
});

expect(responsePageAllPages.results.length).toEqual(2);
expect(responsePageAllPages.results[0]).toEqual(expect.objectContaining(defaultTransfer));
expect(responsePageAllPages.results[1]).toEqual(expect.objectContaining(transfer2));
expect(responsePageAllPages.offset).toEqual(0);
expect(responsePageAllPages.total).toEqual(2);
});

it('Successfully finds Transfer with eventId', async () => {
await Promise.all([
TransferTable.create(defaultTransfer),
Expand Down Expand Up @@ -234,7 +285,7 @@ describe('Transfer store', () => {
TransferTable.create(transfer2),
]);

const transfers: TransferFromDatabase[] = await TransferTable.findAllToOrFromSubaccountId(
const { results: transfers } = await TransferTable.findAllToOrFromSubaccountId(
{
subaccountId: [defaultSubaccountId],
createdBeforeOrAt: '2000-05-25T00:00:00.000Z',
Expand Down
38 changes: 35 additions & 3 deletions indexer/packages/postgres/src/stores/fill-table.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import {
CostOfFills,
QueryableField,
QueryConfig,
PaginationFromDatabase,
} from '../types';

export function uuid(eventId: Buffer, liquidity: Liquidity): string {
Expand All @@ -49,10 +50,11 @@ export async function findAll(
createdOnOrAfter,
clientMetadata,
fee,
page,
}: FillQueryConfig,
requiredFields: QueryableField[],
options: Options = DEFAULT_POSTGRES_OPTIONS,
): Promise<FillFromDatabase[]> {
): Promise<PaginationFromDatabase<FillFromDatabase>> {
verifyAllRequiredFields(
{
limit,
Expand Down Expand Up @@ -156,11 +158,41 @@ export async function findAll(
Ordering.DESC,
);

if (limit !== undefined) {
if (limit !== undefined && page === undefined) {
baseQuery = baseQuery.limit(limit);
}

return baseQuery.returning('*');
/**
* If a query is made using a page number, then the limit property is used as 'page limit'
* TODO: Improve pagination by adding a required eventId for orderBy clause
*/
if (page !== undefined && limit !== undefined) {
/**
* We make sure that the page number is always >= 1
*/
const currentPage: number = Math.max(1, page);
const offset: number = (currentPage - 1) * limit;

/**
* Ensure sorting is applied to maintain consistent pagination results.
* Also a casting of the ts type is required since the infer of the type
* obtained from the count is not performed.
*/
const count: { count?: string } = await baseQuery.clone().clearOrder().count({ count: '*' }).first() as unknown as { count?: string };

baseQuery = baseQuery.offset(offset).limit(limit);

return {
results: await baseQuery.returning('*'),
limit,
offset,
total: parseInt(count.count ?? '0', 10),
};
}

return {
results: await baseQuery.returning('*'),
};
}

export async function create(
Expand Down
Loading

0 comments on commit 07a2b3d

Please sign in to comment.