From e6eea6f7a4917934221996490e1744921c46fd08 Mon Sep 17 00:00:00 2001
From: King-Hin Leung <kinghin.leung@commercetools.com>
Date: Mon, 14 Aug 2023 10:20:29 +0200
Subject: [PATCH] Update event.controller.js

---
 .../src/controllers/event.controller.js       | 60 +++++++++++++------
 1 file changed, 41 insertions(+), 19 deletions(-)

diff --git a/full-ingestion/src/controllers/event.controller.js b/full-ingestion/src/controllers/event.controller.js
index 88024e0..daa8de8 100644
--- a/full-ingestion/src/controllers/event.controller.js
+++ b/full-ingestion/src/controllers/event.controller.js
@@ -3,6 +3,8 @@ import { createApiRoot } from '../clients/create.client.js';
 import CustomError from '../errors/custom.error.js';
 import { default as saveProducts } from '../extensions/algolia-example/clients/client.js';
 
+const CHUNK_SIZE = 100;
+
 async function syncProducts(storeKey) {
   const products = await getProductsByStore(storeKey);
 
@@ -12,25 +14,45 @@ async function syncProducts(storeKey) {
 }
 
 async function getProductsByStore(storeKey) {
-  return await createApiRoot()
-    .inStoreKeyWithStoreKeyValue({ storeKey: Buffer.from(storeKey).toString() })
-    .productSelectionAssignments()
-    .get({
-      queryArgs: {
-        expand: [
-          'product',
-          'product.productType',
-          'product.taxCategory',
-          'product.masterData.current.categories[*]',
-        ],
-      },
-    })
-    .execute()
-    .then((response) => response.body.results)
-    .then((results) => results.map((result) => result.product))
-    .catch((error) => {
-      throw new CustomError(400, `Bad request: ${error.message}`, error);
-    });
+  let lastProductId = undefined;
+  let hasNextQuery = true;
+  let allProducts = [];
+
+  while (hasNextQuery) {
+    let queryArgs = {
+      limit: CHUNK_SIZE,
+      withTotal: false,
+      sort: 'product.id asc',
+      expand: [
+        'product',
+        'product.productType',
+        'product.taxCategory',
+        'product.masterData.current.categories[*]',
+      ],
+    };
+    if (lastProductId) {
+      queryArgs.where = `product(id>"${lastProductId}")`;
+    }
+
+    let productChunk = await createApiRoot()
+      .inStoreKeyWithStoreKeyValue({
+        storeKey: Buffer.from(storeKey).toString(),
+      })
+      .productSelectionAssignments()
+      .get({ queryArgs })
+      .execute()
+      .then((response) => response.body.results)
+      .then((results) => results.map((result) => result.product))
+      .catch((error) => {
+        throw new CustomError(400, `Bad request: ${error.message}`, error);
+      });
+    hasNextQuery = productChunk.length == CHUNK_SIZE;
+    if (productChunk.length > 0) {
+      lastProductId = productChunk[productChunk.length - 1].id;
+      allProducts = allProducts.concat(productChunk);
+    }
+  }
+  return allProducts;
 }
 
 export const eventHandler = async (request, response) => {