From e6eea6f7a4917934221996490e1744921c46fd08 Mon Sep 17 00:00:00 2001 From: King-Hin Leung Date: Mon, 14 Aug 2023 10:20:29 +0200 Subject: [PATCH 1/4] Update event.controller.js --- .../src/controllers/event.controller.js | 60 +++++++++++++------ 1 file changed, 41 insertions(+), 19 deletions(-) diff --git a/full-ingestion/src/controllers/event.controller.js b/full-ingestion/src/controllers/event.controller.js index 88024e0..daa8de8 100644 --- a/full-ingestion/src/controllers/event.controller.js +++ b/full-ingestion/src/controllers/event.controller.js @@ -3,6 +3,8 @@ import { createApiRoot } from '../clients/create.client.js'; import CustomError from '../errors/custom.error.js'; import { default as saveProducts } from '../extensions/algolia-example/clients/client.js'; +const CHUNK_SIZE = 100; + async function syncProducts(storeKey) { const products = await getProductsByStore(storeKey); @@ -12,25 +14,45 @@ async function syncProducts(storeKey) { } async function getProductsByStore(storeKey) { - return await createApiRoot() - .inStoreKeyWithStoreKeyValue({ storeKey: Buffer.from(storeKey).toString() }) - .productSelectionAssignments() - .get({ - queryArgs: { - expand: [ - 'product', - 'product.productType', - 'product.taxCategory', - 'product.masterData.current.categories[*]', - ], - }, - }) - .execute() - .then((response) => response.body.results) - .then((results) => results.map((result) => result.product)) - .catch((error) => { - throw new CustomError(400, `Bad request: ${error.message}`, error); - }); + let lastProductId = undefined; + let hasNextQuery = true; + let allProducts = []; + + while (hasNextQuery) { + let queryArgs = { + limit: CHUNK_SIZE, + withTotal: false, + sort: 'product.id asc', + expand: [ + 'product', + 'product.productType', + 'product.taxCategory', + 'product.masterData.current.categories[*]', + ], + }; + if (lastProductId) { + queryArgs.where = `product(id>"${lastProductId}")`; + } + + let productChunk = await createApiRoot() + .inStoreKeyWithStoreKeyValue({ + storeKey: Buffer.from(storeKey).toString(), + }) + .productSelectionAssignments() + .get({ queryArgs }) + .execute() + .then((response) => response.body.results) + .then((results) => results.map((result) => result.product)) + .catch((error) => { + throw new CustomError(400, `Bad request: ${error.message}`, error); + }); + hasNextQuery = productChunk.length == CHUNK_SIZE; + if (productChunk.length > 0) { + lastProductId = productChunk[productChunk.length - 1].id; + allProducts = allProducts.concat(productChunk); + } + } + return allProducts; } export const eventHandler = async (request, response) => { From 51879793fc05d995c0654ecf5854a26932591634 Mon Sep 17 00:00:00 2001 From: King-Hin Leung Date: Mon, 14 Aug 2023 10:28:11 +0200 Subject: [PATCH 2/4] Update REST api resources name in URL --- full-ingestion/src/routes/event.route.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/full-ingestion/src/routes/event.route.js b/full-ingestion/src/routes/event.route.js index 74f8b5f..89d1100 100644 --- a/full-ingestion/src/routes/event.route.js +++ b/full-ingestion/src/routes/event.route.js @@ -4,6 +4,6 @@ import { eventHandler } from '../controllers/event.controller.js'; const eventRouter = Router(); -eventRouter.post('/:storeKey', eventHandler); +eventRouter.post('/fullSync/:storeKey', eventHandler); export default eventRouter; From 9faeef857f2676f3a2800f6c18358d2898c00d45 Mon Sep 17 00:00:00 2001 From: King-Hin Leung Date: Mon, 14 Aug 2023 10:36:20 +0200 Subject: [PATCH 3/4] Rename files and variables --- .../{event.controller.js => sync.controller.js} | 2 +- full-ingestion/src/index.js | 4 ++-- full-ingestion/src/routes/event.route.js | 9 --------- full-ingestion/src/routes/sync.route.js | 9 +++++++++ 4 files changed, 12 insertions(+), 12 deletions(-) rename full-ingestion/src/controllers/{event.controller.js => sync.controller.js} (97%) delete mode 100644 full-ingestion/src/routes/event.route.js create mode 100644 full-ingestion/src/routes/sync.route.js diff --git a/full-ingestion/src/controllers/event.controller.js b/full-ingestion/src/controllers/sync.controller.js similarity index 97% rename from full-ingestion/src/controllers/event.controller.js rename to full-ingestion/src/controllers/sync.controller.js index daa8de8..d52fc98 100644 --- a/full-ingestion/src/controllers/event.controller.js +++ b/full-ingestion/src/controllers/sync.controller.js @@ -55,7 +55,7 @@ async function getProductsByStore(storeKey) { return allProducts; } -export const eventHandler = async (request, response) => { +export const syncHandler = async (request, response) => { try { const storeKey = request.params.storeKey; if (!storeKey) { diff --git a/full-ingestion/src/index.js b/full-ingestion/src/index.js index 8a04059..33b15fd 100644 --- a/full-ingestion/src/index.js +++ b/full-ingestion/src/index.js @@ -4,7 +4,7 @@ import express from 'express'; import bodyParser from 'body-parser'; // Import routes -import EventRoutes from './routes/event.route.js'; +import SyncRoutes from './routes/sync.route.js'; import { logger } from './utils/logger.utils.js'; const PORT = 8080; @@ -17,7 +17,7 @@ app.use(bodyParser.json()); app.use(bodyParser.urlencoded({ extended: true })); // Define routes -app.use('/', EventRoutes); +app.use('/', SyncRoutes); // Listen the application const server = app.listen(PORT, () => { diff --git a/full-ingestion/src/routes/event.route.js b/full-ingestion/src/routes/event.route.js deleted file mode 100644 index 89d1100..0000000 --- a/full-ingestion/src/routes/event.route.js +++ /dev/null @@ -1,9 +0,0 @@ -import { Router } from 'express'; - -import { eventHandler } from '../controllers/event.controller.js'; - -const eventRouter = Router(); - -eventRouter.post('/fullSync/:storeKey', eventHandler); - -export default eventRouter; diff --git a/full-ingestion/src/routes/sync.route.js b/full-ingestion/src/routes/sync.route.js new file mode 100644 index 0000000..b1cb54b --- /dev/null +++ b/full-ingestion/src/routes/sync.route.js @@ -0,0 +1,9 @@ +import { Router } from 'express'; + +import { syncHandler } from '../controllers/sync.controller.js'; + +const syncRouter = Router(); + +syncRouter.post('/fullSync/:storeKey', syncHandler); + +export default syncRouter; From 4765ad90ff91c7b9ab165cb452ee6e9f09f7f935 Mon Sep 17 00:00:00 2001 From: King-Hin Leung Date: Mon, 14 Aug 2023 12:04:20 +0200 Subject: [PATCH 4/4] Update build.client.js --- full-ingestion/src/clients/build.client.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/full-ingestion/src/clients/build.client.js b/full-ingestion/src/clients/build.client.js index 07c1f74..993bb2f 100644 --- a/full-ingestion/src/clients/build.client.js +++ b/full-ingestion/src/clients/build.client.js @@ -5,7 +5,7 @@ import { readConfiguration } from '../utils/config.utils.js'; /** * Create a new client builder. - * This code creates a new Client that can be used to make API calls + * This code creates a new client builder that can be used to make API calls */ export const createClient = () => new ClientBuilder()