From 5c3ecebefa7fc888d849cd0d76d9ae40400dbfa1 Mon Sep 17 00:00:00 2001 From: King-Hin Leung Date: Fri, 4 Aug 2023 11:36:04 +0200 Subject: [PATCH 01/15] Update ci.yaml --- .github/workflows/ci.yaml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index eb4b70d..7105322 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -20,15 +20,15 @@ jobs: build-full-ingestion: name: Build the application for full-ingestion - runs-on: ubuntu-latest - defaults: - run: - working-directory: full-ingestion - steps: - - uses: actions/checkout@v3 - - name: Use Node.js 16.x - uses: actions/setup-node@v3 - with: - node-version: '16.x' - - run: npm ci - - run: npm run lint && npm run prettier \ No newline at end of file + runs-on: ubuntu-latest + defaults: + run: + working-directory: full-ingestion + steps: + - uses: actions/checkout@v3 + - name: Use Node.js 16.x + uses: actions/setup-node@v3 + with: + node-version: '16.x' + - run: npm ci + - run: npm run lint && npm run prettier \ No newline at end of file From c81e71068216cecd0458860e69a337420c7add35 Mon Sep 17 00:00:00 2001 From: King-Hin Leung Date: Mon, 7 Aug 2023 13:06:39 +0200 Subject: [PATCH 02/15] Add Algolia sample extension to sync products --- full-ingestion/package-lock.json | 276 +++++++++++++++++- full-ingestion/package.json | 6 +- full-ingestion/src/clients/build.client.js | 15 + full-ingestion/src/clients/create.client.js | 29 ++ .../src/controllers/event.controller.js | 73 ++++- .../algolia-example/clients/client.js | 12 + .../algolia-example/configurations/config.js | 5 + .../algolia-example/mappers/product.mapper.js | 5 + .../src/middlewares/auth.middleware.js | 14 + .../src/middlewares/error.middleware.js | 24 ++ .../src/middlewares/http.middleware.js | 8 + full-ingestion/src/routes/event.route.js | 2 +- full-ingestion/src/utils/config.utils.js | 32 ++ 13 files changed, 493 insertions(+), 8 deletions(-) create mode 100644 full-ingestion/src/clients/build.client.js create mode 100644 full-ingestion/src/clients/create.client.js create mode 100644 full-ingestion/src/extensions/algolia-example/clients/client.js create mode 100644 full-ingestion/src/extensions/algolia-example/configurations/config.js create mode 100644 full-ingestion/src/extensions/algolia-example/mappers/product.mapper.js create mode 100644 full-ingestion/src/middlewares/auth.middleware.js create mode 100644 full-ingestion/src/middlewares/error.middleware.js create mode 100644 full-ingestion/src/middlewares/http.middleware.js create mode 100644 full-ingestion/src/utils/config.utils.js diff --git a/full-ingestion/package-lock.json b/full-ingestion/package-lock.json index d67ee7c..4f6a71e 100644 --- a/full-ingestion/package-lock.json +++ b/full-ingestion/package-lock.json @@ -11,10 +11,12 @@ "dependencies": { "@commercetools-backend/loggers": "^21.19.0", "@commercetools/platform-sdk": "^4.1.0", - "@commercetools/sdk-client-v2": "^2.0.1", + "@commercetools/sdk-client-v2": "^2.2.0", + "algoliasearch": "^4.19.1", "body-parser": "^1.20.1", "dotenv": "^16.0.3", "express": "4.18.2", + "lodash": "^4.17.21", "validator": "^13.7.0" }, "devDependencies": { @@ -35,6 +37,121 @@ "node": ">=0.10.0" } }, + "node_modules/@algolia/cache-browser-local-storage": { + "version": "4.19.1", + "resolved": "https://registry.npmjs.org/@algolia/cache-browser-local-storage/-/cache-browser-local-storage-4.19.1.tgz", + "integrity": "sha512-FYAZWcGsFTTaSAwj9Std8UML3Bu8dyWDncM7Ls8g+58UOe4XYdlgzXWbrIgjaguP63pCCbMoExKr61B+ztK3tw==", + "dependencies": { + "@algolia/cache-common": "4.19.1" + } + }, + "node_modules/@algolia/cache-common": { + "version": "4.19.1", + "resolved": "https://registry.npmjs.org/@algolia/cache-common/-/cache-common-4.19.1.tgz", + "integrity": "sha512-XGghi3l0qA38HiqdoUY+wvGyBsGvKZ6U3vTiMBT4hArhP3fOGLXpIINgMiiGjTe4FVlTa5a/7Zf2bwlIHfRqqg==" + }, + "node_modules/@algolia/cache-in-memory": { + "version": "4.19.1", + "resolved": "https://registry.npmjs.org/@algolia/cache-in-memory/-/cache-in-memory-4.19.1.tgz", + "integrity": "sha512-+PDWL+XALGvIginigzu8oU6eWw+o76Z8zHbBovWYcrtWOEtinbl7a7UTt3x3lthv+wNuFr/YD1Gf+B+A9V8n5w==", + "dependencies": { + "@algolia/cache-common": "4.19.1" + } + }, + "node_modules/@algolia/client-account": { + "version": "4.19.1", + "resolved": "https://registry.npmjs.org/@algolia/client-account/-/client-account-4.19.1.tgz", + "integrity": "sha512-Oy0ritA2k7AMxQ2JwNpfaEcgXEDgeyKu0V7E7xt/ZJRdXfEpZcwp9TOg4TJHC7Ia62gIeT2Y/ynzsxccPw92GA==", + "dependencies": { + "@algolia/client-common": "4.19.1", + "@algolia/client-search": "4.19.1", + "@algolia/transporter": "4.19.1" + } + }, + "node_modules/@algolia/client-analytics": { + "version": "4.19.1", + "resolved": "https://registry.npmjs.org/@algolia/client-analytics/-/client-analytics-4.19.1.tgz", + "integrity": "sha512-5QCq2zmgdZLIQhHqwl55ZvKVpLM3DNWjFI4T+bHr3rGu23ew2bLO4YtyxaZeChmDb85jUdPDouDlCumGfk6wOg==", + "dependencies": { + "@algolia/client-common": "4.19.1", + "@algolia/client-search": "4.19.1", + "@algolia/requester-common": "4.19.1", + "@algolia/transporter": "4.19.1" + } + }, + "node_modules/@algolia/client-common": { + "version": "4.19.1", + "resolved": "https://registry.npmjs.org/@algolia/client-common/-/client-common-4.19.1.tgz", + "integrity": "sha512-3kAIVqTcPrjfS389KQvKzliC559x+BDRxtWamVJt8IVp7LGnjq+aVAXg4Xogkur1MUrScTZ59/AaUd5EdpyXgA==", + "dependencies": { + "@algolia/requester-common": "4.19.1", + "@algolia/transporter": "4.19.1" + } + }, + "node_modules/@algolia/client-personalization": { + "version": "4.19.1", + "resolved": "https://registry.npmjs.org/@algolia/client-personalization/-/client-personalization-4.19.1.tgz", + "integrity": "sha512-8CWz4/H5FA+krm9HMw2HUQenizC/DxUtsI5oYC0Jxxyce1vsr8cb1aEiSJArQT6IzMynrERif1RVWLac1m36xw==", + "dependencies": { + "@algolia/client-common": "4.19.1", + "@algolia/requester-common": "4.19.1", + "@algolia/transporter": "4.19.1" + } + }, + "node_modules/@algolia/client-search": { + "version": "4.19.1", + "resolved": "https://registry.npmjs.org/@algolia/client-search/-/client-search-4.19.1.tgz", + "integrity": "sha512-mBecfMFS4N+yK/p0ZbK53vrZbL6OtWMk8YmnOv1i0LXx4pelY8TFhqKoTit3NPVPwoSNN0vdSN9dTu1xr1XOVw==", + "dependencies": { + "@algolia/client-common": "4.19.1", + "@algolia/requester-common": "4.19.1", + "@algolia/transporter": "4.19.1" + } + }, + "node_modules/@algolia/logger-common": { + "version": "4.19.1", + "resolved": "https://registry.npmjs.org/@algolia/logger-common/-/logger-common-4.19.1.tgz", + "integrity": "sha512-i6pLPZW/+/YXKis8gpmSiNk1lOmYCmRI6+x6d2Qk1OdfvX051nRVdalRbEcVTpSQX6FQAoyeaui0cUfLYW5Elw==" + }, + "node_modules/@algolia/logger-console": { + "version": "4.19.1", + "resolved": "https://registry.npmjs.org/@algolia/logger-console/-/logger-console-4.19.1.tgz", + "integrity": "sha512-jj72k9GKb9W0c7TyC3cuZtTr0CngLBLmc8trzZlXdfvQiigpUdvTi1KoWIb2ZMcRBG7Tl8hSb81zEY3zI2RlXg==", + "dependencies": { + "@algolia/logger-common": "4.19.1" + } + }, + "node_modules/@algolia/requester-browser-xhr": { + "version": "4.19.1", + "resolved": "https://registry.npmjs.org/@algolia/requester-browser-xhr/-/requester-browser-xhr-4.19.1.tgz", + "integrity": "sha512-09K/+t7lptsweRTueHnSnmPqIxbHMowejAkn9XIcJMLdseS3zl8ObnS5GWea86mu3vy4+8H+ZBKkUN82Zsq/zg==", + "dependencies": { + "@algolia/requester-common": "4.19.1" + } + }, + "node_modules/@algolia/requester-common": { + "version": "4.19.1", + "resolved": "https://registry.npmjs.org/@algolia/requester-common/-/requester-common-4.19.1.tgz", + "integrity": "sha512-BisRkcWVxrDzF1YPhAckmi2CFYK+jdMT60q10d7z3PX+w6fPPukxHRnZwooiTUrzFe50UBmLItGizWHP5bDzVQ==" + }, + "node_modules/@algolia/requester-node-http": { + "version": "4.19.1", + "resolved": "https://registry.npmjs.org/@algolia/requester-node-http/-/requester-node-http-4.19.1.tgz", + "integrity": "sha512-6DK52DHviBHTG2BK/Vv2GIlEw7i+vxm7ypZW0Z7vybGCNDeWzADx+/TmxjkES2h15+FZOqVf/Ja677gePsVItA==", + "dependencies": { + "@algolia/requester-common": "4.19.1" + } + }, + "node_modules/@algolia/transporter": { + "version": "4.19.1", + "resolved": "https://registry.npmjs.org/@algolia/transporter/-/transporter-4.19.1.tgz", + "integrity": "sha512-nkpvPWbpuzxo1flEYqNIbGz7xhfhGOKGAZS7tzC+TELgEmi7z99qRyTfNSUlW7LZmB3ACdnqAo+9A9KFBENviQ==", + "dependencies": { + "@algolia/cache-common": "4.19.1", + "@algolia/logger-common": "4.19.1", + "@algolia/requester-common": "4.19.1" + } + }, "node_modules/@ampproject/remapping": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.2.1.tgz", @@ -2981,6 +3098,27 @@ "url": "https://github.com/sponsors/epoberezkin" } }, + "node_modules/algoliasearch": { + "version": "4.19.1", + "resolved": "https://registry.npmjs.org/algoliasearch/-/algoliasearch-4.19.1.tgz", + "integrity": "sha512-IJF5b93b2MgAzcE/tuzW0yOPnuUyRgGAtaPv5UUywXM8kzqfdwZTO4sPJBzoGz1eOy6H9uEchsJsBFTELZSu+g==", + "dependencies": { + "@algolia/cache-browser-local-storage": "4.19.1", + "@algolia/cache-common": "4.19.1", + "@algolia/cache-in-memory": "4.19.1", + "@algolia/client-account": "4.19.1", + "@algolia/client-analytics": "4.19.1", + "@algolia/client-common": "4.19.1", + "@algolia/client-personalization": "4.19.1", + "@algolia/client-search": "4.19.1", + "@algolia/logger-common": "4.19.1", + "@algolia/logger-console": "4.19.1", + "@algolia/requester-browser-xhr": "4.19.1", + "@algolia/requester-common": "4.19.1", + "@algolia/requester-node-http": "4.19.1", + "@algolia/transporter": "4.19.1" + } + }, "node_modules/ansi-escapes": { "version": "4.3.2", "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-4.3.2.tgz", @@ -8494,6 +8632,121 @@ "integrity": "sha512-1Yjs2SvM8TflER/OD3cOjhWWOZb58A2t7wpE2S9XfBYTiIl+XFhQG2bjy4Pu1I+EAlCNUzRDYDdFwFYUKvXcIA==", "dev": true }, + "@algolia/cache-browser-local-storage": { + "version": "4.19.1", + "resolved": "https://registry.npmjs.org/@algolia/cache-browser-local-storage/-/cache-browser-local-storage-4.19.1.tgz", + "integrity": "sha512-FYAZWcGsFTTaSAwj9Std8UML3Bu8dyWDncM7Ls8g+58UOe4XYdlgzXWbrIgjaguP63pCCbMoExKr61B+ztK3tw==", + "requires": { + "@algolia/cache-common": "4.19.1" + } + }, + "@algolia/cache-common": { + "version": "4.19.1", + "resolved": "https://registry.npmjs.org/@algolia/cache-common/-/cache-common-4.19.1.tgz", + "integrity": "sha512-XGghi3l0qA38HiqdoUY+wvGyBsGvKZ6U3vTiMBT4hArhP3fOGLXpIINgMiiGjTe4FVlTa5a/7Zf2bwlIHfRqqg==" + }, + "@algolia/cache-in-memory": { + "version": "4.19.1", + "resolved": "https://registry.npmjs.org/@algolia/cache-in-memory/-/cache-in-memory-4.19.1.tgz", + "integrity": "sha512-+PDWL+XALGvIginigzu8oU6eWw+o76Z8zHbBovWYcrtWOEtinbl7a7UTt3x3lthv+wNuFr/YD1Gf+B+A9V8n5w==", + "requires": { + "@algolia/cache-common": "4.19.1" + } + }, + "@algolia/client-account": { + "version": "4.19.1", + "resolved": "https://registry.npmjs.org/@algolia/client-account/-/client-account-4.19.1.tgz", + "integrity": "sha512-Oy0ritA2k7AMxQ2JwNpfaEcgXEDgeyKu0V7E7xt/ZJRdXfEpZcwp9TOg4TJHC7Ia62gIeT2Y/ynzsxccPw92GA==", + "requires": { + "@algolia/client-common": "4.19.1", + "@algolia/client-search": "4.19.1", + "@algolia/transporter": "4.19.1" + } + }, + "@algolia/client-analytics": { + "version": "4.19.1", + "resolved": "https://registry.npmjs.org/@algolia/client-analytics/-/client-analytics-4.19.1.tgz", + "integrity": "sha512-5QCq2zmgdZLIQhHqwl55ZvKVpLM3DNWjFI4T+bHr3rGu23ew2bLO4YtyxaZeChmDb85jUdPDouDlCumGfk6wOg==", + "requires": { + "@algolia/client-common": "4.19.1", + "@algolia/client-search": "4.19.1", + "@algolia/requester-common": "4.19.1", + "@algolia/transporter": "4.19.1" + } + }, + "@algolia/client-common": { + "version": "4.19.1", + "resolved": "https://registry.npmjs.org/@algolia/client-common/-/client-common-4.19.1.tgz", + "integrity": "sha512-3kAIVqTcPrjfS389KQvKzliC559x+BDRxtWamVJt8IVp7LGnjq+aVAXg4Xogkur1MUrScTZ59/AaUd5EdpyXgA==", + "requires": { + "@algolia/requester-common": "4.19.1", + "@algolia/transporter": "4.19.1" + } + }, + "@algolia/client-personalization": { + "version": "4.19.1", + "resolved": "https://registry.npmjs.org/@algolia/client-personalization/-/client-personalization-4.19.1.tgz", + "integrity": "sha512-8CWz4/H5FA+krm9HMw2HUQenizC/DxUtsI5oYC0Jxxyce1vsr8cb1aEiSJArQT6IzMynrERif1RVWLac1m36xw==", + "requires": { + "@algolia/client-common": "4.19.1", + "@algolia/requester-common": "4.19.1", + "@algolia/transporter": "4.19.1" + } + }, + "@algolia/client-search": { + "version": "4.19.1", + "resolved": "https://registry.npmjs.org/@algolia/client-search/-/client-search-4.19.1.tgz", + "integrity": "sha512-mBecfMFS4N+yK/p0ZbK53vrZbL6OtWMk8YmnOv1i0LXx4pelY8TFhqKoTit3NPVPwoSNN0vdSN9dTu1xr1XOVw==", + "requires": { + "@algolia/client-common": "4.19.1", + "@algolia/requester-common": "4.19.1", + "@algolia/transporter": "4.19.1" + } + }, + "@algolia/logger-common": { + "version": "4.19.1", + "resolved": "https://registry.npmjs.org/@algolia/logger-common/-/logger-common-4.19.1.tgz", + "integrity": "sha512-i6pLPZW/+/YXKis8gpmSiNk1lOmYCmRI6+x6d2Qk1OdfvX051nRVdalRbEcVTpSQX6FQAoyeaui0cUfLYW5Elw==" + }, + "@algolia/logger-console": { + "version": "4.19.1", + "resolved": "https://registry.npmjs.org/@algolia/logger-console/-/logger-console-4.19.1.tgz", + "integrity": "sha512-jj72k9GKb9W0c7TyC3cuZtTr0CngLBLmc8trzZlXdfvQiigpUdvTi1KoWIb2ZMcRBG7Tl8hSb81zEY3zI2RlXg==", + "requires": { + "@algolia/logger-common": "4.19.1" + } + }, + "@algolia/requester-browser-xhr": { + "version": "4.19.1", + "resolved": "https://registry.npmjs.org/@algolia/requester-browser-xhr/-/requester-browser-xhr-4.19.1.tgz", + "integrity": "sha512-09K/+t7lptsweRTueHnSnmPqIxbHMowejAkn9XIcJMLdseS3zl8ObnS5GWea86mu3vy4+8H+ZBKkUN82Zsq/zg==", + "requires": { + "@algolia/requester-common": "4.19.1" + } + }, + "@algolia/requester-common": { + "version": "4.19.1", + "resolved": "https://registry.npmjs.org/@algolia/requester-common/-/requester-common-4.19.1.tgz", + "integrity": "sha512-BisRkcWVxrDzF1YPhAckmi2CFYK+jdMT60q10d7z3PX+w6fPPukxHRnZwooiTUrzFe50UBmLItGizWHP5bDzVQ==" + }, + "@algolia/requester-node-http": { + "version": "4.19.1", + "resolved": "https://registry.npmjs.org/@algolia/requester-node-http/-/requester-node-http-4.19.1.tgz", + "integrity": "sha512-6DK52DHviBHTG2BK/Vv2GIlEw7i+vxm7ypZW0Z7vybGCNDeWzADx+/TmxjkES2h15+FZOqVf/Ja677gePsVItA==", + "requires": { + "@algolia/requester-common": "4.19.1" + } + }, + "@algolia/transporter": { + "version": "4.19.1", + "resolved": "https://registry.npmjs.org/@algolia/transporter/-/transporter-4.19.1.tgz", + "integrity": "sha512-nkpvPWbpuzxo1flEYqNIbGz7xhfhGOKGAZS7tzC+TELgEmi7z99qRyTfNSUlW7LZmB3ACdnqAo+9A9KFBENviQ==", + "requires": { + "@algolia/cache-common": "4.19.1", + "@algolia/logger-common": "4.19.1", + "@algolia/requester-common": "4.19.1" + } + }, "@ampproject/remapping": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.2.1.tgz", @@ -10636,6 +10889,27 @@ "uri-js": "^4.2.2" } }, + "algoliasearch": { + "version": "4.19.1", + "resolved": "https://registry.npmjs.org/algoliasearch/-/algoliasearch-4.19.1.tgz", + "integrity": "sha512-IJF5b93b2MgAzcE/tuzW0yOPnuUyRgGAtaPv5UUywXM8kzqfdwZTO4sPJBzoGz1eOy6H9uEchsJsBFTELZSu+g==", + "requires": { + "@algolia/cache-browser-local-storage": "4.19.1", + "@algolia/cache-common": "4.19.1", + "@algolia/cache-in-memory": "4.19.1", + "@algolia/client-account": "4.19.1", + "@algolia/client-analytics": "4.19.1", + "@algolia/client-common": "4.19.1", + "@algolia/client-personalization": "4.19.1", + "@algolia/client-search": "4.19.1", + "@algolia/logger-common": "4.19.1", + "@algolia/logger-console": "4.19.1", + "@algolia/requester-browser-xhr": "4.19.1", + "@algolia/requester-common": "4.19.1", + "@algolia/requester-node-http": "4.19.1", + "@algolia/transporter": "4.19.1" + } + }, "ansi-escapes": { "version": "4.3.2", "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-4.3.2.tgz", diff --git a/full-ingestion/package.json b/full-ingestion/package.json index 51450a1..f9b32a6 100644 --- a/full-ingestion/package.json +++ b/full-ingestion/package.json @@ -27,10 +27,12 @@ "dependencies": { "@commercetools-backend/loggers": "^21.19.0", "@commercetools/platform-sdk": "^4.1.0", - "@commercetools/sdk-client-v2": "^2.0.1", + "@commercetools/sdk-client-v2": "^2.2.0", + "algoliasearch": "^4.19.1", "body-parser": "^1.20.1", "dotenv": "^16.0.3", "express": "4.18.2", + "lodash": "^4.17.21", "validator": "^13.7.0" } -} \ No newline at end of file +} diff --git a/full-ingestion/src/clients/build.client.js b/full-ingestion/src/clients/build.client.js new file mode 100644 index 0000000..07c1f74 --- /dev/null +++ b/full-ingestion/src/clients/build.client.js @@ -0,0 +1,15 @@ +import { ClientBuilder } from '@commercetools/sdk-client-v2'; +import { authMiddlewareOptions } from '../middlewares/auth.middleware.js'; +import { httpMiddlewareOptions } from '../middlewares/http.middleware.js'; +import { readConfiguration } from '../utils/config.utils.js'; + +/** + * Create a new client builder. + * This code creates a new Client that can be used to make API calls + */ +export const createClient = () => + new ClientBuilder() + .withProjectKey(readConfiguration().projectKey) + .withClientCredentialsFlow(authMiddlewareOptions) + .withHttpMiddleware(httpMiddlewareOptions) + .build(); diff --git a/full-ingestion/src/clients/create.client.js b/full-ingestion/src/clients/create.client.js new file mode 100644 index 0000000..57ffd2e --- /dev/null +++ b/full-ingestion/src/clients/create.client.js @@ -0,0 +1,29 @@ +import { createApiBuilderFromCtpClient } from '@commercetools/platform-sdk'; +import { createClient } from './build.client.js'; +import { readConfiguration } from '../utils/config.utils.js'; + +/** + * Create client with apiRoot + * apiRoot can now be used to build requests to de Composable Commerce API + */ +export const createApiRoot = ((root) => () => { + if (root) { + return root; + } + + root = createApiBuilderFromCtpClient(createClient()).withProjectKey({ + projectKey: readConfiguration().projectKey, + }); + + return root; +})(); + +/** + * Example code to get the Project details + * This code has the same effect as sending a GET + * request to the commercetools Composable Commerce API without any endpoints. + * + */ +export const getProject = async () => { + return await createApiRoot().get().execute(); +}; diff --git a/full-ingestion/src/controllers/event.controller.js b/full-ingestion/src/controllers/event.controller.js index e8bdf5a..44a1dfc 100644 --- a/full-ingestion/src/controllers/event.controller.js +++ b/full-ingestion/src/controllers/event.controller.js @@ -1,5 +1,70 @@ import { logger } from '../utils/logger.utils.js'; +import { createApiRoot } from '../clients/create.client.js'; import CustomError from '../errors/custom.error.js'; +import { Process as process } from '@commercetools/sdk-client-v2'; +import { default as productMapping } from '../extensions/algolia-example/mappers/product.mapper.js'; +import { default as saveProducts } from '../extensions/algolia-example/clients/algolia.client.js'; + +// async function getProductsByProductSelection(productSelectionId) { +// +// return await createApiRoot() +// .productSelections() +// .withId({ ID: Buffer.from(productSelectionId).toString() }) +// .products() +// .get({ +// queryArgs : { +// expand : 'product' +// } +// }) +// .execute() +// .then(response => response.body.results) +// +// } + +async function getProductsByProcess() { + const request = await createApiRoot().products().get().request; + + const processFn = (data) => { + const results = data.body.results; + + const mappedProducts = results.map((product) => productMapping(product)); + saveProducts(mappedProducts); + }; + const opt = { + accumulate: false, // accumulate all the processed result into an array, default `true` + }; + + return await process(request, processFn, opt); +} + +// async function getProductsByStoreId(storeId) { +// try { +// const store = await createApiRoot() +// .stores() +// .withId({ ID: Buffer.from(storeId).toString() }) +// .get({ +// queryArgs : { +// expand : 'productSelections[*].productSelection' +// } +// }) +// .execute(); +// +// +// +// const productSelections = store.body.productSelections +// const stream = +// await Promise.all(_.chain(productSelections) +// .map(productSelectionItem => productSelectionItem.productSelection.id) +// .map(async productSelectionId => { +// return await getProductsByProductSelection(productSelectionId) +// })) +// stream.map(item => console.log(item)); +// +// +// } catch (error) { +// throw new CustomError(400, `Bad request: ${error}`); +// } +// } export const eventHandler = async (request, response) => { // Check request body @@ -11,12 +76,12 @@ export const eventHandler = async (request, response) => { // Check if the body comes in a message if (!request.body.message) { logger.error('Missing body message'); - throw new CustomError(400, 'Bad request: Wrong No Pub/Sub message format'); + throw new CustomError(400, 'Bad request: Wrong Pub/Sub message format'); } - // TODO : - // 1. Invoke API call to Commercetools to retrieve all products by store key/ID - // 2. Calling SDK function within Streaming API to persist full set of products to external search engine + // const storeId = request.params.id + + await getProductsByProcess(); // Return the response for the client response.status(204).send(); diff --git a/full-ingestion/src/extensions/algolia-example/clients/client.js b/full-ingestion/src/extensions/algolia-example/clients/client.js new file mode 100644 index 0000000..8930da7 --- /dev/null +++ b/full-ingestion/src/extensions/algolia-example/clients/client.js @@ -0,0 +1,12 @@ +import algoliasearch from 'algoliasearch'; +import { config } from '../configurations/config.js'; + +export default function save(products) { + const client = algoliasearch(config.applicationId, config.searchApiKey); + const index = client.initIndex(config.index); + index + .saveObjects(products, { autoGenerateObjectIDIfNotExist: false }) + .catch((error) => { + throw error; + }); +} diff --git a/full-ingestion/src/extensions/algolia-example/configurations/config.js b/full-ingestion/src/extensions/algolia-example/configurations/config.js new file mode 100644 index 0000000..9712e9b --- /dev/null +++ b/full-ingestion/src/extensions/algolia-example/configurations/config.js @@ -0,0 +1,5 @@ +export const config = { + applicationId: process.env.AGOLIA_APPLICATION_ID, + searchApiKey: process.env.AGOLIA_SEARCH_API_KEY, + index: process.env.AGOLIA_INDEX, +}; diff --git a/full-ingestion/src/extensions/algolia-example/mappers/product.mapper.js b/full-ingestion/src/extensions/algolia-example/mappers/product.mapper.js new file mode 100644 index 0000000..e1e62ed --- /dev/null +++ b/full-ingestion/src/extensions/algolia-example/mappers/product.mapper.js @@ -0,0 +1,5 @@ +export default function map(product) { + let mappedProduct = {}; + mappedProduct.objectID = product.id; + return mappedProduct; +} diff --git a/full-ingestion/src/middlewares/auth.middleware.js b/full-ingestion/src/middlewares/auth.middleware.js new file mode 100644 index 0000000..0214676 --- /dev/null +++ b/full-ingestion/src/middlewares/auth.middleware.js @@ -0,0 +1,14 @@ +import { readConfiguration } from '../utils/config.utils.js'; + +/** + * Configure Middleware. Example only. Adapt on your own + */ +export const authMiddlewareOptions = { + host: `https://auth.${readConfiguration().region}.commercetools.com`, + projectKey: readConfiguration().projectKey, + credentials: { + clientId: readConfiguration().clientId, + clientSecret: readConfiguration().clientSecret, + }, + scopes: [readConfiguration().scope ? readConfiguration().scope : 'default'], +}; diff --git a/full-ingestion/src/middlewares/error.middleware.js b/full-ingestion/src/middlewares/error.middleware.js new file mode 100644 index 0000000..0addc58 --- /dev/null +++ b/full-ingestion/src/middlewares/error.middleware.js @@ -0,0 +1,24 @@ +import CustomError from '../errors/custom.error.js'; + +/** + * Middleware for error handling + * @param error The error object + * @param req The express request + * @param res The Express response + * @param next + * @returns + */ +export const errorMiddleware = (error, _req, res, _next) => { + if (error instanceof CustomError) { + if (typeof error.statusCode === 'number') { + res.status(error.statusCode).json({ + message: error.message, + errors: error.errors, + }); + + return; + } + } + + res.status(500).send('Internal server error'); +}; diff --git a/full-ingestion/src/middlewares/http.middleware.js b/full-ingestion/src/middlewares/http.middleware.js new file mode 100644 index 0000000..614e36d --- /dev/null +++ b/full-ingestion/src/middlewares/http.middleware.js @@ -0,0 +1,8 @@ +import { readConfiguration } from '../utils/config.utils.js'; + +/** + * Configure Middleware. Example only. Adapt on your own + */ +export const httpMiddlewareOptions = { + host: `https://api.${readConfiguration().region}.commercetools.com`, +}; diff --git a/full-ingestion/src/routes/event.route.js b/full-ingestion/src/routes/event.route.js index 06e081d..1653a7d 100644 --- a/full-ingestion/src/routes/event.route.js +++ b/full-ingestion/src/routes/event.route.js @@ -4,6 +4,6 @@ import { eventHandler } from '../controllers/event.controller.js'; const eventRouter = Router(); -eventRouter.post('/', eventHandler); +eventRouter.post('/:id', eventHandler); export default eventRouter; diff --git a/full-ingestion/src/utils/config.utils.js b/full-ingestion/src/utils/config.utils.js new file mode 100644 index 0000000..99a732a --- /dev/null +++ b/full-ingestion/src/utils/config.utils.js @@ -0,0 +1,32 @@ +// import CustomError from '../errors/custom.error.js'; +// import envValidators from '../validators/env.validators.js'; +// import { getValidateMessages } from '../validators/helpers.validators.js'; + +/** + * Read the configuration env vars + * (Add yours accordingly) + * + * @returns The configuration with the correct env vars + */ + +export const readConfiguration = () => { + const envVars = { + clientId: process.env.CTP_CLIENT_ID, + clientSecret: process.env.CTP_CLIENT_SECRET, + projectKey: process.env.CTP_PROJECT_KEY, + scope: process.env.CTP_SCOPE, + region: process.env.CTP_REGION, + }; + + // const validationErrors = getValidateMessages(envValidators, envVars); + // + // if (validationErrors.length) { + // throw new CustomError( + // 'InvalidEnvironmentVariablesError', + // 'Invalid Environment Variables please check your .env file', + // validationErrors + // ); + // } + + return envVars; +}; From e89b336fe8e9152f96a5da3287224ba72acfe797 Mon Sep 17 00:00:00 2001 From: King-Hin Leung Date: Tue, 8 Aug 2023 10:21:23 +0200 Subject: [PATCH 03/15] Update event.controller.js --- .../src/controllers/event.controller.js | 83 +++++++++---------- 1 file changed, 41 insertions(+), 42 deletions(-) diff --git a/full-ingestion/src/controllers/event.controller.js b/full-ingestion/src/controllers/event.controller.js index 44a1dfc..d5a8840 100644 --- a/full-ingestion/src/controllers/event.controller.js +++ b/full-ingestion/src/controllers/event.controller.js @@ -3,7 +3,7 @@ import { createApiRoot } from '../clients/create.client.js'; import CustomError from '../errors/custom.error.js'; import { Process as process } from '@commercetools/sdk-client-v2'; import { default as productMapping } from '../extensions/algolia-example/mappers/product.mapper.js'; -import { default as saveProducts } from '../extensions/algolia-example/clients/algolia.client.js'; +import { default as saveProducts } from '../extensions/algolia-example/clients/client.js'; // async function getProductsByProductSelection(productSelectionId) { // @@ -21,50 +21,49 @@ import { default as saveProducts } from '../extensions/algolia-example/clients/a // // } -async function getProductsByProcess() { - const request = await createApiRoot().products().get().request; - - const processFn = (data) => { - const results = data.body.results; - +async function syncProcess(data) { + const results = data.body.results; + if (results) { const mappedProducts = results.map((product) => productMapping(product)); saveProducts(mappedProducts); - }; - const opt = { - accumulate: false, // accumulate all the processed result into an array, default `true` - }; + } +} - return await process(request, processFn, opt); +async function syncProducts(storeId) { + const productSelections = await getProductSelectionsByStoreId(storeId) + + productSelections.map(async (productionSelecton) => { + const productSelectionId = productionSelecton.productSelection.id + const request = await createApiRoot() + .productSelections() + .withId({ ID: Buffer.from(productSelectionId).toString() }) + .products() + .get({ queryArgs: { sort: 'sort' } }).request + + await process(request, syncProcess, { + accumulate: false + }); + }) } -// async function getProductsByStoreId(storeId) { -// try { -// const store = await createApiRoot() -// .stores() -// .withId({ ID: Buffer.from(storeId).toString() }) -// .get({ -// queryArgs : { -// expand : 'productSelections[*].productSelection' -// } -// }) -// .execute(); -// -// -// -// const productSelections = store.body.productSelections -// const stream = -// await Promise.all(_.chain(productSelections) -// .map(productSelectionItem => productSelectionItem.productSelection.id) -// .map(async productSelectionId => { -// return await getProductsByProductSelection(productSelectionId) -// })) -// stream.map(item => console.log(item)); -// -// -// } catch (error) { -// throw new CustomError(400, `Bad request: ${error}`); -// } -// } +async function getProductSelectionsByStoreId(storeId) { + return await createApiRoot() + .stores() + .withId({ ID: Buffer.from(storeId).toString() }) + .get({ + queryArgs : { + expand : 'productSelections[*].productSelection' + } + }) + .execute() + .then(response => + response.body.productSelections + ) + .catch(error => { + throw new CustomError(400, `Bad request: ${error}`); + }); + +} export const eventHandler = async (request, response) => { // Check request body @@ -79,9 +78,9 @@ export const eventHandler = async (request, response) => { throw new CustomError(400, 'Bad request: Wrong Pub/Sub message format'); } - // const storeId = request.params.id + const storeId = request.params.id - await getProductsByProcess(); + await syncProducts(storeId); // Return the response for the client response.status(204).send(); From d93cfae6fb41dd3fc8a3e8655c65c0a83b0edf66 Mon Sep 17 00:00:00 2001 From: King-Hin Leung Date: Wed, 9 Aug 2023 15:57:12 +0200 Subject: [PATCH 04/15] Fix error handling --- full-ingestion/src/connectors/post-deploy.js | 27 +++++ .../src/controllers/event.controller.js | 103 +++++++----------- .../algolia-example/clients/client.js | 26 ++++- .../configurations/constants.js | 3 + .../algolia-example/mappers/product.mapper.js | 1 + 5 files changed, 88 insertions(+), 72 deletions(-) create mode 100644 full-ingestion/src/connectors/post-deploy.js create mode 100644 full-ingestion/src/extensions/algolia-example/configurations/constants.js diff --git a/full-ingestion/src/connectors/post-deploy.js b/full-ingestion/src/connectors/post-deploy.js new file mode 100644 index 0000000..1e25f64 --- /dev/null +++ b/full-ingestion/src/connectors/post-deploy.js @@ -0,0 +1,27 @@ +import fetch from 'node-fetch'; + +import { assertError, assertString } from '../utils/assert.utils.js'; + +const CONNECT_SERVICE_URL = 'CONNECT_SERVICE_URL'; +const COMMERCETOOLS_STORE_ID = 'COMMERCETOOLS_STORE_ID'; + +async function postDeploy(properties) { + const deploymentUrl = properties.get(CONNECT_SERVICE_URL); + const storeId = properties.get(COMMERCETOOLS_STORE_ID); + + const response = await fetch(deploymentUrl, { method: 'POST', body: params }); + const data = await response.json(); +} + +async function run() { + try { + const properties = new Map(Object.entries(process.env)); + await postDeploy(properties); + } catch (error) { + assertError(error); + process.stderr.write(`Post-deploy failed: ${error.message}\n`); + process.exitCode = 1; + } +} + +run(); diff --git a/full-ingestion/src/controllers/event.controller.js b/full-ingestion/src/controllers/event.controller.js index d5a8840..3033e92 100644 --- a/full-ingestion/src/controllers/event.controller.js +++ b/full-ingestion/src/controllers/event.controller.js @@ -1,87 +1,58 @@ import { logger } from '../utils/logger.utils.js'; import { createApiRoot } from '../clients/create.client.js'; import CustomError from '../errors/custom.error.js'; -import { Process as process } from '@commercetools/sdk-client-v2'; -import { default as productMapping } from '../extensions/algolia-example/mappers/product.mapper.js'; import { default as saveProducts } from '../extensions/algolia-example/clients/client.js'; -// async function getProductsByProductSelection(productSelectionId) { -// -// return await createApiRoot() -// .productSelections() -// .withId({ ID: Buffer.from(productSelectionId).toString() }) -// .products() -// .get({ -// queryArgs : { -// expand : 'product' -// } -// }) -// .execute() -// .then(response => response.body.results) -// -// } +async function syncProducts(storeKey) { + const products = await getProductsByStore(storeKey); -async function syncProcess(data) { - const results = data.body.results; - if (results) { - const mappedProducts = results.map((product) => productMapping(product)); - saveProducts(mappedProducts); - } -} - -async function syncProducts(storeId) { - const productSelections = await getProductSelectionsByStoreId(storeId) - - productSelections.map(async (productionSelecton) => { - const productSelectionId = productionSelecton.productSelection.id - const request = await createApiRoot() - .productSelections() - .withId({ ID: Buffer.from(productSelectionId).toString() }) - .products() - .get({ queryArgs: { sort: 'sort' } }).request - - await process(request, syncProcess, { - accumulate: false - }); - }) + await saveProducts(products).catch((error) => { + throw new CustomError(400, `Bad request: ${error.message}`, error); + }); } -async function getProductSelectionsByStoreId(storeId) { +async function getProductsByStore(storeKey) { return await createApiRoot() - .stores() - .withId({ ID: Buffer.from(storeId).toString() }) + .inStoreKeyWithStoreKeyValue({ storeKey: Buffer.from(storeKey).toString() }) + .productSelectionAssignments() .get({ - queryArgs : { - expand : 'productSelections[*].productSelection' - } + queryArgs: { + expand: 'product', + }, }) .execute() - .then(response => - response.body.productSelections - ) - .catch(error => { - throw new CustomError(400, `Bad request: ${error}`); + .then((response) => response.body.results) + .then((results) => results.map((result) => result.product)) + .catch((error) => { + throw new CustomError(400, `Bad request: ${error.message}`, error); }); - } export const eventHandler = async (request, response) => { // Check request body - if (!request.body) { - logger.error('Missing request body.'); - throw new CustomError(400, 'Bad request: No Pub/Sub message was received'); - } - - // Check if the body comes in a message - if (!request.body.message) { - logger.error('Missing body message'); - throw new CustomError(400, 'Bad request: Wrong Pub/Sub message format'); + try { + if (!request.body) { + logger.error('Missing request body.'); + throw new CustomError( + 400, + 'Bad request: No Pub/Sub message was received' + ); + } + + // Check if the body comes in a message + if (!request.body.message) { + logger.error('Missing body message'); + throw new CustomError(400, 'Bad request: Wrong Pub/Sub message format'); + } + + const storeKey = request.params.id; + await syncProducts(storeKey); + } catch (err) { + logger.error(err); + + return response.status(err.statusCode).send(err); } - const storeId = request.params.id - - await syncProducts(storeId); - // Return the response for the client - response.status(204).send(); + return response.status(204).send(); }; diff --git a/full-ingestion/src/extensions/algolia-example/clients/client.js b/full-ingestion/src/extensions/algolia-example/clients/client.js index 8930da7..df2b43f 100644 --- a/full-ingestion/src/extensions/algolia-example/clients/client.js +++ b/full-ingestion/src/extensions/algolia-example/clients/client.js @@ -1,12 +1,26 @@ import algoliasearch from 'algoliasearch'; import { config } from '../configurations/config.js'; +import { constants } from '../configurations/constants.js'; + +import { default as productMapping } from '../mappers/product.mapper.js'; + +export default async function save(products) { + let productChunks = []; -export default function save(products) { const client = algoliasearch(config.applicationId, config.searchApiKey); const index = client.initIndex(config.index); - index - .saveObjects(products, { autoGenerateObjectIDIfNotExist: false }) - .catch((error) => { - throw error; - }); + + for (const product of products) { + productChunks.push(productMapping(product)); + if ( + productChunks.length === constants.CHUNK_LIMIT || + product === products[products.length - 1] + ) { + await index + .saveObjects(productChunks, { autoGenerateObjectIDIfNotExist: false }) + .then(() => { + productChunks = []; + }); + } + } } diff --git a/full-ingestion/src/extensions/algolia-example/configurations/constants.js b/full-ingestion/src/extensions/algolia-example/configurations/constants.js new file mode 100644 index 0000000..4cb224b --- /dev/null +++ b/full-ingestion/src/extensions/algolia-example/configurations/constants.js @@ -0,0 +1,3 @@ +export const constants = { + CHUNK_LIMIT: 1000, +}; diff --git a/full-ingestion/src/extensions/algolia-example/mappers/product.mapper.js b/full-ingestion/src/extensions/algolia-example/mappers/product.mapper.js index e1e62ed..de75913 100644 --- a/full-ingestion/src/extensions/algolia-example/mappers/product.mapper.js +++ b/full-ingestion/src/extensions/algolia-example/mappers/product.mapper.js @@ -1,5 +1,6 @@ export default function map(product) { let mappedProduct = {}; mappedProduct.objectID = product.id; + mappedProduct.masterData = product.masterData; return mappedProduct; } From 49f3732a945f4fb2ad1f907a08ce3b09e05d787f Mon Sep 17 00:00:00 2001 From: King-Hin Leung Date: Thu, 10 Aug 2023 09:15:57 +0200 Subject: [PATCH 05/15] Enhance the mapper module --- full-ingestion/package.json | 1 - .../src/controllers/event.controller.js | 10 +++++++--- .../algolia-example/mappers/product.mapper.js | 19 ++++++++++++++++++- full-ingestion/src/routes/event.route.js | 2 +- 4 files changed, 26 insertions(+), 6 deletions(-) diff --git a/full-ingestion/package.json b/full-ingestion/package.json index f9b32a6..33d8fe8 100644 --- a/full-ingestion/package.json +++ b/full-ingestion/package.json @@ -32,7 +32,6 @@ "body-parser": "^1.20.1", "dotenv": "^16.0.3", "express": "4.18.2", - "lodash": "^4.17.21", "validator": "^13.7.0" } } diff --git a/full-ingestion/src/controllers/event.controller.js b/full-ingestion/src/controllers/event.controller.js index 3033e92..e13963a 100644 --- a/full-ingestion/src/controllers/event.controller.js +++ b/full-ingestion/src/controllers/event.controller.js @@ -17,7 +17,12 @@ async function getProductsByStore(storeKey) { .productSelectionAssignments() .get({ queryArgs: { - expand: 'product', + expand: [ + 'product', + 'product.productType', + 'product.taxCategory', + 'product.masterData.current.categories[*]', + ], }, }) .execute() @@ -45,11 +50,10 @@ export const eventHandler = async (request, response) => { throw new CustomError(400, 'Bad request: Wrong Pub/Sub message format'); } - const storeKey = request.params.id; + const storeKey = request.params.storeKey; await syncProducts(storeKey); } catch (err) { logger.error(err); - return response.status(err.statusCode).send(err); } diff --git a/full-ingestion/src/extensions/algolia-example/mappers/product.mapper.js b/full-ingestion/src/extensions/algolia-example/mappers/product.mapper.js index de75913..103f7af 100644 --- a/full-ingestion/src/extensions/algolia-example/mappers/product.mapper.js +++ b/full-ingestion/src/extensions/algolia-example/mappers/product.mapper.js @@ -1,6 +1,23 @@ export default function map(product) { let mappedProduct = {}; + let categories = product.obj.masterData.current.categories.map((category) => { + return { + key: category.obj.key, + name: category.obj.name, + slug: category.obj.slug, + }; + }); + mappedProduct.objectID = product.id; - mappedProduct.masterData = product.masterData; + mappedProduct.productType = product.obj.productType.obj; + mappedProduct.taxCategory = product.obj.taxCategory.obj; + mappedProduct.masterData = { + current: { + categories, + name: product.obj.masterData.current.name, + slug: product.obj.masterData.current.slug, + metaTitle: product.obj.masterData.current.metaTitle, + }, + }; return mappedProduct; } diff --git a/full-ingestion/src/routes/event.route.js b/full-ingestion/src/routes/event.route.js index 1653a7d..74f8b5f 100644 --- a/full-ingestion/src/routes/event.route.js +++ b/full-ingestion/src/routes/event.route.js @@ -4,6 +4,6 @@ import { eventHandler } from '../controllers/event.controller.js'; const eventRouter = Router(); -eventRouter.post('/:id', eventHandler); +eventRouter.post('/:storeKey', eventHandler); export default eventRouter; From 9c532551a1a862537470ea8e036e076d733ebafb Mon Sep 17 00:00:00 2001 From: King-Hin Leung Date: Thu, 10 Aug 2023 09:17:34 +0200 Subject: [PATCH 06/15] Update package-lock.json --- full-ingestion/package-lock.json | 86 +++++++++++++++++++++----------- 1 file changed, 58 insertions(+), 28 deletions(-) diff --git a/full-ingestion/package-lock.json b/full-ingestion/package-lock.json index 4f6a71e..cd2cefc 100644 --- a/full-ingestion/package-lock.json +++ b/full-ingestion/package-lock.json @@ -16,7 +16,6 @@ "body-parser": "^1.20.1", "dotenv": "^16.0.3", "express": "4.18.2", - "lodash": "^4.17.21", "validator": "^13.7.0" }, "devDependencies": { @@ -1933,6 +1932,25 @@ "node": ">=14" } }, + "node_modules/@commercetools/sdk-client-v2/node_modules/node-fetch": { + "version": "2.6.12", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.12.tgz", + "integrity": "sha512-C/fGU2E8ToujUivIO0H+tpQ6HWo4eEmchoPIoXtxCrVghxdKq+QOHqEZW7tuP3KlV3bC8FRMO5nMCC7Zm1VP6g==", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, "node_modules/@commercetools/sdk-middleware-auth": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/@commercetools/sdk-middleware-auth/-/sdk-middleware-auth-7.0.1.tgz", @@ -1944,6 +1962,25 @@ "node": ">=14" } }, + "node_modules/@commercetools/sdk-middleware-auth/node_modules/node-fetch": { + "version": "2.6.12", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.12.tgz", + "integrity": "sha512-C/fGU2E8ToujUivIO0H+tpQ6HWo4eEmchoPIoXtxCrVghxdKq+QOHqEZW7tuP3KlV3bC8FRMO5nMCC7Zm1VP6g==", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, "node_modules/@commercetools/sdk-middleware-http": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/@commercetools/sdk-middleware-http/-/sdk-middleware-http-7.0.3.tgz", @@ -7095,25 +7132,6 @@ "node": ">= 0.6" } }, - "node_modules/node-fetch": { - "version": "2.6.12", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.12.tgz", - "integrity": "sha512-C/fGU2E8ToujUivIO0H+tpQ6HWo4eEmchoPIoXtxCrVghxdKq+QOHqEZW7tuP3KlV3bC8FRMO5nMCC7Zm1VP6g==", - "dependencies": { - "whatwg-url": "^5.0.0" - }, - "engines": { - "node": "4.x || >=6.0.0" - }, - "peerDependencies": { - "encoding": "^0.1.0" - }, - "peerDependenciesMeta": { - "encoding": { - "optional": true - } - } - }, "node_modules/node-int64": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz", @@ -9977,6 +9995,16 @@ "requires": { "buffer": "^6.0.3", "node-fetch": "^2.6.1" + }, + "dependencies": { + "node-fetch": { + "version": "2.6.12", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.12.tgz", + "integrity": "sha512-C/fGU2E8ToujUivIO0H+tpQ6HWo4eEmchoPIoXtxCrVghxdKq+QOHqEZW7tuP3KlV3bC8FRMO5nMCC7Zm1VP6g==", + "requires": { + "whatwg-url": "^5.0.0" + } + } } }, "@commercetools/sdk-middleware-auth": { @@ -9985,6 +10013,16 @@ "integrity": "sha512-XLRm+o3Yd0mkVoyzsOA98PUu0U0ajQdBHMhZ8N2XMOtL4OY8zsgT8ap5JneXV8zWZNiwIYYAYoUDwBlLZh2lAQ==", "requires": { "node-fetch": "^2.6.7" + }, + "dependencies": { + "node-fetch": { + "version": "2.6.12", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.12.tgz", + "integrity": "sha512-C/fGU2E8ToujUivIO0H+tpQ6HWo4eEmchoPIoXtxCrVghxdKq+QOHqEZW7tuP3KlV3bC8FRMO5nMCC7Zm1VP6g==", + "requires": { + "whatwg-url": "^5.0.0" + } + } } }, "@commercetools/sdk-middleware-http": { @@ -13841,14 +13879,6 @@ "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==" }, - "node-fetch": { - "version": "2.6.12", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.12.tgz", - "integrity": "sha512-C/fGU2E8ToujUivIO0H+tpQ6HWo4eEmchoPIoXtxCrVghxdKq+QOHqEZW7tuP3KlV3bC8FRMO5nMCC7Zm1VP6g==", - "requires": { - "whatwg-url": "^5.0.0" - } - }, "node-int64": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz", From d966094f9749e152f98c89802175b7e97aa051d9 Mon Sep 17 00:00:00 2001 From: King-Hin Leung Date: Thu, 10 Aug 2023 09:42:07 +0200 Subject: [PATCH 07/15] Delete post-deploy.js --- full-ingestion/src/connectors/post-deploy.js | 27 -------------------- 1 file changed, 27 deletions(-) delete mode 100644 full-ingestion/src/connectors/post-deploy.js diff --git a/full-ingestion/src/connectors/post-deploy.js b/full-ingestion/src/connectors/post-deploy.js deleted file mode 100644 index 1e25f64..0000000 --- a/full-ingestion/src/connectors/post-deploy.js +++ /dev/null @@ -1,27 +0,0 @@ -import fetch from 'node-fetch'; - -import { assertError, assertString } from '../utils/assert.utils.js'; - -const CONNECT_SERVICE_URL = 'CONNECT_SERVICE_URL'; -const COMMERCETOOLS_STORE_ID = 'COMMERCETOOLS_STORE_ID'; - -async function postDeploy(properties) { - const deploymentUrl = properties.get(CONNECT_SERVICE_URL); - const storeId = properties.get(COMMERCETOOLS_STORE_ID); - - const response = await fetch(deploymentUrl, { method: 'POST', body: params }); - const data = await response.json(); -} - -async function run() { - try { - const properties = new Map(Object.entries(process.env)); - await postDeploy(properties); - } catch (error) { - assertError(error); - process.stderr.write(`Post-deploy failed: ${error.message}\n`); - process.exitCode = 1; - } -} - -run(); From b6da7dc5247714f53bf268e417e56d9ce9126262 Mon Sep 17 00:00:00 2001 From: King-Hin Leung Date: Thu, 10 Aug 2023 11:21:43 +0200 Subject: [PATCH 08/15] Update connect.yaml --- connect.yaml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/connect.yaml b/connect.yaml index b9c7fa6..45b2a3b 100644 --- a/connect.yaml +++ b/connect.yaml @@ -6,8 +6,18 @@ deployAs: postDeploy: npm install configuration: securedConfiguration: - - key: COMMERCETOOLS_STORE_ID - description: Unique identifier of Commercetools Store + - key: COMMERCETOOLS_STORE_KEY + description: Unique key of Commercetools Store + - key: CTP_PROJECT_KEY + description: Commercetools project key + - key: CTP_CLIENT_ID + description: Commercetools client ID + - key: CTP_CLIENT_SECRET + description: Commercetools client secreet + - key: CTP_SCOPE + description: Commercetools client scope + - key: CTP_REGION + description: Region of Commercetools project - name: incremental-updater applicationType: event endpoint: / From 5031a5ef83986a62a18fd00afaad8de894d5ee56 Mon Sep 17 00:00:00 2001 From: King-Hin Leung Date: Thu, 10 Aug 2023 12:13:19 +0200 Subject: [PATCH 09/15] Update connect.yaml and edit config inside algolia-sample extension --- connect.yaml | 2 ++ .../algolia-example/configurations/config.js | 16 +++++++++++++--- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/connect.yaml b/connect.yaml index 45b2a3b..84d25d3 100644 --- a/connect.yaml +++ b/connect.yaml @@ -18,6 +18,8 @@ deployAs: description: Commercetools client scope - key: CTP_REGION description: Region of Commercetools project + - key: SEARCH_PLATFORM_CONFIG + description: Escaped JSON object including credentails to search platform and other settings - name: incremental-updater applicationType: event endpoint: / diff --git a/full-ingestion/src/extensions/algolia-example/configurations/config.js b/full-ingestion/src/extensions/algolia-example/configurations/config.js index 9712e9b..5c3717a 100644 --- a/full-ingestion/src/extensions/algolia-example/configurations/config.js +++ b/full-ingestion/src/extensions/algolia-example/configurations/config.js @@ -1,5 +1,15 @@ +function loadConfig() { + try { + return JSON.parse(process.env.SEARCH_PLATFORM_CONFIG); + } catch (e) { + throw new Error( + 'Search platform configuration is not provided in the JSON format' + ); + } +} + export const config = { - applicationId: process.env.AGOLIA_APPLICATION_ID, - searchApiKey: process.env.AGOLIA_SEARCH_API_KEY, - index: process.env.AGOLIA_INDEX, + applicationId: loadConfig().applicationId, + searchApiKey: loadConfig().searchApiKey, + index: loadConfig().index, }; From c9bb5c322e1bf27dc04fc9ab18de8b41fd05e396 Mon Sep 17 00:00:00 2001 From: King-Hin Leung Date: Thu, 10 Aug 2023 12:40:39 +0200 Subject: [PATCH 10/15] Add validators for environment variables --- full-ingestion/package-lock.json | 14 +- full-ingestion/package.json | 2 +- .../algolia-example/configurations/config.js | 6 +- .../validators/helpers.validators.js | 141 ++++++++++++++++++ full-ingestion/src/utils/config.utils.js | 24 +-- .../src/validators/env-var.validators.js | 55 +++++++ .../src/validators/helpers.validators.js | 141 ++++++++++++++++++ 7 files changed, 360 insertions(+), 23 deletions(-) create mode 100644 full-ingestion/src/middlewares/validators/helpers.validators.js create mode 100644 full-ingestion/src/validators/env-var.validators.js create mode 100644 full-ingestion/src/validators/helpers.validators.js diff --git a/full-ingestion/package-lock.json b/full-ingestion/package-lock.json index cd2cefc..bf7b4d2 100644 --- a/full-ingestion/package-lock.json +++ b/full-ingestion/package-lock.json @@ -16,7 +16,7 @@ "body-parser": "^1.20.1", "dotenv": "^16.0.3", "express": "4.18.2", - "validator": "^13.7.0" + "validator": "^13.11.0" }, "devDependencies": { "@babel/preset-env": "^7.21.5", @@ -8432,9 +8432,9 @@ } }, "node_modules/validator": { - "version": "13.9.0", - "resolved": "https://registry.npmjs.org/validator/-/validator-13.9.0.tgz", - "integrity": "sha512-B+dGG8U3fdtM0/aNK4/X8CXq/EcxU2WPrPEkJGslb47qyHsxmbggTWK0yEA4qnYVNF+nxNlN88o14hIcPmSIEA==", + "version": "13.11.0", + "resolved": "https://registry.npmjs.org/validator/-/validator-13.11.0.tgz", + "integrity": "sha512-Ii+sehpSfZy+At5nPdnyMhx78fEoPDkR2XW/zimHEL3MyGJQOCQ7WeP20jPYRz7ZCpcKLB21NxuXHF3bxjStBQ==", "engines": { "node": ">= 0.10" } @@ -14820,9 +14820,9 @@ } }, "validator": { - "version": "13.9.0", - "resolved": "https://registry.npmjs.org/validator/-/validator-13.9.0.tgz", - "integrity": "sha512-B+dGG8U3fdtM0/aNK4/X8CXq/EcxU2WPrPEkJGslb47qyHsxmbggTWK0yEA4qnYVNF+nxNlN88o14hIcPmSIEA==" + "version": "13.11.0", + "resolved": "https://registry.npmjs.org/validator/-/validator-13.11.0.tgz", + "integrity": "sha512-Ii+sehpSfZy+At5nPdnyMhx78fEoPDkR2XW/zimHEL3MyGJQOCQ7WeP20jPYRz7ZCpcKLB21NxuXHF3bxjStBQ==" }, "vary": { "version": "1.1.2", diff --git a/full-ingestion/package.json b/full-ingestion/package.json index 33d8fe8..fc49a47 100644 --- a/full-ingestion/package.json +++ b/full-ingestion/package.json @@ -32,6 +32,6 @@ "body-parser": "^1.20.1", "dotenv": "^16.0.3", "express": "4.18.2", - "validator": "^13.7.0" + "validator": "^13.11.0" } } diff --git a/full-ingestion/src/extensions/algolia-example/configurations/config.js b/full-ingestion/src/extensions/algolia-example/configurations/config.js index 5c3717a..bafd88e 100644 --- a/full-ingestion/src/extensions/algolia-example/configurations/config.js +++ b/full-ingestion/src/extensions/algolia-example/configurations/config.js @@ -9,7 +9,7 @@ function loadConfig() { } export const config = { - applicationId: loadConfig().applicationId, - searchApiKey: loadConfig().searchApiKey, - index: loadConfig().index, + applicationId: loadConfig()?.applicationId, + searchApiKey: loadConfig()?.searchApiKey, + index: loadConfig()?.index, }; diff --git a/full-ingestion/src/middlewares/validators/helpers.validators.js b/full-ingestion/src/middlewares/validators/helpers.validators.js new file mode 100644 index 0000000..211a141 --- /dev/null +++ b/full-ingestion/src/middlewares/validators/helpers.validators.js @@ -0,0 +1,141 @@ +import validator from 'validator'; + +/** + * File used to create helpers to validate the fields + */ + +const required = + (fn) => + (value, ...args) => + !(value === undefined || value === null) && fn(...[String(value), ...args]); + +export const standardString = (path, message, overrideConfig = {}) => [ + path, + [ + [ + required(validator.isLength), + message, + [{ min: 2, max: 20, ...overrideConfig }], + ], + ], +]; + +export const standardEmail = (path, message) => [ + path, + [[required(validator.isEmail), message]], +]; + +export const standardNaturalNumber = (path, message) => [ + path, + [ + [ + required((value) => + validator.isNumeric(String(value), { no_symbols: true }) + ), + message, + ], + ], +]; + +export const standardKey = (path, message) => [ + path, + [ + [ + required( + (value) => + validator.isLength(String(value), { min: 2 }) && + /^[a-zA-Z0-9-_]+$/.test(value) + ), + + message, + ], + ], +]; + +export const standardUrl = (path, message, overrideOptions = {}) => [ + path, + [ + [ + required(validator.isURL), + message, + [ + { + require_protocol: true, + require_valid_protocol: true, + protocols: ['http', 'https'], + require_host: true, + require_port: false, + allow_protocol_relative_urls: false, + allow_fragments: false, + allow_query_components: true, + validate_length: true, + ...overrideOptions, + }, + ], + ], + ], +]; + +export const getValidateMessages = (validatorConfigs, item) => + validatorConfigs.flatMap(([path, validators]) => { + return validators.reduce((acc, [validatorFn, message, args = []]) => { + const valueToValidate = path.reduce((val, property) => { + return val[property]; + }, item); + if (!validatorFn(...[valueToValidate, ...args])) { + return acc.concat(message); + } + return acc; + }, []); + }); + +export const optional = + (fn) => + (...args) => { + const [path, validators] = fn(...args); + return [ + path, + validators.map(([fn, message, validatorArgs]) => [ + (value, ...args) => + value === undefined ? true : fn(...[value, ...args]), + message, + validatorArgs, + ]), + ]; + }; + +export const array = + (fn) => + (...args) => { + const [path, validators] = fn(...args); + return [ + path, + validators.map(([fn, message, validatorArgs]) => [ + (value, ...args) => + Array.isArray(value) && + value.every((value) => fn(...[value, ...args])), + message, + validatorArgs, + ]), + ]; + }; + +export const region = (path, message) => [ + path, + [ + [ + required( + required((value) => + validator.isIn(value, [ + 'us-central1.gcp', + 'us-east-2.aws', + 'europe-west1.gcp', + 'eu-central-1.aws', + 'australia-southeast1.gcp', + ]) + ) + ), + message, + ], + ], +]; diff --git a/full-ingestion/src/utils/config.utils.js b/full-ingestion/src/utils/config.utils.js index 99a732a..067cff1 100644 --- a/full-ingestion/src/utils/config.utils.js +++ b/full-ingestion/src/utils/config.utils.js @@ -1,6 +1,6 @@ -// import CustomError from '../errors/custom.error.js'; -// import envValidators from '../validators/env.validators.js'; -// import { getValidateMessages } from '../validators/helpers.validators.js'; +import CustomError from '../errors/custom.error.js'; +import envValidators from '../validators/env-var.validators.js'; +import { getValidateMessages } from '../validators/helpers.validators.js'; /** * Read the configuration env vars @@ -18,15 +18,15 @@ export const readConfiguration = () => { region: process.env.CTP_REGION, }; - // const validationErrors = getValidateMessages(envValidators, envVars); - // - // if (validationErrors.length) { - // throw new CustomError( - // 'InvalidEnvironmentVariablesError', - // 'Invalid Environment Variables please check your .env file', - // validationErrors - // ); - // } + const validationErrors = getValidateMessages(envValidators, envVars); + + if (validationErrors.length) { + throw new CustomError( + 'InvalidEnvironmentVariablesError', + 'Invalid Environment Variables please check your .env file', + validationErrors + ); + } return envVars; }; diff --git a/full-ingestion/src/validators/env-var.validators.js b/full-ingestion/src/validators/env-var.validators.js new file mode 100644 index 0000000..3f211b2 --- /dev/null +++ b/full-ingestion/src/validators/env-var.validators.js @@ -0,0 +1,55 @@ +import { + optional, + standardString, + standardKey, + region, +} from './helpers.validators.js'; + +/** + * Create here your own validators + */ +const envValidators = [ + standardString( + ['clientId'], + { + code: 'InValidClientId', + message: 'Client id should be 24 characters.', + referencedBy: 'environmentVariables', + }, + { min: 24, max: 24 } + ), + + standardString( + ['clientSecret'], + { + code: 'InvalidClientSecret', + message: 'Client secret should be 32 characters.', + referencedBy: 'environmentVariables', + }, + { min: 32, max: 32 } + ), + + standardKey(['projectKey'], { + code: 'InvalidProjectKey', + message: 'Project key should be a valid string.', + referencedBy: 'environmentVariables', + }), + + optional(standardString)( + ['scope'], + { + code: 'InvalidScope', + message: 'Scope should be at least 2 characters long.', + referencedBy: 'environmentVariables', + }, + { min: 2, max: undefined } + ), + + region(['region'], { + code: 'InvalidRegion', + message: 'Not a valid region.', + referencedBy: 'environmentVariables', + }), +]; + +export default envValidators; diff --git a/full-ingestion/src/validators/helpers.validators.js b/full-ingestion/src/validators/helpers.validators.js new file mode 100644 index 0000000..211a141 --- /dev/null +++ b/full-ingestion/src/validators/helpers.validators.js @@ -0,0 +1,141 @@ +import validator from 'validator'; + +/** + * File used to create helpers to validate the fields + */ + +const required = + (fn) => + (value, ...args) => + !(value === undefined || value === null) && fn(...[String(value), ...args]); + +export const standardString = (path, message, overrideConfig = {}) => [ + path, + [ + [ + required(validator.isLength), + message, + [{ min: 2, max: 20, ...overrideConfig }], + ], + ], +]; + +export const standardEmail = (path, message) => [ + path, + [[required(validator.isEmail), message]], +]; + +export const standardNaturalNumber = (path, message) => [ + path, + [ + [ + required((value) => + validator.isNumeric(String(value), { no_symbols: true }) + ), + message, + ], + ], +]; + +export const standardKey = (path, message) => [ + path, + [ + [ + required( + (value) => + validator.isLength(String(value), { min: 2 }) && + /^[a-zA-Z0-9-_]+$/.test(value) + ), + + message, + ], + ], +]; + +export const standardUrl = (path, message, overrideOptions = {}) => [ + path, + [ + [ + required(validator.isURL), + message, + [ + { + require_protocol: true, + require_valid_protocol: true, + protocols: ['http', 'https'], + require_host: true, + require_port: false, + allow_protocol_relative_urls: false, + allow_fragments: false, + allow_query_components: true, + validate_length: true, + ...overrideOptions, + }, + ], + ], + ], +]; + +export const getValidateMessages = (validatorConfigs, item) => + validatorConfigs.flatMap(([path, validators]) => { + return validators.reduce((acc, [validatorFn, message, args = []]) => { + const valueToValidate = path.reduce((val, property) => { + return val[property]; + }, item); + if (!validatorFn(...[valueToValidate, ...args])) { + return acc.concat(message); + } + return acc; + }, []); + }); + +export const optional = + (fn) => + (...args) => { + const [path, validators] = fn(...args); + return [ + path, + validators.map(([fn, message, validatorArgs]) => [ + (value, ...args) => + value === undefined ? true : fn(...[value, ...args]), + message, + validatorArgs, + ]), + ]; + }; + +export const array = + (fn) => + (...args) => { + const [path, validators] = fn(...args); + return [ + path, + validators.map(([fn, message, validatorArgs]) => [ + (value, ...args) => + Array.isArray(value) && + value.every((value) => fn(...[value, ...args])), + message, + validatorArgs, + ]), + ]; + }; + +export const region = (path, message) => [ + path, + [ + [ + required( + required((value) => + validator.isIn(value, [ + 'us-central1.gcp', + 'us-east-2.aws', + 'europe-west1.gcp', + 'eu-central-1.aws', + 'australia-southeast1.gcp', + ]) + ) + ), + message, + ], + ], +]; From 9b698e8218bbed521367165a8bfbfe277e73a707 Mon Sep 17 00:00:00 2001 From: King-Hin Leung Date: Thu, 10 Aug 2023 16:46:30 +0200 Subject: [PATCH 11/15] Update event.controller.js --- .../src/controllers/event.controller.js | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/full-ingestion/src/controllers/event.controller.js b/full-ingestion/src/controllers/event.controller.js index e13963a..88024e0 100644 --- a/full-ingestion/src/controllers/event.controller.js +++ b/full-ingestion/src/controllers/event.controller.js @@ -34,23 +34,15 @@ async function getProductsByStore(storeKey) { } export const eventHandler = async (request, response) => { - // Check request body try { - if (!request.body) { - logger.error('Missing request body.'); + const storeKey = request.params.storeKey; + if (!storeKey) { + logger.error('Missing store key in query parameter.'); throw new CustomError( 400, - 'Bad request: No Pub/Sub message was received' + 'Bad request: No store key is defined in query parameter' ); } - - // Check if the body comes in a message - if (!request.body.message) { - logger.error('Missing body message'); - throw new CustomError(400, 'Bad request: Wrong Pub/Sub message format'); - } - - const storeKey = request.params.storeKey; await syncProducts(storeKey); } catch (err) { logger.error(err); From e6eea6f7a4917934221996490e1744921c46fd08 Mon Sep 17 00:00:00 2001 From: King-Hin Leung Date: Mon, 14 Aug 2023 10:20:29 +0200 Subject: [PATCH 12/15] Update event.controller.js --- .../src/controllers/event.controller.js | 60 +++++++++++++------ 1 file changed, 41 insertions(+), 19 deletions(-) diff --git a/full-ingestion/src/controllers/event.controller.js b/full-ingestion/src/controllers/event.controller.js index 88024e0..daa8de8 100644 --- a/full-ingestion/src/controllers/event.controller.js +++ b/full-ingestion/src/controllers/event.controller.js @@ -3,6 +3,8 @@ import { createApiRoot } from '../clients/create.client.js'; import CustomError from '../errors/custom.error.js'; import { default as saveProducts } from '../extensions/algolia-example/clients/client.js'; +const CHUNK_SIZE = 100; + async function syncProducts(storeKey) { const products = await getProductsByStore(storeKey); @@ -12,25 +14,45 @@ async function syncProducts(storeKey) { } async function getProductsByStore(storeKey) { - return await createApiRoot() - .inStoreKeyWithStoreKeyValue({ storeKey: Buffer.from(storeKey).toString() }) - .productSelectionAssignments() - .get({ - queryArgs: { - expand: [ - 'product', - 'product.productType', - 'product.taxCategory', - 'product.masterData.current.categories[*]', - ], - }, - }) - .execute() - .then((response) => response.body.results) - .then((results) => results.map((result) => result.product)) - .catch((error) => { - throw new CustomError(400, `Bad request: ${error.message}`, error); - }); + let lastProductId = undefined; + let hasNextQuery = true; + let allProducts = []; + + while (hasNextQuery) { + let queryArgs = { + limit: CHUNK_SIZE, + withTotal: false, + sort: 'product.id asc', + expand: [ + 'product', + 'product.productType', + 'product.taxCategory', + 'product.masterData.current.categories[*]', + ], + }; + if (lastProductId) { + queryArgs.where = `product(id>"${lastProductId}")`; + } + + let productChunk = await createApiRoot() + .inStoreKeyWithStoreKeyValue({ + storeKey: Buffer.from(storeKey).toString(), + }) + .productSelectionAssignments() + .get({ queryArgs }) + .execute() + .then((response) => response.body.results) + .then((results) => results.map((result) => result.product)) + .catch((error) => { + throw new CustomError(400, `Bad request: ${error.message}`, error); + }); + hasNextQuery = productChunk.length == CHUNK_SIZE; + if (productChunk.length > 0) { + lastProductId = productChunk[productChunk.length - 1].id; + allProducts = allProducts.concat(productChunk); + } + } + return allProducts; } export const eventHandler = async (request, response) => { From 51879793fc05d995c0654ecf5854a26932591634 Mon Sep 17 00:00:00 2001 From: King-Hin Leung Date: Mon, 14 Aug 2023 10:28:11 +0200 Subject: [PATCH 13/15] Update REST api resources name in URL --- full-ingestion/src/routes/event.route.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/full-ingestion/src/routes/event.route.js b/full-ingestion/src/routes/event.route.js index 74f8b5f..89d1100 100644 --- a/full-ingestion/src/routes/event.route.js +++ b/full-ingestion/src/routes/event.route.js @@ -4,6 +4,6 @@ import { eventHandler } from '../controllers/event.controller.js'; const eventRouter = Router(); -eventRouter.post('/:storeKey', eventHandler); +eventRouter.post('/fullSync/:storeKey', eventHandler); export default eventRouter; From 9faeef857f2676f3a2800f6c18358d2898c00d45 Mon Sep 17 00:00:00 2001 From: King-Hin Leung Date: Mon, 14 Aug 2023 10:36:20 +0200 Subject: [PATCH 14/15] Rename files and variables --- .../{event.controller.js => sync.controller.js} | 2 +- full-ingestion/src/index.js | 4 ++-- full-ingestion/src/routes/event.route.js | 9 --------- full-ingestion/src/routes/sync.route.js | 9 +++++++++ 4 files changed, 12 insertions(+), 12 deletions(-) rename full-ingestion/src/controllers/{event.controller.js => sync.controller.js} (97%) delete mode 100644 full-ingestion/src/routes/event.route.js create mode 100644 full-ingestion/src/routes/sync.route.js diff --git a/full-ingestion/src/controllers/event.controller.js b/full-ingestion/src/controllers/sync.controller.js similarity index 97% rename from full-ingestion/src/controllers/event.controller.js rename to full-ingestion/src/controllers/sync.controller.js index daa8de8..d52fc98 100644 --- a/full-ingestion/src/controllers/event.controller.js +++ b/full-ingestion/src/controllers/sync.controller.js @@ -55,7 +55,7 @@ async function getProductsByStore(storeKey) { return allProducts; } -export const eventHandler = async (request, response) => { +export const syncHandler = async (request, response) => { try { const storeKey = request.params.storeKey; if (!storeKey) { diff --git a/full-ingestion/src/index.js b/full-ingestion/src/index.js index 8a04059..33b15fd 100644 --- a/full-ingestion/src/index.js +++ b/full-ingestion/src/index.js @@ -4,7 +4,7 @@ import express from 'express'; import bodyParser from 'body-parser'; // Import routes -import EventRoutes from './routes/event.route.js'; +import SyncRoutes from './routes/sync.route.js'; import { logger } from './utils/logger.utils.js'; const PORT = 8080; @@ -17,7 +17,7 @@ app.use(bodyParser.json()); app.use(bodyParser.urlencoded({ extended: true })); // Define routes -app.use('/', EventRoutes); +app.use('/', SyncRoutes); // Listen the application const server = app.listen(PORT, () => { diff --git a/full-ingestion/src/routes/event.route.js b/full-ingestion/src/routes/event.route.js deleted file mode 100644 index 89d1100..0000000 --- a/full-ingestion/src/routes/event.route.js +++ /dev/null @@ -1,9 +0,0 @@ -import { Router } from 'express'; - -import { eventHandler } from '../controllers/event.controller.js'; - -const eventRouter = Router(); - -eventRouter.post('/fullSync/:storeKey', eventHandler); - -export default eventRouter; diff --git a/full-ingestion/src/routes/sync.route.js b/full-ingestion/src/routes/sync.route.js new file mode 100644 index 0000000..b1cb54b --- /dev/null +++ b/full-ingestion/src/routes/sync.route.js @@ -0,0 +1,9 @@ +import { Router } from 'express'; + +import { syncHandler } from '../controllers/sync.controller.js'; + +const syncRouter = Router(); + +syncRouter.post('/fullSync/:storeKey', syncHandler); + +export default syncRouter; From 4765ad90ff91c7b9ab165cb452ee6e9f09f7f935 Mon Sep 17 00:00:00 2001 From: King-Hin Leung Date: Mon, 14 Aug 2023 12:04:20 +0200 Subject: [PATCH 15/15] Update build.client.js --- full-ingestion/src/clients/build.client.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/full-ingestion/src/clients/build.client.js b/full-ingestion/src/clients/build.client.js index 07c1f74..993bb2f 100644 --- a/full-ingestion/src/clients/build.client.js +++ b/full-ingestion/src/clients/build.client.js @@ -5,7 +5,7 @@ import { readConfiguration } from '../utils/config.utils.js'; /** * Create a new client builder. - * This code creates a new Client that can be used to make API calls + * This code creates a new client builder that can be used to make API calls */ export const createClient = () => new ClientBuilder()