feat(pulumi-aws): add shared indexes param (#4306)

webiny · Oct 2, 2024 · 73996eb · 73996eb
1 parent f5ade1a
commit 73996eb
Show file tree

Hide file tree

Showing 29 changed files with 133 additions and 75 deletions.
diff --git a/packages/api-dynamodb-to-elasticsearch/jest.config.js b/packages/api-dynamodb-to-elasticsearch/jest.config.js
@@ -1,6 +1,7 @@
 const base = require("../../jest.config.base");
+const { getElasticsearchIndexPrefix } = require("@webiny/api-elasticsearch");
 
-const prefix = process.env.ELASTIC_SEARCH_INDEX_PREFIX || "";
+const prefix = getElasticsearchIndexPrefix();
 process.env.ELASTIC_SEARCH_INDEX_PREFIX = `${prefix}api-elasticsearch-`;
 
 module.exports = base({ path: __dirname });
diff --git a/packages/api-elasticsearch-tasks/src/settings/EnableIndexing.ts b/packages/api-elasticsearch-tasks/src/settings/EnableIndexing.ts
@@ -11,10 +11,11 @@ export class EnableIndexing {
 
     public async exec(index: string, settings: IIndexSettingsValues): Promise<void> {
         try {
+            const refreshInterval = parseInt(settings.refreshInterval || "", 10) || 0;
             await this.settings.setSettings(index, {
                 ...settings,
                 numberOfReplicas: settings.numberOfReplicas < 1 ? 1 : settings.numberOfReplicas,
-                refreshInterval: settings.refreshInterval === "-1" ? "1s" : settings.refreshInterval
+                refreshInterval: refreshInterval <= 0 ? "1s" : settings.refreshInterval
             });
         } catch (ex) {
             throw new IndexingEnableError(ex);

diff --git a/packages/api-elasticsearch-tasks/src/tasks/createIndexes/CreateIndexesTaskRunner.ts b/packages/api-elasticsearch-tasks/src/tasks/createIndexes/CreateIndexesTaskRunner.ts
@@ -40,7 +40,12 @@ export class CreateIndexesTaskRunner {
                         tenant: tenant.id,
                         locale: locale.code
                     });
-                    indexes.push(...results);
+                    for (const result of results) {
+                        if (indexes.some(i => i.index === result.index)) {
+                            continue;
+                        }
+                        indexes.push(result);
+                    }
                 }
             }
         }

diff --git a/packages/api-elasticsearch/__tests__/index.test.ts b/packages/api-elasticsearch/__tests__/index.test.ts
@@ -1,6 +1,7 @@
 import { getBaseConfiguration, getJapaneseConfiguration } from "~/indexConfiguration";
 import { ElasticsearchIndexRequestBody } from "~/types";
 import { createElasticsearchClient } from "./helpers";
+import { getElasticsearchIndexPrefix } from "~/indexPrefix";
 
 /**
  * Add configurations when added to the code.
@@ -13,7 +14,7 @@ const settings: [string, ElasticsearchIndexRequestBody][] = [
 describe("Elasticsearch Index Mapping And Settings", () => {
     const client = createElasticsearchClient();
 
-    const prefix: string = process.env.ELASTIC_SEARCH_INDEX_PREFIX || "";
+    const prefix: string = getElasticsearchIndexPrefix();
 
     const testIndexName = `${prefix}dummy-index-test`;
 

diff --git a/packages/api-elasticsearch/__tests__/index/japanese.test.ts b/packages/api-elasticsearch/__tests__/index/japanese.test.ts
@@ -1,10 +1,11 @@
 import { getJapaneseConfiguration } from "~/indexConfiguration";
 import { createElasticsearchClient } from "../helpers";
+import { getElasticsearchIndexPrefix } from "~/indexPrefix";
 
 describe("Elasticsearch Japanese", () => {
     const client = createElasticsearchClient();
 
-    const prefix: string = process.env.ELASTIC_SEARCH_INDEX_PREFIX || "";
+    const prefix: string = getElasticsearchIndexPrefix();
 
     const indexTestName = `${prefix}index-japanese-index-test`;
 

diff --git a/packages/api-elasticsearch/__tests__/search/base.test.ts b/packages/api-elasticsearch/__tests__/search/base.test.ts
@@ -3,11 +3,12 @@ import { people } from "./base.entries";
 import { getBaseConfiguration } from "~/indexConfiguration";
 import { ElasticsearchBoolQueryConfig } from "~/types";
 import { ElasticsearchQueryBuilderOperatorContainsPlugin } from "~/plugins/operator/contains";
+import { getElasticsearchIndexPrefix } from "~/indexPrefix";
 
 describe("Elasticsearch Base Search", () => {
     const client = createElasticsearchClient();
 
-    const prefix: string = process.env.ELASTIC_SEARCH_INDEX_PREFIX || "";
+    const prefix = getElasticsearchIndexPrefix();
 
     const indexTestName = `${prefix}search-base-index-test`;
 

diff --git a/packages/api-elasticsearch/__tests__/search/japanese.test.ts b/packages/api-elasticsearch/__tests__/search/japanese.test.ts
@@ -5,11 +5,12 @@ import { ElasticsearchBoolQueryConfig } from "~/types";
 import { entries, searchTargets } from "./japanese.entries";
 import * as RequestParams from "@elastic/elasticsearch/api/requestParams";
 import WebinyError from "@webiny/error";
+import { getElasticsearchIndexPrefix } from "~/indexPrefix";
 
 describe("Japanese search", () => {
     const client = createElasticsearchClient();
 
-    const prefix: string = process.env.ELASTIC_SEARCH_INDEX_PREFIX || "";
+    const prefix = getElasticsearchIndexPrefix();
 
     const indexName = `${prefix}search-japanese-index-test`;
 

diff --git a/packages/api-elasticsearch/jest.config.js b/packages/api-elasticsearch/jest.config.js
@@ -1,6 +1,8 @@
 const base = require("../../jest.config.base");
+const { getElasticsearchIndexPrefix } = require("@webiny/api-elasticsearch");
+
+const prefix = getElasticsearchIndexPrefix();
 
-const prefix = process.env.ELASTIC_SEARCH_INDEX_PREFIX || "";
 process.env.ELASTIC_SEARCH_INDEX_PREFIX = `${prefix}api-elasticsearch-`;
 
 module.exports = base({ path: __dirname });
diff --git a/packages/api-elasticsearch/src/index.ts b/packages/api-elasticsearch/src/index.ts
@@ -21,6 +21,8 @@ export * from "./cursors";
 export * from "./client";
 export * from "./utils";
 export * from "./operations";
+export * from "./sharedIndex";
+export * from "./indexPrefix";
 export { createGzipCompression } from "./plugins/GzipCompression";
 
 /**

diff --git a/packages/api-elasticsearch/src/indexPrefix.ts b/packages/api-elasticsearch/src/indexPrefix.ts
@@ -0,0 +1,7 @@
+export const getElasticsearchIndexPrefix = (): string => {
+    return (
+        process.env.ELASTIC_SEARCH_INDEX_PREFIX ||
+        process.env.WEBINY_ELASTIC_SEARCH_INDEX_PREFIX ||
+        ""
+    );
+};
diff --git a/packages/api-elasticsearch/src/sharedIndex.ts b/packages/api-elasticsearch/src/sharedIndex.ts
@@ -0,0 +1,3 @@
+export const isSharedElasticsearchIndex = () => {
+    return process.env.ELASTICSEARCH_SHARED_INDEXES === "true";
+};
diff --git a/packages/api-form-builder-so-ddb-es/__tests__/__api__/setupFile.js b/packages/api-form-builder-so-ddb-es/__tests__/__api__/setupFile.js
@@ -6,8 +6,9 @@ const { configurations } = require("../../dist/configurations");
 const { base: baseConfigurationPlugin } = require("../../dist/elasticsearch/indices/base");
 const { setStorageOps } = require("@webiny/project-utils/testing/environment");
 const { getElasticsearchClient } = require("@webiny/project-utils/testing/elasticsearch");
+const { getElasticsearchIndexPrefix } = require("@webiny/api-elasticsearch");
 
-const prefix = process.env.ELASTIC_SEARCH_INDEX_PREFIX || "";
+const prefix = getElasticsearchIndexPrefix();
 if (!prefix.includes("api-")) {
     process.env.ELASTIC_SEARCH_INDEX_PREFIX = `${prefix}api-form-builder-env-`;
 }

diff --git a/packages/api-form-builder-so-ddb-es/__tests__/elasticsearchIndex.test.ts b/packages/api-form-builder-so-ddb-es/__tests__/elasticsearchIndex.test.ts
@@ -1,4 +1,5 @@
 import { configurations } from "~/configurations";
+import { getElasticsearchIndexPrefix } from "@webiny/api-elasticsearch";
 
 describe("Elasticsearch index", () => {
     const withLocaleItems = [
@@ -18,7 +19,7 @@ describe("Elasticsearch index", () => {
         async (tenant, locale) => {
             process.env.WEBINY_ELASTICSEARCH_INDEX_LOCALE = "true";
 
-            const prefix = process.env.ELASTIC_SEARCH_INDEX_PREFIX || "";
+            const prefix = getElasticsearchIndexPrefix();
 
             const { index } = configurations.es({
                 tenant,
@@ -32,7 +33,7 @@ describe("Elasticsearch index", () => {
     it.each(withLocaleItems)(
         "should create index without locale code as part of the name",
         async (tenant, locale) => {
-            const prefix = process.env.ELASTIC_SEARCH_INDEX_PREFIX || "";
+            const prefix = getElasticsearchIndexPrefix();
 
             const { index } = configurations.es({
                 tenant,
@@ -82,7 +83,7 @@ describe("Elasticsearch index", () => {
         async (tenant, locale) => {
             process.env.ELASTICSEARCH_SHARED_INDEXES = "true";
 
-            const prefix = process.env.ELASTIC_SEARCH_INDEX_PREFIX || "";
+            const prefix = getElasticsearchIndexPrefix();
 
             const { index: noLocaleIndex } = configurations.es({
                 tenant,
@@ -98,7 +99,7 @@ describe("Elasticsearch index", () => {
             process.env.ELASTICSEARCH_SHARED_INDEXES = "true";
             process.env.WEBINY_ELASTICSEARCH_INDEX_LOCALE = "true";
 
-            const prefix = process.env.ELASTIC_SEARCH_INDEX_PREFIX || "";
+            const prefix = getElasticsearchIndexPrefix();
 
             const { index: noLocaleIndex } = configurations.es({
                 tenant,

diff --git a/packages/api-form-builder-so-ddb-es/jest.config.js b/packages/api-form-builder-so-ddb-es/jest.config.js
@@ -1,6 +1,7 @@
 const base = require("../../jest.config.base");
+const { getElasticsearchIndexPrefix } = require("@webiny/api-elasticsearch");
 
-const prefix = process.env.ELASTIC_SEARCH_INDEX_PREFIX || "";
+const prefix = getElasticsearchIndexPrefix();
 process.env.ELASTIC_SEARCH_INDEX_PREFIX = `${prefix}api-form-builder-`;
 
 module.exports = {

diff --git a/packages/api-form-builder-so-ddb-es/src/configurations.ts b/packages/api-form-builder-so-ddb-es/src/configurations.ts
@@ -1,5 +1,9 @@
 import WebinyError from "@webiny/error";
-import { getLastAddedIndexPlugin } from "@webiny/api-elasticsearch";
+import {
+    getElasticsearchIndexPrefix,
+    getLastAddedIndexPlugin,
+    isSharedElasticsearchIndex
+} from "@webiny/api-elasticsearch";
 import { FormElasticsearchIndexPlugin } from "~/plugins";
 import { ElasticsearchIndexRequestBody } from "@webiny/api-elasticsearch/types";
 import { FormBuilderContext } from "@webiny/api-form-builder/types";
@@ -35,7 +39,7 @@ export const configurations: Configurations = {
             );
         }
 
-        const sharedIndex = process.env.ELASTICSEARCH_SHARED_INDEXES === "true";
+        const sharedIndex = isSharedElasticsearchIndex();
 
         const tenantId = sharedIndex ? "root" : tenant;
         let localeCode: string | null = null;
@@ -54,7 +58,7 @@ export const configurations: Configurations = {
             .join("-")
             .toLowerCase();
 
-        const prefix = process.env.ELASTIC_SEARCH_INDEX_PREFIX || "";
+        const prefix = getElasticsearchIndexPrefix();
         if (!prefix) {
             return {
                 index

diff --git a/packages/api-form-builder-so-ddb-es/src/operations/form/elasticsearchBody.ts b/packages/api-form-builder-so-ddb-es/src/operations/form/elasticsearchBody.ts
@@ -3,7 +3,8 @@ import {
     applyWhere,
     createLimit,
     createSort,
-    getElasticsearchOperatorPluginsByLocale
+    getElasticsearchOperatorPluginsByLocale,
+    isSharedElasticsearchIndex
 } from "@webiny/api-elasticsearch";
 import { ElasticsearchBoolQueryConfig } from "@webiny/api-elasticsearch/types";
 import { FormElasticsearchFieldPlugin } from "~/plugins/FormElasticsearchFieldPlugin";
@@ -56,7 +57,7 @@ const createElasticsearchQuery = (params: CreateElasticsearchQueryParams) => {
      * When ES index is shared between tenants, we need to filter records by tenant ID.
      * No need for the tenant filtering otherwise as each index is for single tenant.
      */
-    const sharedIndex = process.env.ELASTICSEARCH_SHARED_INDEXES === "true";
+    const sharedIndex = isSharedElasticsearchIndex();
     if (sharedIndex && where.tenant) {
         query.must.push({
             term: {

diff --git a/packages/api-form-builder-so-ddb-es/src/operations/submission/elasticsearchBody.ts b/packages/api-form-builder-so-ddb-es/src/operations/submission/elasticsearchBody.ts
@@ -3,7 +3,8 @@ import {
     applyWhere,
     createLimit,
     createSort,
-    ElasticsearchQueryBuilderOperatorPlugin
+    ElasticsearchQueryBuilderOperatorPlugin,
+    isSharedElasticsearchIndex
 } from "@webiny/api-elasticsearch";
 import { ElasticsearchBoolQueryConfig } from "@webiny/api-elasticsearch/types";
 import { SubmissionElasticsearchFieldPlugin } from "~/plugins/SubmissionElasticsearchFieldPlugin";
@@ -66,7 +67,7 @@ const createElasticsearchQuery = (params: CreateElasticsearchQueryParams) => {
      * When ES index is shared between tenants, we need to filter records by tenant ID.
      * No need for the tenant filtering otherwise as each index is for single tenant.
      */
-    const sharedIndex = process.env.ELASTICSEARCH_SHARED_INDEXES === "true";
+    const sharedIndex = isSharedElasticsearchIndex();
     if (sharedIndex && where.tenant) {
         query.must.push({
             term: {

diff --git a/packages/api-headless-cms-ddb-es/__tests__/__api__/setupFile.js b/packages/api-headless-cms-ddb-es/__tests__/__api__/setupFile.js
@@ -16,7 +16,9 @@ if (typeof createStorageOperations !== "function") {
     throw new Error(`Loaded plugins file must export a function that returns an array of plugins.`);
 }
 
-const prefix = process.env.ELASTIC_SEARCH_INDEX_PREFIX || "";
+const { getElasticsearchIndexPrefix } = require("@webiny/api-elasticsearch");
+
+const prefix = getElasticsearchIndexPrefix();
 if (!prefix.includes("api-")) {
     process.env.ELASTIC_SEARCH_INDEX_PREFIX = `${prefix}api-headless-cms-env-`;
 }

diff --git a/packages/api-headless-cms-ddb-es/__tests__/elasticsearchIndex.test.ts b/packages/api-headless-cms-ddb-es/__tests__/elasticsearchIndex.test.ts
@@ -1,5 +1,6 @@
 import { configurations } from "~/configurations";
 import { CmsModel } from "@webiny/api-headless-cms/types";
+import { getElasticsearchIndexPrefix } from "@webiny/api-elasticsearch";
 
 describe("Elasticsearch index", () => {
     const withLocaleItems = [
@@ -19,7 +20,7 @@ describe("Elasticsearch index", () => {
         async (tenant, locale) => {
             process.env.WEBINY_ELASTICSEARCH_INDEX_LOCALE = "true";
 
-            const prefix = process.env.ELASTIC_SEARCH_INDEX_PREFIX || "";
+            const prefix = getElasticsearchIndexPrefix();
 
             const { index } = configurations.es({
                 model: {
@@ -76,7 +77,7 @@ describe("Elasticsearch index", () => {
         async (tenant, locale) => {
             process.env.ELASTICSEARCH_SHARED_INDEXES = "true";
 
-            const prefix = process.env.ELASTIC_SEARCH_INDEX_PREFIX || "";
+            const prefix = getElasticsearchIndexPrefix();
 
             const { index: noLocaleIndex } = configurations.es({
                 model: {

diff --git a/packages/api-headless-cms-ddb-es/src/configurations.ts b/packages/api-headless-cms-ddb-es/src/configurations.ts
@@ -1,7 +1,11 @@
 import { CmsModel } from "@webiny/api-headless-cms/types";
 import WebinyError from "@webiny/error";
 import { CmsContext } from "~/types";
-import { getLastAddedIndexPlugin } from "@webiny/api-elasticsearch";
+import {
+    getElasticsearchIndexPrefix,
+    getLastAddedIndexPlugin,
+    isSharedElasticsearchIndex
+} from "@webiny/api-elasticsearch";
 import { ElasticsearchIndexRequestBody } from "@webiny/api-elasticsearch/types";
 import { CmsEntryElasticsearchIndexPlugin } from "~/plugins";
 
@@ -41,12 +45,13 @@ export const configurations: Configurations = {
             );
         }
 
-        const sharedIndex = process.env.ELASTICSEARCH_SHARED_INDEXES === "true";
+        const sharedIndex = isSharedElasticsearchIndex();
         const index = [sharedIndex ? "root" : tenant, "headless-cms", locale, model.modelId]
             .join("-")
             .toLowerCase();
 
-        const prefix = process.env.ELASTIC_SEARCH_INDEX_PREFIX || "";
+        const prefix = getElasticsearchIndexPrefix();
+
         if (!prefix) {
             return {
                 index

diff --git a/packages/api-headless-cms-ddb-es/src/operations/entry/elasticsearch/initialQuery.ts b/packages/api-headless-cms-ddb-es/src/operations/entry/elasticsearch/initialQuery.ts
@@ -2,6 +2,7 @@ import WebinyError from "@webiny/error";
 import { ElasticsearchBoolQueryConfig } from "@webiny/api-elasticsearch/types";
 import { CmsEntryListWhere, CmsModel } from "@webiny/api-headless-cms/types";
 import { createLatestRecordType, createPublishedRecordType } from "../recordType";
+import { isSharedElasticsearchIndex } from "@webiny/api-elasticsearch";
 
 export const createBaseQuery = (): ElasticsearchBoolQueryConfig => {
     return {
@@ -34,7 +35,7 @@ export const createInitialQuery = (params: Params): ElasticsearchBoolQueryConfig
      *
      * TODO determine if we want to search across tenants in shared index?
      */
-    const sharedIndex = process.env.ELASTICSEARCH_SHARED_INDEXES === "true";
+    const sharedIndex = isSharedElasticsearchIndex();
     if (sharedIndex) {
         /**
          * Tenant for the filtering is taken from the model.

diff --git a/packages/api-page-builder-so-ddb-es/__tests__/elasticsearchIndex.test.ts b/packages/api-page-builder-so-ddb-es/__tests__/elasticsearchIndex.test.ts
@@ -1,4 +1,5 @@
 import { configurations } from "~/configurations";
+import { getElasticsearchIndexPrefix } from "@webiny/api-elasticsearch";
 
 describe("Elasticsearch index", () => {
     const withLocaleItems = [
@@ -18,7 +19,7 @@ describe("Elasticsearch index", () => {
         async (tenant, locale) => {
             process.env.WEBINY_ELASTICSEARCH_INDEX_LOCALE = "true";
 
-            const prefix = process.env.ELASTIC_SEARCH_INDEX_PREFIX || "";
+            const prefix = getElasticsearchIndexPrefix();
 
             const { index } = configurations.es({
                 tenant,
@@ -32,7 +33,7 @@ describe("Elasticsearch index", () => {
     it.each(withLocaleItems)(
         "should create index without locale code as part of the name",
         async (tenant, locale) => {
-            const prefix = process.env.ELASTIC_SEARCH_INDEX_PREFIX || "";
+            const prefix = getElasticsearchIndexPrefix();
 
             const { index } = configurations.es({
                 tenant,
@@ -79,7 +80,7 @@ describe("Elasticsearch index", () => {
         async (tenant, locale) => {
             process.env.ELASTICSEARCH_SHARED_INDEXES = "true";
 
-            const prefix = process.env.ELASTIC_SEARCH_INDEX_PREFIX || "";
+            const prefix = getElasticsearchIndexPrefix();
 
             const { index: noLocaleIndex } = configurations.es({
                 tenant,
@@ -95,7 +96,7 @@ describe("Elasticsearch index", () => {
             process.env.ELASTICSEARCH_SHARED_INDEXES = "true";
             process.env.WEBINY_ELASTICSEARCH_INDEX_LOCALE = "true";
 
-            const prefix = process.env.ELASTIC_SEARCH_INDEX_PREFIX || "";
+            const prefix = getElasticsearchIndexPrefix();
 
             const { index: noLocaleIndex } = configurations.es({
                 tenant,

diff --git a/packages/api-page-builder-so-ddb-es/jest.config.js b/packages/api-page-builder-so-ddb-es/jest.config.js
@@ -1,6 +1,7 @@
 const base = require("../../jest.config.base");
+const { getElasticsearchIndexPrefix } = require("@webiny/api-elasticsearch");
 
-const prefix = process.env.ELASTIC_SEARCH_INDEX_PREFIX || "";
+const prefix = getElasticsearchIndexPrefix();
 process.env.ELASTIC_SEARCH_INDEX_PREFIX = `${prefix}api-page-builder-`;
 
 module.exports = { ...base({ path: __dirname }) };