From 3e443dabde8ee55897d49b19d88c99cff8520da5 Mon Sep 17 00:00:00 2001 From: Leonardo Giacone Date: Mon, 11 Nov 2024 09:03:05 +0100 Subject: [PATCH 01/16] fix(api-headless-cms-bulk-actions): empty trash bin processing entries in series (#4351) --- .../src/handlers/eventBridgeEventHandler.ts | 5 +- .../src/plugins/createBulkActionGraphQL.ts | 10 +- .../src/plugins/createDefaultGraphQL.ts | 10 +- .../src/tasks/createEmptyTrashBinsTask.ts | 174 ++++++++---------- .../src/types.ts | 22 ++- .../src/graphql/index.ts | 9 +- .../crud/contentModel/validateModelFields.ts | 1 + .../src/plugins/GraphQLSchemaPlugin.ts | 9 + 8 files changed, 121 insertions(+), 119 deletions(-) diff --git a/packages/api-headless-cms-bulk-actions/src/handlers/eventBridgeEventHandler.ts b/packages/api-headless-cms-bulk-actions/src/handlers/eventBridgeEventHandler.ts index 8165f7be399..b2055083d59 100644 --- a/packages/api-headless-cms-bulk-actions/src/handlers/eventBridgeEventHandler.ts +++ b/packages/api-headless-cms-bulk-actions/src/handlers/eventBridgeEventHandler.ts @@ -23,10 +23,9 @@ export const createEventBridgeHandler = () => { /** * Since the event is at the infrastructure level, it has no knowledge about tenancy. - * We loop through all tenants in the system and trigger the "EmptyTrashBins" task. + * We trigger the `hcmsEntriesEmptyTrashBins` using root tenant. */ - const tenants = await context.tenancy.listTenants(); - await context.tenancy.withEachTenant(tenants, async () => { + await context.tenancy.withRootTenant(async () => { await context.tasks.trigger({ definition: "hcmsEntriesEmptyTrashBins" }); diff --git a/packages/api-headless-cms-bulk-actions/src/plugins/createBulkActionGraphQL.ts b/packages/api-headless-cms-bulk-actions/src/plugins/createBulkActionGraphQL.ts index b7f5658f75c..bae751efa58 100644 --- a/packages/api-headless-cms-bulk-actions/src/plugins/createBulkActionGraphQL.ts +++ b/packages/api-headless-cms-bulk-actions/src/plugins/createBulkActionGraphQL.ts @@ -10,7 +10,10 @@ export interface CreateBulkActionGraphQL { export const createBulkActionGraphQL = (config: CreateBulkActionGraphQL) => { return new ContextPlugin(async context => { - if (!(await isHeadlessCmsReady(context))) { + const tenant = context.tenancy.getCurrentTenant(); + const locale = context.i18n.getContentLocale(); + + if (!locale || !(await isHeadlessCmsReady(context))) { return; } @@ -53,7 +56,10 @@ export const createBulkActionGraphQL = (config: CreateBulkActionGraphQL) => { }); } } - } + }, + isApplicable: context => + context.tenancy.getCurrentTenant().id === tenant.id && + context.i18n.getContentLocale()?.code === locale.code }); plugin.name = `headless-cms.graphql.schema.bulkAction.${model.modelId}.${config.name}`; diff --git a/packages/api-headless-cms-bulk-actions/src/plugins/createDefaultGraphQL.ts b/packages/api-headless-cms-bulk-actions/src/plugins/createDefaultGraphQL.ts index 1dba3223372..89d67fed29a 100644 --- a/packages/api-headless-cms-bulk-actions/src/plugins/createDefaultGraphQL.ts +++ b/packages/api-headless-cms-bulk-actions/src/plugins/createDefaultGraphQL.ts @@ -4,7 +4,10 @@ import { CmsGraphQLSchemaPlugin, isHeadlessCmsReady } from "@webiny/api-headless export const createDefaultGraphQL = () => { return new ContextPlugin(async context => { - if (!(await isHeadlessCmsReady(context))) { + const tenant = context.tenancy.getCurrentTenant(); + const locale = context.i18n.getContentLocale(); + + if (!locale || !(await isHeadlessCmsReady(context))) { return; } @@ -44,7 +47,10 @@ export const createDefaultGraphQL = () => { data: JSON ): BulkActionResponse } - ` + `, + isApplicable: context => + context.tenancy.getCurrentTenant().id === tenant.id && + context.i18n.getContentLocale()?.code === locale.code }); plugin.name = `headless-cms.graphql.schema.bulkAction.default.${model.modelId}`; diff --git a/packages/api-headless-cms-bulk-actions/src/tasks/createEmptyTrashBinsTask.ts b/packages/api-headless-cms-bulk-actions/src/tasks/createEmptyTrashBinsTask.ts index 3cdab9b7ca4..71bd89fb49a 100644 --- a/packages/api-headless-cms-bulk-actions/src/tasks/createEmptyTrashBinsTask.ts +++ b/packages/api-headless-cms-bulk-actions/src/tasks/createEmptyTrashBinsTask.ts @@ -1,10 +1,11 @@ -import { createPrivateTaskDefinition, TaskDataStatus } from "@webiny/tasks"; +import { createTaskDefinition } from "@webiny/tasks"; +import { createDeleteEntry, createListDeletedEntries } from "~/useCases"; import { HcmsBulkActionsContext, - IBulkActionOperationByModelInput, - TrashBinCleanUpParams + IEmptyTrashBinsInput, + IEmptyTrashBinsOutput, + IEmptyTrashBinsTaskParams } from "~/types"; -import { ChildTasksCleanup } from "~/useCases/internals"; const calculateDateTimeString = () => { // Retrieve the retention period from the environment variable WEBINY_TRASH_BIN_RETENTION_PERIOD_DAYS, @@ -23,122 +24,95 @@ const calculateDateTimeString = () => { return currentDate.toISOString(); }; -const cleanup = async ({ context, task }: TrashBinCleanUpParams) => { - // We want to clean all child tasks and logs, which have no errors. - const childTasksCleanup = new ChildTasksCleanup(); - try { - await childTasksCleanup.execute({ - context, - task - }); - } catch (ex) { - console.error(`Error while cleaning "EmptyTrashBins" child tasks.`, ex); - } -}; - export const createEmptyTrashBinsTask = () => { - return createPrivateTaskDefinition({ + return createTaskDefinition< + HcmsBulkActionsContext, + IEmptyTrashBinsInput, + IEmptyTrashBinsOutput + >({ + isPrivate: true, id: "hcmsEntriesEmptyTrashBins", title: "Headless CMS - Empty all trash bins", - description: - "Delete all entries found in the trash bin, for each model found in the system.", - maxIterations: 24, + description: "Delete all entries in the trash bin for each model in the system.", + maxIterations: 120, disableDatabaseLogs: true, - run: async params => { - const { response, isAborted, isCloseToTimeout, context, trigger, input, store } = - params; + run: async (params: IEmptyTrashBinsTaskParams) => { + const { response, isAborted, context, input, isCloseToTimeout } = params; + + // Abort the task if needed. if (isAborted()) { return response.aborted(); - } else if (isCloseToTimeout()) { - return response.continue( - { - ...input - }, - { - seconds: 30 - } - ); } - if (input.triggered) { - const { items } = await context.tasks.listTasks({ - where: { - parentId: store.getTask().id, - taskStatus_in: [TaskDataStatus.RUNNING, TaskDataStatus.PENDING] - }, - limit: 100000 - }); + // Fetch all tenants, excluding those already processed. + const baseTenants = await context.tenancy.listTenants(); + const executedTenantIds = input.executedTenantIds || []; + const tenants = baseTenants.filter(tenant => !executedTenantIds.includes(tenant.id)); + let shouldContinue = false; // Flag to check if task should continue. - if (items.length === 0) { - return response.done( - "Task done: emptying the trash bin for all registered models." - ); + // Iterate over each tenant. + await context.tenancy.withEachTenant(tenants, async tenant => { + if (isCloseToTimeout()) { + shouldContinue = true; + return; } - for (const item of items) { - const status = await context.tasks.fetchServiceInfo(item.id); - - if (status?.status === "FAILED" || status?.status === "TIMED_OUT") { - await context.tasks.updateTask(item.id, { - taskStatus: TaskDataStatus.FAILED - }); - continue; - } - - if (status?.status === "ABORTED") { - await context.tasks.updateTask(item.id, { - taskStatus: TaskDataStatus.ABORTED - }); + // Fetch all locales for the tenant. + const locales = context.i18n.getLocales(); + await context.i18n.withEachLocale(locales, async () => { + if (isCloseToTimeout()) { + shouldContinue = true; + return; } - } - return response.continue( - { - ...input - }, - { - seconds: 3600 - } - ); - } + // List all non-private models for the current locale. + const models = await context.security.withoutAuthorization(async () => + (await context.cms.listModels()).filter(m => !m.isPrivate) + ); - try { - const locales = context.i18n.getLocales(); + // Process each model to delete trashed entries. + for (const model of models) { + const list = createListDeletedEntries(context); // List trashed entries. + const mutation = createDeleteEntry(context); // Mutation to delete entries. - await context.i18n.withEachLocale(locales, async () => { - const models = await context.security.withoutAuthorization(async () => { - return (await context.cms.listModels()).filter(model => !model.isPrivate); - }); + // Query parameters for fetching deleted entries older than a minute ago. + const listEntriesParams = { + where: { deletedOn_lt: calculateDateTimeString() }, + limit: 50 + }; - for (const model of models) { - await trigger({ - name: `Headless CMS - Empty trash bin for "${model.name}" model.`, - definition: "hcmsBulkListDeleteEntries", - input: { - modelId: model.modelId, - where: { - deletedOn_lt: calculateDateTimeString() + let result; + // Continue deleting entries while there are entries left to delete. + while ( + (result = await list.execute(model.modelId, listEntriesParams)) && + result.meta.totalCount > 0 + ) { + if (isCloseToTimeout()) { + shouldContinue = true; + break; + } + for (const entry of result.entries) { + if (isCloseToTimeout()) { + shouldContinue = true; + break; } + // Delete each entry individually. + await mutation.execute(model, entry.id); } - }); + } } }); - return response.continue( - { - triggered: true - }, - { - seconds: 120 - } - ); - } catch (ex) { - return response.error(ex.message ?? "Error while executing EmptyTrashBins task"); - } - }, - onMaxIterations: cleanup, - onDone: cleanup, - onError: cleanup, - onAbort: cleanup + // If the task isn't continuing, add the tenant to the executed list. + if (!shouldContinue) { + executedTenantIds.push(tenant.id); + } + }); + + // Continue the task or mark it as done based on the `shouldContinue` flag. + return shouldContinue + ? response.continue({ ...input, executedTenantIds }) + : response.done("Task done: emptied the trash bin for all registered models."); + } }); }; diff --git a/packages/api-headless-cms-bulk-actions/src/types.ts b/packages/api-headless-cms-bulk-actions/src/types.ts index 913818c8b75..2ba5e556580 100644 --- a/packages/api-headless-cms-bulk-actions/src/types.ts +++ b/packages/api-headless-cms-bulk-actions/src/types.ts @@ -2,10 +2,6 @@ import { CmsContext } from "@webiny/api-headless-cms/types"; import { Context as BaseContext } from "@webiny/handler/types"; import { Context as TasksContext, - ITaskOnAbortParams, - ITaskOnErrorParams, - ITaskOnMaxIterationsParams, - ITaskOnSuccessParams, ITaskResponseDoneResultOutput, ITaskRunParams } from "@webiny/tasks/types"; @@ -70,11 +66,17 @@ export type IBulkActionOperationByModelTaskParams = ITaskRunParams< >; /** - * Trash Bin + * Empty Trash Bin */ -export type TrashBinCleanUpParams = - | ITaskOnSuccessParams - | ITaskOnErrorParams - | ITaskOnAbortParams - | ITaskOnMaxIterationsParams; +export interface IEmptyTrashBinsInput { + executedTenantIds?: string[] | null; +} + +export type IEmptyTrashBinsOutput = ITaskResponseDoneResultOutput; + +export type IEmptyTrashBinsTaskParams = ITaskRunParams< + HcmsBulkActionsContext, + IEmptyTrashBinsInput, + IEmptyTrashBinsOutput +>; diff --git a/packages/api-headless-cms-import-export/src/graphql/index.ts b/packages/api-headless-cms-import-export/src/graphql/index.ts index a18a1e07407..b021c9ff419 100644 --- a/packages/api-headless-cms-import-export/src/graphql/index.ts +++ b/packages/api-headless-cms-import-export/src/graphql/index.ts @@ -7,15 +7,20 @@ import type { NonEmptyArray } from "@webiny/api/types"; import { CmsModel } from "@webiny/api-headless-cms/types"; export const attachHeadlessCmsImportExportGraphQL = async (context: Context): Promise => { + const tenant = context.tenancy.getCurrentTenant(); + const locale = context.i18n.getContentLocale(); const models = await listModels(context); - if (models.length === 0) { + if (!locale || models.length === 0) { return; } const plugin = new CmsGraphQLSchemaPlugin({ typeDefs: createTypeDefs(models as NonEmptyArray), - resolvers: createResolvers(models as NonEmptyArray) + resolvers: createResolvers(models as NonEmptyArray), + isApplicable: context => + context.tenancy.getCurrentTenant().id === tenant.id && + context.i18n.getContentLocale()?.code === locale.code }); plugin.name = "headlessCms.graphql.importExport"; diff --git a/packages/api-headless-cms/src/crud/contentModel/validateModelFields.ts b/packages/api-headless-cms/src/crud/contentModel/validateModelFields.ts index 1ec1e69f229..a10bb3f6373 100644 --- a/packages/api-headless-cms/src/crud/contentModel/validateModelFields.ts +++ b/packages/api-headless-cms/src/crud/contentModel/validateModelFields.ts @@ -232,6 +232,7 @@ const createGraphQLSchema = async (params: CreateGraphQLSchemaParams): Promise(CmsGraphQLSchemaPlugin.type) + .filter(plugin => plugin.isApplicable(context)) .reduce>((collection, plugin) => { const name = plugin.name || `${CmsGraphQLSchemaPlugin.type}-${generateAlphaNumericId(16)}`; diff --git a/packages/handler-graphql/src/plugins/GraphQLSchemaPlugin.ts b/packages/handler-graphql/src/plugins/GraphQLSchemaPlugin.ts index a20a637ac92..ef5fb822c31 100644 --- a/packages/handler-graphql/src/plugins/GraphQLSchemaPlugin.ts +++ b/packages/handler-graphql/src/plugins/GraphQLSchemaPlugin.ts @@ -4,12 +4,14 @@ import { GraphQLSchemaDefinition, ResolverDecorators, Resolvers, TypeDefs } from export interface IGraphQLSchemaPlugin extends Plugin { schema: GraphQLSchemaDefinition; + isApplicable: (context: TContext) => boolean; } export interface GraphQLSchemaPluginConfig { typeDefs?: TypeDefs; resolvers?: Resolvers; resolverDecorators?: ResolverDecorators; + isApplicable?: (context: TContext) => boolean; } export class GraphQLSchemaPlugin @@ -31,6 +33,13 @@ export class GraphQLSchemaPlugin resolverDecorators: this.config.resolverDecorators }; } + + isApplicable(context: TContext): boolean { + if (this.config.isApplicable) { + return this.config.isApplicable(context); + } + return true; + } } export const createGraphQLSchemaPlugin = (config: GraphQLSchemaPluginConfig) => { From 9bd4fa2778b787f07c835ca5eeb2b2758e6c6975 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20Zori=C4=87?= Date: Mon, 11 Nov 2024 12:02:04 +0100 Subject: [PATCH 02/16] fix(api-elasticsearch-tasks): data synchronization (#4337) --- .../src/Operations.ts | 10 +- .../src/SynchronizationBuilder.ts | 72 +++++ .../src/eventHandler.ts | 11 +- .../src/execute.ts | 13 +- .../src/executeWithRetry.ts | 10 +- .../src/index.ts | 1 + .../src/types.ts | 4 + .../__tests__/mocks/store.ts | 6 +- ...ticsearchToDynamoDbSynchronization.test.ts | 172 ++++++++++++ .../dataSynchronizationTask.test.ts | 119 ++++++++ .../tasks/dataSynchronization/managers.ts | 40 +++ .../api-elasticsearch-tasks/jest.setup.js | 2 + packages/api-elasticsearch-tasks/package.json | 11 +- .../src/definitions/entry.ts | 4 +- .../src/definitions/table.ts | 4 +- .../src/helpers/scan.ts | 10 +- packages/api-elasticsearch-tasks/src/index.ts | 4 +- .../src/settings/IndexManager.ts | 20 +- .../src/tasks/Manager.ts | 22 +- .../createIndexes/CreateIndexesTaskRunner.ts | 8 +- .../src/tasks/createIndexes/index.ts | 11 +- .../DataSynchronizationTaskRunner.ts | 69 +++++ .../dataSynchronization/createFactories.ts | 10 + .../elasticsearch/ElasticsearchFetcher.ts | 107 ++++++++ .../elasticsearch/ElasticsearchSynchronize.ts | 101 +++++++ .../ElasticsearchToDynamoDbSynchronization.ts | 95 +++++++ .../abstractions/ElasticsearchFetcher.ts | 25 ++ .../abstractions/ElasticsearchSynchronize.ts | 21 ++ .../shouldIgnoreEsResponseError.ts | 11 + .../entities/getElasticsearchEntity.ts | 52 ++++ .../entities/getElasticsearchEntityType.ts | 27 ++ .../dataSynchronization/entities/getTable.ts | 30 +++ .../dataSynchronization/entities/index.ts | 3 + .../src/tasks/dataSynchronization/index.ts | 79 ++++++ .../src/tasks/dataSynchronization/types.ts | 62 +++++ .../EnableIndexingTaskRunner.ts | 8 +- .../src/tasks/enableIndexing/index.ts | 13 +- .../src/tasks/index.ts | 1 + .../tasks/reindexing/ReindexingTaskRunner.ts | 8 +- .../reindexing/reindexingTaskDefinition.ts | 13 +- packages/api-elasticsearch-tasks/src/types.ts | 19 +- .../tsconfig.build.json | 2 + .../api-elasticsearch-tasks/tsconfig.json | 6 + packages/api-elasticsearch/src/cursors.ts | 12 +- packages/api-elasticsearch/src/types.ts | 47 ++-- .../api-form-builder-so-ddb-es/src/index.ts | 21 ++ .../api-form-builder-so-ddb-es/src/types.ts | 22 +- packages/api-headless-cms-ddb-es/src/index.ts | 10 + packages/api-headless-cms-ddb-es/src/types.ts | 16 +- .../api-page-builder-so-ddb-es/src/index.ts | 10 + .../api-page-builder-so-ddb-es/src/types.ts | 15 +- packages/db-dynamodb/src/DynamoDbDriver.ts | 35 +-- packages/db-dynamodb/src/types.ts | 24 -- packages/db-dynamodb/src/utils/batchRead.ts | 5 +- packages/db/package.json | 3 + packages/db/src/DbRegistry.ts | 49 ++++ packages/db/src/index.ts | 254 +----------------- packages/db/src/types.ts | 26 ++ packages/db/tsconfig.build.json | 2 +- packages/db/tsconfig.json | 9 +- packages/handler-db/src/index.ts | 9 +- packages/handler-db/src/types.ts | 2 +- packages/tasks/src/runner/TaskManager.ts | 9 +- packages/tasks/src/runner/TaskRunner.ts | 2 +- .../src/runner/abstractions/TaskRunner.ts | 2 + packages/tasks/src/types.ts | 2 + yarn.lock | 3 + 67 files changed, 1479 insertions(+), 426 deletions(-) create mode 100644 packages/api-dynamodb-to-elasticsearch/src/SynchronizationBuilder.ts create mode 100644 packages/api-elasticsearch-tasks/__tests__/tasks/dataSynchronization/ElasticsearchToDynamoDbSynchronization.test.ts create mode 100644 packages/api-elasticsearch-tasks/__tests__/tasks/dataSynchronization/dataSynchronizationTask.test.ts create mode 100644 packages/api-elasticsearch-tasks/__tests__/tasks/dataSynchronization/managers.ts create mode 100644 packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/DataSynchronizationTaskRunner.ts create mode 100644 packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/createFactories.ts create mode 100644 packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/elasticsearch/ElasticsearchFetcher.ts create mode 100644 packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/elasticsearch/ElasticsearchSynchronize.ts create mode 100644 packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/elasticsearch/ElasticsearchToDynamoDbSynchronization.ts create mode 100644 packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/elasticsearch/abstractions/ElasticsearchFetcher.ts create mode 100644 packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/elasticsearch/abstractions/ElasticsearchSynchronize.ts create mode 100644 packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/elasticsearch/shouldIgnoreEsResponseError.ts create mode 100644 packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/entities/getElasticsearchEntity.ts create mode 100644 packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/entities/getElasticsearchEntityType.ts create mode 100644 packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/entities/getTable.ts create mode 100644 packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/entities/index.ts create mode 100644 packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/index.ts create mode 100644 packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/types.ts create mode 100644 packages/db/src/DbRegistry.ts create mode 100644 packages/db/src/types.ts diff --git a/packages/api-dynamodb-to-elasticsearch/src/Operations.ts b/packages/api-dynamodb-to-elasticsearch/src/Operations.ts index d69ec29a1ed..0a52c7cb0e4 100644 --- a/packages/api-dynamodb-to-elasticsearch/src/Operations.ts +++ b/packages/api-dynamodb-to-elasticsearch/src/Operations.ts @@ -13,12 +13,20 @@ export enum OperationType { } export class Operations implements IOperations { - public readonly items: GenericRecord[] = []; + private _items: GenericRecord[] = []; + + public get items(): GenericRecord[] { + return this._items; + } public get total(): number { return this.items.length; } + public clear() { + this._items = []; + } + public insert(params: IInsertOperationParams): void { this.items.push( { diff --git a/packages/api-dynamodb-to-elasticsearch/src/SynchronizationBuilder.ts b/packages/api-dynamodb-to-elasticsearch/src/SynchronizationBuilder.ts new file mode 100644 index 00000000000..81fe68a73dd --- /dev/null +++ b/packages/api-dynamodb-to-elasticsearch/src/SynchronizationBuilder.ts @@ -0,0 +1,72 @@ +import { + Context, + IDeleteOperationParams, + IInsertOperationParams, + IModifyOperationParams, + IOperations +} from "~/types"; +import { Operations } from "~/Operations"; +import { executeWithRetry, IExecuteWithRetryParams } from "~/executeWithRetry"; +import { ITimer } from "@webiny/handler-aws"; + +export type ISynchronizationBuilderExecuteWithRetryParams = Omit< + IExecuteWithRetryParams, + "context" | "timer" | "maxRunningTime" | "operations" +>; + +export interface ISynchronizationBuilder { + insert(params: IInsertOperationParams): void; + delete(params: IDeleteOperationParams): void; + build: () => (params?: ISynchronizationBuilderExecuteWithRetryParams) => Promise; +} + +export interface ISynchronizationBuilderParams { + timer: ITimer; + context: Pick; +} + +export class SynchronizationBuilder implements ISynchronizationBuilder { + private readonly timer: ITimer; + private readonly context: Pick; + private readonly operations: IOperations; + + public constructor(params: ISynchronizationBuilderParams) { + this.timer = params.timer; + this.context = params.context; + this.operations = new Operations(); + } + + public insert(params: IInsertOperationParams): void { + return this.operations.insert(params); + } + + public modify(params: IModifyOperationParams): void { + return this.operations.modify(params); + } + + public delete(params: IDeleteOperationParams): void { + return this.operations.delete(params); + } + + public build() { + return async (params?: ISynchronizationBuilderExecuteWithRetryParams) => { + if (this.operations.total === 0) { + return; + } + await executeWithRetry({ + ...params, + maxRunningTime: this.timer.getRemainingMilliseconds(), + timer: this.timer, + context: this.context, + operations: this.operations + }); + this.operations.clear(); + }; + } +} + +export const createSynchronizationBuilder = ( + params: ISynchronizationBuilderParams +): ISynchronizationBuilder => { + return new SynchronizationBuilder(params); +}; diff --git a/packages/api-dynamodb-to-elasticsearch/src/eventHandler.ts b/packages/api-dynamodb-to-elasticsearch/src/eventHandler.ts index ef4b1e0f58f..c683a72c9a6 100644 --- a/packages/api-dynamodb-to-elasticsearch/src/eventHandler.ts +++ b/packages/api-dynamodb-to-elasticsearch/src/eventHandler.ts @@ -1,15 +1,9 @@ -import { getNumberEnvVariable } from "~/helpers/getNumberEnvVariable"; import { createDynamoDBEventHandler, timerFactory } from "@webiny/handler-aws"; -import { ElasticsearchContext } from "@webiny/api-elasticsearch/types"; +import { Context } from "~/types"; import { Decompressor } from "~/Decompressor"; import { OperationsBuilder } from "~/OperationsBuilder"; import { executeWithRetry } from "~/executeWithRetry"; -const MAX_PROCESSOR_PERCENT = getNumberEnvVariable( - "MAX_ES_PROCESSOR", - process.env.NODE_ENV === "test" ? 101 : 98 -); - /** * Also, we need to set the maximum running time for the Lambda Function. * https://github.com/webiny/webiny-js/blob/f7352d418da2b5ae0b781376be46785aa7ac6ae0/packages/pulumi-aws/src/apps/core/CoreOpenSearch.ts#L232 @@ -20,7 +14,7 @@ const MAX_RUNNING_TIME = 900; export const createEventHandler = () => { return createDynamoDBEventHandler(async ({ event, context: ctx, lambdaContext }) => { const timer = timerFactory(lambdaContext); - const context = ctx as unknown as ElasticsearchContext; + const context = ctx as unknown as Context; if (!context.elasticsearch) { console.error("Missing elasticsearch definition on context."); return null; @@ -49,7 +43,6 @@ export const createEventHandler = () => { await executeWithRetry({ timer, maxRunningTime: MAX_RUNNING_TIME, - maxProcessorPercent: MAX_PROCESSOR_PERCENT, context, operations }); diff --git a/packages/api-dynamodb-to-elasticsearch/src/execute.ts b/packages/api-dynamodb-to-elasticsearch/src/execute.ts index 2bbcd6b5569..a050f6fb691 100644 --- a/packages/api-dynamodb-to-elasticsearch/src/execute.ts +++ b/packages/api-dynamodb-to-elasticsearch/src/execute.ts @@ -5,9 +5,9 @@ import { WaitingHealthyClusterAbortedError } from "@webiny/api-elasticsearch"; import { ITimer } from "@webiny/handler-aws"; -import { ApiResponse, ElasticsearchContext } from "@webiny/api-elasticsearch/types"; +import { ApiResponse } from "@webiny/api-elasticsearch/types"; import { WebinyError } from "@webiny/error"; -import { IOperations } from "./types"; +import { Context, IOperations } from "./types"; export interface BulkOperationsResponseBodyItemIndexError { reason?: string; @@ -30,8 +30,8 @@ export interface IExecuteParams { timer: ITimer; maxRunningTime: number; maxProcessorPercent: number; - context: Pick; - operations: IOperations; + context: Pick; + operations: Pick; } const getError = (item: BulkOperationsResponseBodyItem): string | null => { @@ -67,6 +67,11 @@ const checkErrors = (result?: ApiResponse): void => export const execute = (params: IExecuteParams) => { return async (): Promise => { const { context, timer, maxRunningTime, maxProcessorPercent, operations } = params; + + if (operations.total === 0) { + return; + } + const remainingTime = timer.getRemainingSeconds(); const runningTime = maxRunningTime - remainingTime; const maxWaitingTime = remainingTime - 90; diff --git a/packages/api-dynamodb-to-elasticsearch/src/executeWithRetry.ts b/packages/api-dynamodb-to-elasticsearch/src/executeWithRetry.ts index f9e442fe08d..2411b7a27a9 100644 --- a/packages/api-dynamodb-to-elasticsearch/src/executeWithRetry.ts +++ b/packages/api-dynamodb-to-elasticsearch/src/executeWithRetry.ts @@ -5,11 +5,17 @@ import { getNumberEnvVariable } from "./helpers/getNumberEnvVariable"; const minRemainingSecondsToTimeout = 120; -export interface IExecuteWithRetryParams extends IExecuteParams { +const MAX_PROCESSOR_PERCENT = getNumberEnvVariable( + "MAX_ES_PROCESSOR", + process.env.NODE_ENV === "test" ? 101 : 98 +); + +export interface IExecuteWithRetryParams extends Omit { maxRetryTime?: number; retries?: number; minTimeout?: number; maxTimeout?: number; + maxProcessorPercent?: number; } export const executeWithRetry = async (params: IExecuteWithRetryParams) => { @@ -35,7 +41,7 @@ export const executeWithRetry = async (params: IExecuteWithRetryParams) => { execute({ timer: params.timer, maxRunningTime: params.maxRunningTime, - maxProcessorPercent: params.maxProcessorPercent, + maxProcessorPercent: params.maxProcessorPercent || MAX_PROCESSOR_PERCENT, context: params.context, operations: params.operations }), diff --git a/packages/api-dynamodb-to-elasticsearch/src/index.ts b/packages/api-dynamodb-to-elasticsearch/src/index.ts index 99a39edec60..9c6d27a1d8b 100644 --- a/packages/api-dynamodb-to-elasticsearch/src/index.ts +++ b/packages/api-dynamodb-to-elasticsearch/src/index.ts @@ -6,4 +6,5 @@ export * from "./marshall"; export * from "./NotEnoughRemainingTimeError"; export * from "./Operations"; export * from "./OperationsBuilder"; +export * from "./SynchronizationBuilder"; export * from "./types"; diff --git a/packages/api-dynamodb-to-elasticsearch/src/types.ts b/packages/api-dynamodb-to-elasticsearch/src/types.ts index 2707505d00e..c08e976d72e 100644 --- a/packages/api-dynamodb-to-elasticsearch/src/types.ts +++ b/packages/api-dynamodb-to-elasticsearch/src/types.ts @@ -1,5 +1,8 @@ import { GenericRecord } from "@webiny/cli/types"; import { DynamoDBRecord } from "@webiny/handler-aws/types"; +import { ElasticsearchContext } from "@webiny/api-elasticsearch/types"; + +export type Context = Pick; export interface IOperationsBuilderBuildParams { records: DynamoDBRecord[]; @@ -25,6 +28,7 @@ export interface IDeleteOperationParams { export interface IOperations { items: GenericRecord[]; total: number; + clear(): void; insert(params: IInsertOperationParams): void; modify(params: IModifyOperationParams): void; delete(params: IDeleteOperationParams): void; diff --git a/packages/api-elasticsearch-tasks/__tests__/mocks/store.ts b/packages/api-elasticsearch-tasks/__tests__/mocks/store.ts index 860790ed613..ad4aa8979bc 100644 --- a/packages/api-elasticsearch-tasks/__tests__/mocks/store.ts +++ b/packages/api-elasticsearch-tasks/__tests__/mocks/store.ts @@ -1,5 +1,5 @@ import { TaskManagerStore } from "@webiny/tasks/runner/TaskManagerStore"; -import { Context, ITask, ITaskLog } from "@webiny/tasks/types"; +import { Context, ITask, ITaskDataInput, ITaskLog } from "@webiny/tasks/types"; import { createTaskMock } from "~tests/mocks/task"; import { createContextMock } from "~tests/mocks/context"; import { createTaskLogMock } from "~tests/mocks/log"; @@ -9,11 +9,11 @@ interface Params { task?: ITask; log?: ITaskLog; } -export const createTaskManagerStoreMock = (params?: Params) => { +export const createTaskManagerStoreMock = (params?: Params) => { const context = params?.context || createContextMock(); const task = params?.task || createTaskMock(); const log = params?.log || createTaskLogMock(task); - return new TaskManagerStore({ + return new TaskManagerStore({ context, task, log diff --git a/packages/api-elasticsearch-tasks/__tests__/tasks/dataSynchronization/ElasticsearchToDynamoDbSynchronization.test.ts b/packages/api-elasticsearch-tasks/__tests__/tasks/dataSynchronization/ElasticsearchToDynamoDbSynchronization.test.ts new file mode 100644 index 00000000000..c1486122e62 --- /dev/null +++ b/packages/api-elasticsearch-tasks/__tests__/tasks/dataSynchronization/ElasticsearchToDynamoDbSynchronization.test.ts @@ -0,0 +1,172 @@ +import { ElasticsearchToDynamoDbSynchronization } from "~/tasks/dataSynchronization/elasticsearch/ElasticsearchToDynamoDbSynchronization"; +import { useHandler } from "~tests/helpers/useHandler"; +import { createManagers } from "./managers"; +import { ElasticsearchFetcher } from "~/tasks/dataSynchronization/elasticsearch/ElasticsearchFetcher"; +import { ElasticsearchSynchronize } from "~/tasks/dataSynchronization/elasticsearch/ElasticsearchSynchronize"; +import { DATA_SYNCHRONIZATION_TASK } from "~/tasks"; +import { Context, SynchronizationBuilder } from "@webiny/api-dynamodb-to-elasticsearch"; +import { ITimer } from "@webiny/handler-aws"; +import { IIndexManager } from "~/settings/types"; + +const queryAllRecords = (index: string) => { + return { + index, + body: { + query: { + match_all: {} + }, + size: 10000, + _source: false + } + }; +}; + +interface ICreateSyncBuilderParams { + records: number; + timer: ITimer; + context: Pick; + index: string; +} + +const createRecordsFactory = (params: ICreateSyncBuilderParams) => { + const { timer, context, index, records } = params; + const syncBuilder = new SynchronizationBuilder({ + timer, + context + }); + + for (let i = 0; i < records; i++) { + syncBuilder.insert({ + id: `pkValue${i}:skValue${i}`, + index, + data: { + id: `skValue${i}`, + aText: `myText - ${i}` + } + }); + } + return { + run: () => { + return syncBuilder.build()(); + } + }; +}; + +const getTaskIndex = async (manager: IIndexManager): Promise => { + const indexes = await manager.list(); + const index = indexes.find( + index => index.includes("webinytask") && index.includes("-headless-cms-") + ); + if (!index) { + throw new Error("No index found."); + } + return index; +}; + +describe("ElasticsearchToDynamoDbSynchronization", () => { + it("should run a sync without any indexes and throw an error", async () => { + const handler = useHandler(); + + const context = await handler.rawHandle(); + + const { manager, indexManager } = createManagers({ + context + }); + + const sync = new ElasticsearchToDynamoDbSynchronization({ + manager, + indexManager, + fetcher: new ElasticsearchFetcher({ + client: context.elasticsearch + }), + synchronize: new ElasticsearchSynchronize({ + context, + timer: manager.timer + }) + }); + + try { + const result = await sync.run({ + flow: "elasticsearchToDynamoDb" + }); + expect(result).toEqual("Should not reach this point."); + } catch (ex) { + expect(ex.message).toBe("No Elasticsearch / OpenSearch indexes found."); + } + }); + + it("should run a sync with indexes and finish", async () => { + const handler = useHandler(); + + const context = await handler.rawHandle(); + + await context.tasks.createTask({ + definitionId: DATA_SYNCHRONIZATION_TASK, + input: { + flow: "elasticsearchToDynamoDb" + }, + name: "Data Sync Mock Task" + }); + + const { manager, indexManager } = createManagers({ + context + }); + + const index = await getTaskIndex(indexManager); + + const totalMockItemsToInsert = 101; + const recordsFactory = createRecordsFactory({ + context, + index, + timer: manager.timer, + records: totalMockItemsToInsert + }); + try { + await recordsFactory.run(); + } catch (ex) { + expect(ex.message).toBe("Should not reach this point."); + } + /** + * Now we need to make sure that the mock data is in the index. + */ + const response = await context.elasticsearch.search(queryAllRecords(index)); + expect(response.body.hits.hits).toHaveLength(totalMockItemsToInsert + 1); + + const sync = new ElasticsearchToDynamoDbSynchronization({ + manager, + indexManager, + fetcher: new ElasticsearchFetcher({ + client: context.elasticsearch + }), + synchronize: new ElasticsearchSynchronize({ + context, + timer: manager.timer + }) + }); + + const result = await sync.run({ + flow: "elasticsearchToDynamoDb" + }); + expect(result).toEqual({ + delay: -1, + input: { + elasticsearchToDynamoDb: { + finished: true + }, + flow: "elasticsearchToDynamoDb" + }, + locale: "en-US", + message: undefined, + status: "continue", + tenant: "root", + wait: undefined, + webinyTaskDefinitionId: "mockDefinitionId", + webinyTaskId: "mockEventId" + }); + /** + * Now we need to make sure that the mock data is not in the index anymore. + */ + const afterRunResponse = await context.elasticsearch.search(queryAllRecords(index)); + expect(afterRunResponse.body.hits.hits).toHaveLength(1); + }); +}); diff --git a/packages/api-elasticsearch-tasks/__tests__/tasks/dataSynchronization/dataSynchronizationTask.test.ts b/packages/api-elasticsearch-tasks/__tests__/tasks/dataSynchronization/dataSynchronizationTask.test.ts new file mode 100644 index 00000000000..7f2769d1edc --- /dev/null +++ b/packages/api-elasticsearch-tasks/__tests__/tasks/dataSynchronization/dataSynchronizationTask.test.ts @@ -0,0 +1,119 @@ +import { createDataSynchronization, DATA_SYNCHRONIZATION_TASK } from "~/tasks"; +import { TaskDefinitionPlugin, TaskResponseStatus } from "@webiny/tasks"; +import { createRunner } from "@webiny/project-utils/testing/tasks"; +import { useHandler } from "~tests/helpers/useHandler"; +import { IDataSynchronizationInput, IFactories } from "~/tasks/dataSynchronization/types"; + +jest.mock("~/tasks/dataSynchronization/createFactories", () => { + return { + createFactories: (): IFactories => { + return { + elasticsearchToDynamoDb: ({ manager }) => { + return { + run: async input => { + return manager.response.continue({ + ...input, + elasticsearchToDynamoDb: { + finished: true + } + }); + } + }; + } + }; + } + }; +}); + +describe("data synchronization - elasticsearch", () => { + it("should create a task definition", async () => { + const result = createDataSynchronization(); + + expect(result).toBeInstanceOf(TaskDefinitionPlugin); + expect(result).toEqual({ + isPrivate: false, + task: { + id: DATA_SYNCHRONIZATION_TASK, + isPrivate: false, + title: "Data Synchronization", + description: "Synchronize data between Elasticsearch and DynamoDB", + maxIterations: 100, + disableDatabaseLogs: true, + fields: [], + run: expect.any(Function), + createInputValidation: expect.any(Function) + } + }); + }); + + it("should run a task and end with error due to invalid flow", async () => { + const handler = useHandler({}); + + const context = await handler.rawHandle(); + + try { + const task = await context.tasks.createTask({ + definitionId: DATA_SYNCHRONIZATION_TASK, + input: { + // @ts-expect-error + flow: "unknownFlow" + }, + name: "Data Sync Mock Task" + }); + expect(task).toEqual("Should not reach this point."); + } catch (ex) { + expect(ex.message).toEqual("Validation failed."); + expect(ex.data).toEqual({ + invalidFields: { + flow: { + code: "invalid_enum_value", + data: { + fatal: undefined, + path: ["flow"] + }, + message: + "Invalid enum value. Expected 'elasticsearchToDynamoDb', received 'unknownFlow'" + } + } + }); + } + }); + + it("should run a task and end with done", async () => { + const handler = useHandler({}); + + const context = await handler.rawHandle(); + + const task = await context.tasks.createTask({ + definitionId: DATA_SYNCHRONIZATION_TASK, + input: { + flow: "elasticsearchToDynamoDb" + }, + name: "Data Sync Mock Task" + }); + + const runner = createRunner({ + context, + task: createDataSynchronization(), + onContinue: async () => { + return; + } + }); + + const result = await runner({ + webinyTaskId: task.id + }); + + expect(result).toEqual({ + status: TaskResponseStatus.DONE, + webinyTaskId: task.id, + webinyTaskDefinitionId: DATA_SYNCHRONIZATION_TASK, + tenant: "root", + locale: "en-US", + message: undefined, + output: undefined + }); + const taskCheck = await context.tasks.getTask(task.id); + expect(taskCheck?.iterations).toEqual(2); + }); +}); diff --git a/packages/api-elasticsearch-tasks/__tests__/tasks/dataSynchronization/managers.ts b/packages/api-elasticsearch-tasks/__tests__/tasks/dataSynchronization/managers.ts new file mode 100644 index 00000000000..1ac6803de3c --- /dev/null +++ b/packages/api-elasticsearch-tasks/__tests__/tasks/dataSynchronization/managers.ts @@ -0,0 +1,40 @@ +import { IndexManager } from "~/settings"; +import { + IDataSynchronizationInput, + IDataSynchronizationManager +} from "~/tasks/dataSynchronization/types"; +import { Context } from "~/types"; +import { Manager } from "~/tasks/Manager"; +import { Response, TaskResponse } from "@webiny/tasks"; +import { createMockEvent } from "~tests/mocks/event"; +import { createTaskManagerStoreMock } from "~tests/mocks/store"; +import { timerFactory } from "@webiny/handler-aws/utils"; + +export interface ICreateManagersParams { + context: Context; +} + +export const createManagers = (params: ICreateManagersParams) => { + const { context } = params; + const manager = new Manager({ + elasticsearchClient: context.elasticsearch, + // @ts-expect-error + documentClient: context.db.driver.documentClient, + response: new TaskResponse(new Response(createMockEvent())), + context, + isAborted: () => { + return false; + }, + isCloseToTimeout: () => { + return false; + }, + timer: timerFactory(), + store: createTaskManagerStoreMock() + }); + + const indexManager = new IndexManager(context.elasticsearch, {}); + return { + manager: manager as unknown as IDataSynchronizationManager, + indexManager + }; +}; diff --git a/packages/api-elasticsearch-tasks/jest.setup.js b/packages/api-elasticsearch-tasks/jest.setup.js index 049095053ae..e9fae563a72 100644 --- a/packages/api-elasticsearch-tasks/jest.setup.js +++ b/packages/api-elasticsearch-tasks/jest.setup.js @@ -1,6 +1,8 @@ const base = require("../../jest.config.base"); const presets = require("@webiny/project-utils/testing/presets")( ["@webiny/api-headless-cms", "storage-operations"], + ["@webiny/api-form-builder", "storage-operations"], + ["@webiny/api-page-builder", "storage-operations"], ["@webiny/api-i18n", "storage-operations"], ["@webiny/api-security", "storage-operations"], ["@webiny/api-tenancy", "storage-operations"] diff --git a/packages/api-elasticsearch-tasks/package.json b/packages/api-elasticsearch-tasks/package.json index 3a0debe2738..9cb842d392c 100644 --- a/packages/api-elasticsearch-tasks/package.json +++ b/packages/api-elasticsearch-tasks/package.json @@ -14,8 +14,10 @@ "dependencies": { "@babel/runtime": "^7.24.0", "@webiny/api": "0.0.0", + "@webiny/api-dynamodb-to-elasticsearch": "0.0.0", "@webiny/api-elasticsearch": "0.0.0", "@webiny/aws-sdk": "0.0.0", + "@webiny/db": "0.0.0", "@webiny/db-dynamodb": "0.0.0", "@webiny/error": "0.0.0", "@webiny/tasks": "0.0.0", @@ -52,5 +54,12 @@ "build": "yarn webiny run build", "watch": "yarn webiny run watch" }, - "gitHead": "8476da73b653c89cc1474d968baf55c1b0ae0e5f" + "gitHead": "8476da73b653c89cc1474d968baf55c1b0ae0e5f", + "adio": { + "ignore": { + "src": [ + "node:util" + ] + } + } } diff --git a/packages/api-elasticsearch-tasks/src/definitions/entry.ts b/packages/api-elasticsearch-tasks/src/definitions/entry.ts index b985c523455..fa3c9bb7afd 100644 --- a/packages/api-elasticsearch-tasks/src/definitions/entry.ts +++ b/packages/api-elasticsearch-tasks/src/definitions/entry.ts @@ -1,7 +1,7 @@ -import { Entity, Table } from "@webiny/db-dynamodb/toolbox"; +import { Entity, TableDef } from "@webiny/db-dynamodb/toolbox"; interface Params { - table: Table; + table: TableDef; entityName: string; } diff --git a/packages/api-elasticsearch-tasks/src/definitions/table.ts b/packages/api-elasticsearch-tasks/src/definitions/table.ts index cc1f5e7a50e..5bd160dfb1e 100644 --- a/packages/api-elasticsearch-tasks/src/definitions/table.ts +++ b/packages/api-elasticsearch-tasks/src/definitions/table.ts @@ -1,11 +1,11 @@ import { DynamoDBDocument } from "@webiny/aws-sdk/client-dynamodb"; -import { Table, TableConstructor } from "@webiny/db-dynamodb/toolbox"; +import { Table, TableConstructor, TableDef } from "@webiny/db-dynamodb/toolbox"; interface Params { documentClient: DynamoDBDocument; } -export const createTable = ({ documentClient }: Params): Table => { +export const createTable = ({ documentClient }: Params): TableDef => { const config: TableConstructor = { name: process.env.DB_TABLE_ELASTICSEARCH as string, partitionKey: "PK", diff --git a/packages/api-elasticsearch-tasks/src/helpers/scan.ts b/packages/api-elasticsearch-tasks/src/helpers/scan.ts index 1f5d4141dbf..2440c0deba5 100644 --- a/packages/api-elasticsearch-tasks/src/helpers/scan.ts +++ b/packages/api-elasticsearch-tasks/src/helpers/scan.ts @@ -1,11 +1,11 @@ import { scan as tableScan, ScanOptions } from "@webiny/db-dynamodb"; -import { Table } from "@webiny/db-dynamodb/toolbox"; +import { TableDef } from "@webiny/db-dynamodb/toolbox"; import { IElasticsearchIndexingTaskValuesKeys } from "~/types"; interface Params { - table: Table; + table: TableDef; keys?: IElasticsearchIndexingTaskValuesKeys; - options?: Pick; + options?: ScanOptions; } export const scan = async (params: Params) => { @@ -13,9 +13,9 @@ export const scan = async (params: Params) => { return tableScan({ table, options: { + ...params.options, startKey: keys, - limit: 200, - ...params.options + limit: params.options?.limit || 200 } }); }; diff --git a/packages/api-elasticsearch-tasks/src/index.ts b/packages/api-elasticsearch-tasks/src/index.ts index e8ca3b86175..12dfab884d1 100644 --- a/packages/api-elasticsearch-tasks/src/index.ts +++ b/packages/api-elasticsearch-tasks/src/index.ts @@ -1,4 +1,5 @@ import { + createDataSynchronization, createElasticsearchReindexingTask, createEnableIndexingTask, createIndexesTaskDefinition @@ -14,7 +15,8 @@ export const createElasticsearchBackgroundTasks = ( return [ createElasticsearchReindexingTask(params), createEnableIndexingTask(params), - createIndexesTaskDefinition(params) + createIndexesTaskDefinition(params), + createDataSynchronization(params) ]; }; diff --git a/packages/api-elasticsearch-tasks/src/settings/IndexManager.ts b/packages/api-elasticsearch-tasks/src/settings/IndexManager.ts index 289e70cd2b9..94a9dca0d32 100644 --- a/packages/api-elasticsearch-tasks/src/settings/IndexManager.ts +++ b/packages/api-elasticsearch-tasks/src/settings/IndexManager.ts @@ -11,6 +11,22 @@ const defaultIndexSettings: IIndexSettingsValues = { refreshInterval: "1s" }; +export interface IListIndicesResponse { + index: string; +} + +const indexPrefix = process.env.ELASTIC_SEARCH_INDEX_PREFIX || ""; +const filterIndex = (item?: string) => { + if (!item) { + return false; + } else if (item.startsWith(".")) { + return false; + } else if (indexPrefix) { + return item.startsWith(indexPrefix); + } + return true; +}; + export class IndexManager implements IIndexManager { private readonly client: Client; private readonly disable: DisableIndexing; @@ -41,13 +57,13 @@ export class IndexManager implements IIndexManager { public async list(): Promise { try { - const response = await this.client.cat.indices({ + const response = await this.client.cat.indices({ format: "json" }); if (!Array.isArray(response.body)) { return []; } - return response.body.map((item: any) => item.index).filter(Boolean); + return response.body.map(item => item.index).filter(filterIndex); } catch (ex) { console.error( JSON.stringify({ diff --git a/packages/api-elasticsearch-tasks/src/tasks/Manager.ts b/packages/api-elasticsearch-tasks/src/tasks/Manager.ts index 1fbebc97913..a62664742a0 100644 --- a/packages/api-elasticsearch-tasks/src/tasks/Manager.ts +++ b/packages/api-elasticsearch-tasks/src/tasks/Manager.ts @@ -1,11 +1,11 @@ import { DynamoDBDocument, getDocumentClient } from "@webiny/aws-sdk/client-dynamodb"; import { Client, createElasticsearchClient } from "@webiny/api-elasticsearch"; import { createTable } from "~/definitions"; -import { Context, IElasticsearchIndexingTaskValues, IManager } from "~/types"; +import { Context, IManager } from "~/types"; import { createEntry } from "~/definitions/entry"; import { Entity } from "@webiny/db-dynamodb/toolbox"; import { ITaskResponse } from "@webiny/tasks/response/abstractions"; -import { ITaskManagerStore } from "@webiny/tasks/runner/abstractions"; +import { IIsCloseToTimeoutCallable, ITaskManagerStore } from "@webiny/tasks/runner/abstractions"; import { batchReadAll, BatchReadItem, @@ -13,30 +13,33 @@ import { BatchWriteItem, BatchWriteResult } from "@webiny/db-dynamodb"; +import { ITimer } from "@webiny/handler-aws/utils"; -export interface ManagerParams { +export interface ManagerParams { context: Context; documentClient?: DynamoDBDocument; elasticsearchClient?: Client; - isCloseToTimeout: () => boolean; + isCloseToTimeout: IIsCloseToTimeoutCallable; isAborted: () => boolean; response: ITaskResponse; - store: ITaskManagerStore; + store: ITaskManagerStore; + timer: ITimer; } -export class Manager implements IManager { +export class Manager implements IManager { public readonly documentClient: DynamoDBDocument; public readonly elasticsearch: Client; public readonly context: Context; public readonly table: ReturnType; - public readonly isCloseToTimeout: () => boolean; + public readonly isCloseToTimeout: IIsCloseToTimeoutCallable; public readonly isAborted: () => boolean; public readonly response: ITaskResponse; - public readonly store: ITaskManagerStore; + public readonly store: ITaskManagerStore; + public readonly timer: ITimer; private readonly entities: Record> = {}; - public constructor(params: ManagerParams) { + public constructor(params: ManagerParams) { this.context = params.context; this.documentClient = params?.documentClient || getDocumentClient(); @@ -58,6 +61,7 @@ export class Manager implements IManager { }; this.response = params.response; this.store = params.store; + this.timer = params.timer; } public getEntity(name: string): Entity { diff --git a/packages/api-elasticsearch-tasks/src/tasks/createIndexes/CreateIndexesTaskRunner.ts b/packages/api-elasticsearch-tasks/src/tasks/createIndexes/CreateIndexesTaskRunner.ts index c9d65ce92e4..62b572e0845 100644 --- a/packages/api-elasticsearch-tasks/src/tasks/createIndexes/CreateIndexesTaskRunner.ts +++ b/packages/api-elasticsearch-tasks/src/tasks/createIndexes/CreateIndexesTaskRunner.ts @@ -6,12 +6,16 @@ import { CreateElasticsearchIndexTaskPluginIndex } from "./CreateElasticsearchIndexTaskPlugin"; import { Context } from "~/types"; +import { IElasticsearchCreateIndexesTaskInput } from "~/tasks/createIndexes/types"; export class CreateIndexesTaskRunner { - private readonly manager: Manager; + private readonly manager: Manager; private readonly indexManager: IndexManager; - public constructor(manager: Manager, indexManager: IndexManager) { + public constructor( + manager: Manager, + indexManager: IndexManager + ) { this.manager = manager; this.indexManager = indexManager; diff --git a/packages/api-elasticsearch-tasks/src/tasks/createIndexes/index.ts b/packages/api-elasticsearch-tasks/src/tasks/createIndexes/index.ts index a84529c4b74..46850d171b2 100644 --- a/packages/api-elasticsearch-tasks/src/tasks/createIndexes/index.ts +++ b/packages/api-elasticsearch-tasks/src/tasks/createIndexes/index.ts @@ -12,23 +12,24 @@ export const createIndexesTaskDefinition = (params?: IElasticsearchTaskConfig) = * No point in having more than 2 runs, as the create index operations should not even take 1 full run, no matter how much indeexs is there to create. */ maxIterations: 2, - run: async ({ response, context, isCloseToTimeout, isAborted, store, input }) => { + run: async ({ response, context, isCloseToTimeout, isAborted, store, input, timer }) => { const { Manager } = await import( - /* webpackChunkName: "ElasticsearchTaskManager" */ + /* webpackChunkName: "Manager" */ "../Manager" ); const { IndexManager } = await import( - /* webpackChunkName: "ElasticsearchTaskSettings" */ "~/settings" + /* webpackChunkName: "IndexManager" */ "~/settings" ); - const manager = new Manager({ + const manager = new Manager({ elasticsearchClient: params?.elasticsearchClient, documentClient: params?.documentClient, response, context, isAborted, isCloseToTimeout, - store + store, + timer }); const indexManager = new IndexManager(manager.elasticsearch, {}); diff --git a/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/DataSynchronizationTaskRunner.ts b/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/DataSynchronizationTaskRunner.ts new file mode 100644 index 00000000000..5e3b1231eae --- /dev/null +++ b/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/DataSynchronizationTaskRunner.ts @@ -0,0 +1,69 @@ +import { + IDataSynchronizationInput, + IDataSynchronizationManager, + IFactories +} from "~/tasks/dataSynchronization/types"; +import { IIndexManager } from "~/settings/types"; +import { ElasticsearchSynchronize } from "~/tasks/dataSynchronization/elasticsearch/ElasticsearchSynchronize"; +import { ElasticsearchFetcher } from "~/tasks/dataSynchronization/elasticsearch/ElasticsearchFetcher"; + +export interface IDataSynchronizationTaskRunnerParams { + manager: IDataSynchronizationManager; + indexManager: IIndexManager; + factories: IFactories; +} + +export class DataSynchronizationTaskRunner { + private readonly manager: IDataSynchronizationManager; + private readonly indexManager: IIndexManager; + private readonly factories: IFactories; + + public constructor(params: IDataSynchronizationTaskRunnerParams) { + this.manager = params.manager; + this.indexManager = params.indexManager; + this.factories = params.factories; + } + + public async run(input: IDataSynchronizationInput) { + this.validateFlow(input); + /** + * Go through the Elasticsearch and delete records which do not exist in the Elasticsearch table. + */ + // + if (input.flow === "elasticsearchToDynamoDb" && !input.elasticsearchToDynamoDb?.finished) { + const sync = this.factories.elasticsearchToDynamoDb({ + manager: this.manager, + indexManager: this.indexManager, + synchronize: new ElasticsearchSynchronize({ + context: this.manager.context, + timer: this.manager.timer + }), + fetcher: new ElasticsearchFetcher({ + client: this.manager.elasticsearch + }) + }); + try { + return await sync.run(input); + } catch (ex) { + return this.manager.response.error(ex); + } + } + /** + * We are done. + */ + return this.manager.response.done(); + } + + private validateFlow(input: IDataSynchronizationInput): void { + if (!input.flow) { + throw new Error(`Missing "flow" in the input.`); + } else if (this.factories[input.flow]) { + return; + } + throw new Error( + `Invalid flow "${input.flow}". Allowed flows: ${Object.keys(this.factories).join( + ", " + )}.` + ); + } +} diff --git a/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/createFactories.ts b/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/createFactories.ts new file mode 100644 index 00000000000..dd1e20cecb3 --- /dev/null +++ b/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/createFactories.ts @@ -0,0 +1,10 @@ +import { IFactories } from "./types"; +import { ElasticsearchToDynamoDbSynchronization } from "./elasticsearch/ElasticsearchToDynamoDbSynchronization"; + +export const createFactories = (): IFactories => { + return { + elasticsearchToDynamoDb: params => { + return new ElasticsearchToDynamoDbSynchronization(params); + } + }; +}; diff --git a/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/elasticsearch/ElasticsearchFetcher.ts b/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/elasticsearch/ElasticsearchFetcher.ts new file mode 100644 index 00000000000..55f58abf601 --- /dev/null +++ b/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/elasticsearch/ElasticsearchFetcher.ts @@ -0,0 +1,107 @@ +import { Client } from "@webiny/api-elasticsearch"; +import { + IElasticsearchFetcher, + IElasticsearchFetcherFetchParams, + IElasticsearchFetcherFetchResponse, + IElasticsearchFetcherFetchResponseItem +} from "./abstractions/ElasticsearchFetcher"; +import { ElasticsearchSearchResponse, PrimitiveValue } from "@webiny/api-elasticsearch/types"; +import { shouldIgnoreEsResponseError } from "./shouldIgnoreEsResponseError"; +import { inspect } from "node:util"; + +export interface IElasticsearchFetcherParams { + client: Client; +} + +export class ElasticsearchFetcher implements IElasticsearchFetcher { + private readonly client: Client; + + public constructor(params: IElasticsearchFetcherParams) { + this.client = params.client; + } + public async fetch({ + index, + cursor, + limit + }: IElasticsearchFetcherFetchParams): Promise { + let response: ElasticsearchSearchResponse; + try { + response = await this.client.search({ + index, + body: { + query: { + match_all: {} + }, + sort: { + "id.keyword": { + order: "asc" + } + }, + size: limit + 1, + track_total_hits: true, + search_after: cursor, + _source: false + } + }); + } catch (ex) { + /** + * If we ignore the error, we can continue with the next index. + */ + if (shouldIgnoreEsResponseError(ex)) { + if (process.env.DEBUG === "true") { + console.error( + inspect(ex, { + depth: 5, + showHidden: true + }) + ); + } + return { + done: true, + totalCount: 0, + items: [] + }; + } + console.error("Failed to fetch data from Elasticsearch.", ex); + throw ex; + } + + const { hits, total } = response.body.hits; + if (hits.length === 0) { + return { + done: true, + cursor: undefined, + totalCount: total.value, + items: [] + }; + } + + const hasMoreItems = hits.length > limit; + let nextCursor: PrimitiveValue[] | undefined; + if (hasMoreItems) { + hits.pop(); + nextCursor = hits.at(-1)?.sort; + } + const items = hits.reduce((collection, hit) => { + const [PK, SK] = hit._id.split(":"); + if (!PK || !SK) { + return collection; + } + collection.push({ + PK, + SK, + _id: hit._id, + index: hit._index + }); + + return collection; + }, []); + + return { + totalCount: total.value, + cursor: nextCursor, + done: !nextCursor, + items + }; + } +} diff --git a/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/elasticsearch/ElasticsearchSynchronize.ts b/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/elasticsearch/ElasticsearchSynchronize.ts new file mode 100644 index 00000000000..428109e460a --- /dev/null +++ b/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/elasticsearch/ElasticsearchSynchronize.ts @@ -0,0 +1,101 @@ +import { batchReadAll } from "@webiny/db-dynamodb"; +import { createSynchronizationBuilder } from "@webiny/api-dynamodb-to-elasticsearch"; +import { + getElasticsearchEntity, + getElasticsearchEntityType, + getTable, + IGetElasticsearchEntityTypeParams +} from "~/tasks/dataSynchronization/entities"; +import { ITimer } from "@webiny/handler-aws"; +import { Context } from "~/types"; +import { + IElasticsearchSynchronize, + IElasticsearchSynchronizeExecuteParams, + IElasticsearchSynchronizeExecuteResponse +} from "./abstractions/ElasticsearchSynchronize"; + +export interface IElasticsearchSynchronizeParams { + timer: ITimer; + context: Context; +} + +interface IDynamoDbItem { + PK: string; + SK: string; +} + +export class ElasticsearchSynchronize implements IElasticsearchSynchronize { + private readonly timer: ITimer; + private readonly context: Context; + + public constructor(params: IElasticsearchSynchronizeParams) { + this.timer = params.timer; + this.context = params.context; + } + + public async execute( + params: IElasticsearchSynchronizeExecuteParams + ): Promise { + const { items, done, index } = params; + if (items.length === 0) { + return { + done: true + }; + } + + const table = getTable({ + type: "es", + context: this.context + }); + + const readableItems = items.map(item => { + const entity = this.getEntity(item); + return entity.item.getBatch({ + PK: item.PK, + SK: item.SK + }); + }); + + const tableItems = await batchReadAll({ + items: readableItems, + table + }); + + const elasticsearchSyncBuilder = createSynchronizationBuilder({ + timer: this.timer, + context: this.context + }); + /** + * We need to find the items we have in the Elasticsearch but not in the DynamoDB-Elasticsearch table. + */ + for (const item of items) { + const exists = tableItems.some(ddbItem => { + return ddbItem.PK === item.PK && ddbItem.SK === item.SK; + }); + if (exists) { + continue; + } + elasticsearchSyncBuilder.delete({ + index, + id: item._id + }); + } + + const executeWithRetry = elasticsearchSyncBuilder.build(); + await executeWithRetry(); + + return { + done + }; + } + + private getEntity( + params: IGetElasticsearchEntityTypeParams + ): ReturnType { + const type = getElasticsearchEntityType(params); + return getElasticsearchEntity({ + type, + context: this.context + }); + } +} diff --git a/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/elasticsearch/ElasticsearchToDynamoDbSynchronization.ts b/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/elasticsearch/ElasticsearchToDynamoDbSynchronization.ts new file mode 100644 index 00000000000..78815e12fe7 --- /dev/null +++ b/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/elasticsearch/ElasticsearchToDynamoDbSynchronization.ts @@ -0,0 +1,95 @@ +import { + IDataSynchronizationInput, + IDataSynchronizationManager, + IElasticsearchSyncParams, + ISynchronization, + ISynchronizationRunResult +} from "../types"; +import { IIndexManager } from "~/settings/types"; +import { NonEmptyArray } from "@webiny/api/types"; +import { IElasticsearchSynchronize } from "./abstractions/ElasticsearchSynchronize"; +import { IElasticsearchFetcher } from "./abstractions/ElasticsearchFetcher"; + +export class ElasticsearchToDynamoDbSynchronization implements ISynchronization { + private readonly manager: IDataSynchronizationManager; + private readonly indexManager: IIndexManager; + private readonly synchronize: IElasticsearchSynchronize; + private readonly fetcher: IElasticsearchFetcher; + + public constructor(params: IElasticsearchSyncParams) { + this.manager = params.manager; + this.indexManager = params.indexManager; + this.synchronize = params.synchronize; + this.fetcher = params.fetcher; + } + + public async run(input: IDataSynchronizationInput): Promise { + const lastIndex = input.elasticsearchToDynamoDb?.index; + let cursor = input.elasticsearchToDynamoDb?.cursor; + const indexes = await this.fetchAllIndexes(); + + let next = 0; + if (lastIndex) { + next = indexes.findIndex(index => index === lastIndex); + } + + let currentIndex = indexes[next]; + + while (currentIndex) { + if (this.manager.isAborted()) { + return this.manager.response.aborted(); + } + /** + * We will put 180 seconds because we are writing to the Elasticsearch/OpenSearch directly. + * We want to leave enough time for possible retries. + */ + // + else if (this.manager.isCloseToTimeout(180)) { + return this.manager.response.continue({ + ...input, + elasticsearchToDynamoDb: { + ...input.elasticsearchToDynamoDb, + index: currentIndex, + cursor + } + }); + } + + const result = await this.fetcher.fetch({ + index: currentIndex, + cursor, + limit: 100 + }); + + const syncResult = await this.synchronize.execute({ + done: result.done, + index: currentIndex, + items: result.items + }); + + if (!syncResult.done && result.cursor) { + cursor = result.cursor; + continue; + } + cursor = undefined; + + const next = indexes.findIndex(index => index === currentIndex) + 1; + currentIndex = indexes[next]; + } + + return this.manager.response.continue({ + ...input, + elasticsearchToDynamoDb: { + finished: true + } + }); + } + + private async fetchAllIndexes(): Promise> { + const result = await this.indexManager.list(); + if (result.length > 0) { + return result as NonEmptyArray; + } + throw new Error("No Elasticsearch / OpenSearch indexes found."); + } +} diff --git a/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/elasticsearch/abstractions/ElasticsearchFetcher.ts b/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/elasticsearch/abstractions/ElasticsearchFetcher.ts new file mode 100644 index 00000000000..242bec7f4d8 --- /dev/null +++ b/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/elasticsearch/abstractions/ElasticsearchFetcher.ts @@ -0,0 +1,25 @@ +import { PrimitiveValue } from "@webiny/api-elasticsearch/types"; + +export interface IElasticsearchFetcherFetchResponseItem { + PK: string; + SK: string; + _id: string; + index: string; +} + +export interface IElasticsearchFetcherFetchParams { + index: string; + cursor?: PrimitiveValue[]; + limit: number; +} + +export interface IElasticsearchFetcherFetchResponse { + done: boolean; + totalCount: number; + cursor?: PrimitiveValue[]; + items: IElasticsearchFetcherFetchResponseItem[]; +} + +export interface IElasticsearchFetcher { + fetch(params: IElasticsearchFetcherFetchParams): Promise; +} diff --git a/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/elasticsearch/abstractions/ElasticsearchSynchronize.ts b/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/elasticsearch/abstractions/ElasticsearchSynchronize.ts new file mode 100644 index 00000000000..09f48a5c763 --- /dev/null +++ b/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/elasticsearch/abstractions/ElasticsearchSynchronize.ts @@ -0,0 +1,21 @@ +export interface IElasticsearchSynchronizeExecuteParamsItem { + PK: string; + SK: string; + _id: string; + index: string; +} + +export interface IElasticsearchSynchronizeExecuteParams { + done: boolean; + index: string; + items: IElasticsearchSynchronizeExecuteParamsItem[]; +} + +export interface IElasticsearchSynchronizeExecuteResponse { + done: boolean; +} +export interface IElasticsearchSynchronize { + execute( + params: IElasticsearchSynchronizeExecuteParams + ): Promise; +} diff --git a/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/elasticsearch/shouldIgnoreEsResponseError.ts b/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/elasticsearch/shouldIgnoreEsResponseError.ts new file mode 100644 index 00000000000..b4d76ba15ad --- /dev/null +++ b/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/elasticsearch/shouldIgnoreEsResponseError.ts @@ -0,0 +1,11 @@ +import WebinyError from "@webiny/error"; + +const IGNORED_ES_SEARCH_EXCEPTIONS = [ + "index_not_found_exception", + "search_phase_execution_exception", + "illegal_argument_exception" +]; + +export const shouldIgnoreEsResponseError = (error: WebinyError) => { + return IGNORED_ES_SEARCH_EXCEPTIONS.includes(error.message); +}; diff --git a/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/entities/getElasticsearchEntity.ts b/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/entities/getElasticsearchEntity.ts new file mode 100644 index 00000000000..cc20f4061ef --- /dev/null +++ b/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/entities/getElasticsearchEntity.ts @@ -0,0 +1,52 @@ +import { Entity } from "@webiny/db-dynamodb/toolbox"; +import { NonEmptyArray } from "@webiny/api/types"; +import { IRegistryItem } from "@webiny/db"; +import { EntityType } from "./getElasticsearchEntityType"; +import { Context } from "~/types"; + +export interface IGetElasticsearchEntityParams { + type: EntityType | unknown; + context: Pick; +} + +const createPredicate = (app: string, tags: NonEmptyArray) => { + return (item: IRegistryItem) => { + return item.app === app && tags.every(tag => item.tags.includes(tag)); + }; +}; + +export const getElasticsearchEntity = (params: IGetElasticsearchEntityParams) => { + const { type, context } = params; + + const getByPredicate = (predicate: (item: IRegistryItem) => boolean) => { + return context.db.registry.getOneItem(predicate); + }; + + try { + switch (type) { + case EntityType.CMS: + return getByPredicate(createPredicate("cms", ["es"])); + case EntityType.PAGE_BUILDER: + return getByPredicate(createPredicate("pb", ["es"])); + case EntityType.FORM_BUILDER: + return getByPredicate(createPredicate("fb", ["es"])); + case EntityType.FORM_BUILDER_SUBMISSION: + return getByPredicate(createPredicate("fb", ["es", "form-submission"])); + } + } catch (ex) {} + throw new Error(`Unknown entity type "${type}".`); +}; + +export interface IListElasticsearchEntitiesParams { + context: Pick; +} + +export const listElasticsearchEntities = ( + params: IListElasticsearchEntitiesParams +): IRegistryItem[] => { + const { context } = params; + + return context.db.registry.getItems(item => { + return item.tags.includes("es"); + }); +}; diff --git a/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/entities/getElasticsearchEntityType.ts b/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/entities/getElasticsearchEntityType.ts new file mode 100644 index 00000000000..5501193cf75 --- /dev/null +++ b/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/entities/getElasticsearchEntityType.ts @@ -0,0 +1,27 @@ +export enum EntityType { + CMS = "headless-cms", + PAGE_BUILDER = "page-builder", + FORM_BUILDER = "form-builder", + FORM_BUILDER_SUBMISSION = "form-builder-submission" +} + +export interface IGetElasticsearchEntityTypeParams { + SK: string; + index: string; +} + +export const getElasticsearchEntityType = ( + params: IGetElasticsearchEntityTypeParams +): EntityType => { + if (params.index.includes("-headless-cms-")) { + return EntityType.CMS; + } else if (params.index.endsWith("-page-builder")) { + return EntityType.PAGE_BUILDER; + } else if (params.index.endsWith("-form-builder")) { + if (params.SK.startsWith("FS#")) { + return EntityType.FORM_BUILDER_SUBMISSION; + } + return EntityType.FORM_BUILDER; + } + throw new Error(`Unknown entity type for item "${JSON.stringify(params)}".`); +}; diff --git a/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/entities/getTable.ts b/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/entities/getTable.ts new file mode 100644 index 00000000000..bb0e5720b9b --- /dev/null +++ b/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/entities/getTable.ts @@ -0,0 +1,30 @@ +import { Entity, TableDef } from "@webiny/db-dynamodb/toolbox"; +import { Context } from "~/types"; +import { NonEmptyArray } from "@webiny/api/types"; +import { IRegistryItem } from "@webiny/db"; + +export interface IGetTableParams { + context: Pick; + type: "regular" | "es"; +} + +const createPredicate = (app: string, tags: NonEmptyArray) => { + return (item: IRegistryItem) => { + return item.app === app && tags.every(tag => item.tags.includes(tag)); + }; +}; + +export const getTable = (params: IGetTableParams): TableDef => { + const { context, type } = params; + + const getByPredicate = (predicate: (item: IRegistryItem) => boolean) => { + const item = context.db.registry.getOneItem(predicate); + return item.item; + }; + + const entity = getByPredicate(createPredicate("cms", [type])); + if (!entity) { + throw new Error(`Unknown entity type "${type}".`); + } + return entity.table as TableDef; +}; diff --git a/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/entities/index.ts b/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/entities/index.ts new file mode 100644 index 00000000000..938b45e64ab --- /dev/null +++ b/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/entities/index.ts @@ -0,0 +1,3 @@ +export * from "./getElasticsearchEntity"; +export * from "./getElasticsearchEntityType"; +export * from "./getTable"; diff --git a/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/index.ts b/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/index.ts new file mode 100644 index 00000000000..19c4473f69b --- /dev/null +++ b/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/index.ts @@ -0,0 +1,79 @@ +import { createTaskDefinition } from "@webiny/tasks"; +import { Context, IElasticsearchTaskConfig } from "~/types"; +import { + IDataSynchronizationInput, + IDataSynchronizationManager, + IDataSynchronizationOutput +} from "~/tasks/dataSynchronization/types"; + +export const DATA_SYNCHRONIZATION_TASK = "dataSynchronization"; + +export const createDataSynchronization = (params?: IElasticsearchTaskConfig) => { + return createTaskDefinition({ + id: DATA_SYNCHRONIZATION_TASK, + isPrivate: false, + title: "Data Synchronization", + description: "Synchronize data between Elasticsearch and DynamoDB", + maxIterations: 100, + disableDatabaseLogs: true, + async run({ context, response, isCloseToTimeout, isAborted, store, input, timer }) { + const { Manager } = await import( + /* webpackChunkName: "Manager" */ + "../Manager" + ); + + const { IndexManager } = await import( + /* webpackChunkName: "IndexManager" */ "~/settings" + ); + + const manager = new Manager({ + elasticsearchClient: params?.elasticsearchClient, + documentClient: params?.documentClient, + response, + context, + isAborted, + isCloseToTimeout, + store, + timer + }); + + const indexManager = new IndexManager(manager.elasticsearch, {}); + + const { DataSynchronizationTaskRunner } = await import( + /* webpackChunkName: "DataSynchronizationTaskRunner" */ "./DataSynchronizationTaskRunner" + ); + + const { createFactories } = await import( + /* webpackChunkName: "createFactories" */ "./createFactories" + ); + + try { + const dataSynchronization = new DataSynchronizationTaskRunner({ + manager: manager as unknown as IDataSynchronizationManager, + indexManager, + factories: createFactories() + }); + return await dataSynchronization.run({ + ...input + }); + } catch (ex) { + return response.error(ex); + } + }, + createInputValidation({ validator }) { + return { + flow: validator.enum(["elasticsearchToDynamoDb"]), + elasticsearchToDynamoDb: validator + .object({ + finished: validator.boolean().optional().default(false), + index: validator.string().optional(), + cursor: validator.array(validator.string()).optional() + }) + .optional() + .default({ + finished: false + }) + }; + } + }); +}; diff --git a/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/types.ts b/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/types.ts new file mode 100644 index 00000000000..c73f7f2d573 --- /dev/null +++ b/packages/api-elasticsearch-tasks/src/tasks/dataSynchronization/types.ts @@ -0,0 +1,62 @@ +import { IManager } from "~/types"; +import { PrimitiveValue } from "@webiny/api-elasticsearch/types"; +import { IIndexManager } from "~/settings/types"; +import { + ITaskResponseAbortedResult, + ITaskResponseContinueResult, + ITaskResponseDoneResult, + ITaskResponseDoneResultOutput, + ITaskResponseErrorResult +} from "@webiny/tasks"; +import { IElasticsearchSynchronize } from "~/tasks/dataSynchronization/elasticsearch/abstractions/ElasticsearchSynchronize"; +import { IElasticsearchFetcher } from "~/tasks/dataSynchronization/elasticsearch/abstractions/ElasticsearchFetcher"; + +export interface IDataSynchronizationInputValue { + finished?: boolean; +} + +export interface IDataSynchronizationInputElasticsearchToDynamoDbValue + extends IDataSynchronizationInputValue { + index?: string; + cursor?: PrimitiveValue[]; +} + +export interface IDataSynchronizationInput { + flow: "elasticsearchToDynamoDb"; + elasticsearchToDynamoDb?: IDataSynchronizationInputElasticsearchToDynamoDbValue; +} + +export type IDataSynchronizationOutput = ITaskResponseDoneResultOutput; + +export type ISynchronizationRunResult = + | ITaskResponseContinueResult + | ITaskResponseDoneResult + | ITaskResponseErrorResult + | ITaskResponseAbortedResult; + +export interface ISynchronization { + run(input: IDataSynchronizationInput): Promise; +} + +export interface IElasticsearchSyncParams { + manager: IDataSynchronizationManager; + indexManager: IIndexManager; + synchronize: IElasticsearchSynchronize; + fetcher: IElasticsearchFetcher; +} + +export interface IElasticsearchSyncFactory { + (params: IElasticsearchSyncParams): ISynchronization; +} + +export interface IFactories { + /** + * Delete all the records which are in the Elasticsearch but not in the Elasticsearch DynamoDB table. + */ + elasticsearchToDynamoDb: IElasticsearchSyncFactory; +} + +export type IDataSynchronizationManager = IManager< + IDataSynchronizationInput, + IDataSynchronizationOutput +>; diff --git a/packages/api-elasticsearch-tasks/src/tasks/enableIndexing/EnableIndexingTaskRunner.ts b/packages/api-elasticsearch-tasks/src/tasks/enableIndexing/EnableIndexingTaskRunner.ts index 938477cbbbf..3dca288e2d4 100644 --- a/packages/api-elasticsearch-tasks/src/tasks/enableIndexing/EnableIndexingTaskRunner.ts +++ b/packages/api-elasticsearch-tasks/src/tasks/enableIndexing/EnableIndexingTaskRunner.ts @@ -2,13 +2,17 @@ import { IManager } from "~/types"; import { ITaskResponse, ITaskResponseResult } from "@webiny/tasks/response/abstractions"; import { IndexManager } from "~/settings"; import { IIndexManager } from "~/settings/types"; +import { IElasticsearchEnableIndexingTaskInput } from "~/tasks/enableIndexing/types"; export class EnableIndexingTaskRunner { - private readonly manager: IManager; + private readonly manager: IManager; private readonly indexManager: IIndexManager; private readonly response: ITaskResponse; - public constructor(manager: IManager, indexManager: IndexManager) { + public constructor( + manager: IManager, + indexManager: IndexManager + ) { this.manager = manager; this.response = manager.response; this.indexManager = indexManager; diff --git a/packages/api-elasticsearch-tasks/src/tasks/enableIndexing/index.ts b/packages/api-elasticsearch-tasks/src/tasks/enableIndexing/index.ts index 6c64ca3c10e..a310ad80db0 100644 --- a/packages/api-elasticsearch-tasks/src/tasks/enableIndexing/index.ts +++ b/packages/api-elasticsearch-tasks/src/tasks/enableIndexing/index.ts @@ -6,27 +6,28 @@ export const createEnableIndexingTask = (params?: IElasticsearchTaskConfig) => { return createTaskDefinition({ id: "elasticsearchEnableIndexing", title: "Enable Indexing on Elasticsearch Indexes", - run: async ({ response, context, isAborted, isCloseToTimeout, input, store }) => { + run: async ({ response, context, isAborted, isCloseToTimeout, input, store, timer }) => { const { Manager } = await import( - /* webpackChunkName: "ElasticsearchTaskManager" */ + /* webpackChunkName: "Manager" */ "../Manager" ); const { IndexManager } = await import( - /* webpackChunkName: "ElasticsearchTaskSettings" */ "~/settings" + /* webpackChunkName: "IndexManager" */ "~/settings" ); const { EnableIndexingTaskRunner } = await import( - /* webpackChunkName: "ElasticsearchEnableIndexingTaskRunner" */ "./EnableIndexingTaskRunner" + /* webpackChunkName: "EnableIndexingTaskRunner" */ "./EnableIndexingTaskRunner" ); - const manager = new Manager({ + const manager = new Manager({ elasticsearchClient: params?.elasticsearchClient, documentClient: params?.documentClient, response, context, isAborted, isCloseToTimeout, - store + store, + timer }); const indexManager = new IndexManager( diff --git a/packages/api-elasticsearch-tasks/src/tasks/index.ts b/packages/api-elasticsearch-tasks/src/tasks/index.ts index 52680e600ca..61972c2d768 100644 --- a/packages/api-elasticsearch-tasks/src/tasks/index.ts +++ b/packages/api-elasticsearch-tasks/src/tasks/index.ts @@ -1,3 +1,4 @@ export * from "./enableIndexing"; +export * from "./dataSynchronization"; export * from "./reindexing"; export * from "./createIndexes"; diff --git a/packages/api-elasticsearch-tasks/src/tasks/reindexing/ReindexingTaskRunner.ts b/packages/api-elasticsearch-tasks/src/tasks/reindexing/ReindexingTaskRunner.ts index 85f974b667f..908d8d911b7 100644 --- a/packages/api-elasticsearch-tasks/src/tasks/reindexing/ReindexingTaskRunner.ts +++ b/packages/api-elasticsearch-tasks/src/tasks/reindexing/ReindexingTaskRunner.ts @@ -1,5 +1,6 @@ import { IDynamoDbElasticsearchRecord, + IElasticsearchIndexingTaskValues, IElasticsearchIndexingTaskValuesKeys, IManager } from "~/types"; @@ -20,13 +21,16 @@ const getKeys = (results: ScanResponse): IElasticsearchIndexingTaskValuesKeys | }; export class ReindexingTaskRunner { - private readonly manager: IManager; + private readonly manager: IManager; private keys?: IElasticsearchIndexingTaskValuesKeys; private readonly indexManager: IIndexManager; private readonly response: ITaskResponse; - public constructor(manager: IManager, indexManager: IndexManager) { + public constructor( + manager: IManager, + indexManager: IndexManager + ) { this.manager = manager; this.response = manager.response; this.indexManager = indexManager; diff --git a/packages/api-elasticsearch-tasks/src/tasks/reindexing/reindexingTaskDefinition.ts b/packages/api-elasticsearch-tasks/src/tasks/reindexing/reindexingTaskDefinition.ts index 6194a69018d..851276356e2 100644 --- a/packages/api-elasticsearch-tasks/src/tasks/reindexing/reindexingTaskDefinition.ts +++ b/packages/api-elasticsearch-tasks/src/tasks/reindexing/reindexingTaskDefinition.ts @@ -5,26 +5,27 @@ export const createElasticsearchReindexingTask = (params?: IElasticsearchTaskCon return createTaskDefinition({ id: "elasticsearchReindexing", title: "Elasticsearch reindexing", - run: async ({ context, isCloseToTimeout, response, input, isAborted, store }) => { + run: async ({ context, isCloseToTimeout, response, input, isAborted, store, timer }) => { const { Manager } = await import( - /* webpackChunkName: "ElasticsearchReindexingManager" */ + /* webpackChunkName: "Manager" */ "../Manager" ); const { IndexManager } = await import( - /* webpackChunkName: "ElasticsearchReindexingSettings" */ "~/settings" + /* webpackChunkName: "IndexManager" */ "~/settings" ); const { ReindexingTaskRunner } = await import( - /* webpackChunkName: "ElasticsearchReindexingTaskRunner" */ "./ReindexingTaskRunner" + /* webpackChunkName: "ReindexingTaskRunner" */ "./ReindexingTaskRunner" ); - const manager = new Manager({ + const manager = new Manager({ elasticsearchClient: params?.elasticsearchClient, documentClient: params?.documentClient, response, context, isAborted, isCloseToTimeout, - store + store, + timer }); const indexManager = new IndexManager(manager.elasticsearch, input.settings || {}); diff --git a/packages/api-elasticsearch-tasks/src/types.ts b/packages/api-elasticsearch-tasks/src/types.ts index 2b832cc6621..b2d5b276551 100644 --- a/packages/api-elasticsearch-tasks/src/types.ts +++ b/packages/api-elasticsearch-tasks/src/types.ts @@ -1,12 +1,17 @@ import { ElasticsearchContext } from "@webiny/api-elasticsearch/types"; import { Entity } from "@webiny/db-dynamodb/toolbox"; -import { Context as TasksContext } from "@webiny/tasks/types"; +import { + Context as TasksContext, + IIsCloseToTimeoutCallable, + ITaskResponseDoneResultOutput +} from "@webiny/tasks/types"; import { DynamoDBDocument } from "@webiny/aws-sdk/client-dynamodb"; import { Client } from "@webiny/api-elasticsearch"; import { createTable } from "~/definitions"; import { ITaskResponse } from "@webiny/tasks/response/abstractions"; import { ITaskManagerStore } from "@webiny/tasks/runner/abstractions"; import { BatchWriteItem, BatchWriteResult } from "@webiny/db-dynamodb"; +import { ITimer } from "@webiny/handler-aws"; export interface Context extends ElasticsearchContext, TasksContext {} @@ -51,15 +56,19 @@ export interface IDynamoDbElasticsearchRecord { modified: string; } -export interface IManager { +export interface IManager< + T, + O extends ITaskResponseDoneResultOutput = ITaskResponseDoneResultOutput +> { readonly documentClient: DynamoDBDocument; readonly elasticsearch: Client; readonly context: Context; readonly table: ReturnType; - readonly isCloseToTimeout: () => boolean; + readonly isCloseToTimeout: IIsCloseToTimeoutCallable; readonly isAborted: () => boolean; - readonly response: ITaskResponse; - readonly store: ITaskManagerStore; + readonly response: ITaskResponse; + readonly store: ITaskManagerStore; + readonly timer: ITimer; getEntity: (name: string) => Entity; diff --git a/packages/api-elasticsearch-tasks/tsconfig.build.json b/packages/api-elasticsearch-tasks/tsconfig.build.json index 014a0b3fcdf..68bbfbb2671 100644 --- a/packages/api-elasticsearch-tasks/tsconfig.build.json +++ b/packages/api-elasticsearch-tasks/tsconfig.build.json @@ -3,8 +3,10 @@ "include": ["src"], "references": [ { "path": "../api/tsconfig.build.json" }, + { "path": "../api-dynamodb-to-elasticsearch/tsconfig.build.json" }, { "path": "../api-elasticsearch/tsconfig.build.json" }, { "path": "../aws-sdk/tsconfig.build.json" }, + { "path": "../db/tsconfig.build.json" }, { "path": "../db-dynamodb/tsconfig.build.json" }, { "path": "../error/tsconfig.build.json" }, { "path": "../tasks/tsconfig.build.json" }, diff --git a/packages/api-elasticsearch-tasks/tsconfig.json b/packages/api-elasticsearch-tasks/tsconfig.json index f2032353ca0..d80ecd6f23f 100644 --- a/packages/api-elasticsearch-tasks/tsconfig.json +++ b/packages/api-elasticsearch-tasks/tsconfig.json @@ -3,8 +3,10 @@ "include": ["src", "__tests__"], "references": [ { "path": "../api" }, + { "path": "../api-dynamodb-to-elasticsearch" }, { "path": "../api-elasticsearch" }, { "path": "../aws-sdk" }, + { "path": "../db" }, { "path": "../db-dynamodb" }, { "path": "../error" }, { "path": "../tasks" }, @@ -29,10 +31,14 @@ "~tests/*": ["./__tests__/*"], "@webiny/api/*": ["../api/src/*"], "@webiny/api": ["../api/src"], + "@webiny/api-dynamodb-to-elasticsearch/*": ["../api-dynamodb-to-elasticsearch/src/*"], + "@webiny/api-dynamodb-to-elasticsearch": ["../api-dynamodb-to-elasticsearch/src"], "@webiny/api-elasticsearch/*": ["../api-elasticsearch/src/*"], "@webiny/api-elasticsearch": ["../api-elasticsearch/src"], "@webiny/aws-sdk/*": ["../aws-sdk/src/*"], "@webiny/aws-sdk": ["../aws-sdk/src"], + "@webiny/db/*": ["../db/src/*"], + "@webiny/db": ["../db/src"], "@webiny/db-dynamodb/*": ["../db-dynamodb/src/*"], "@webiny/db-dynamodb": ["../db-dynamodb/src"], "@webiny/error/*": ["../error/src/*"], diff --git a/packages/api-elasticsearch/src/cursors.ts b/packages/api-elasticsearch/src/cursors.ts index d6c116d1fd4..a6984671259 100644 --- a/packages/api-elasticsearch/src/cursors.ts +++ b/packages/api-elasticsearch/src/cursors.ts @@ -3,12 +3,16 @@ import { PrimitiveValue } from "~/types"; /** * Encode a received cursor value into something that can be passed on to the user. */ -export const encodeCursor = (cursor?: string | string[] | null): string | undefined => { - if (!cursor) { +export const encodeCursor = (input?: PrimitiveValue[]): string | undefined => { + if (!input) { return undefined; } - cursor = Array.isArray(cursor) ? cursor.map(encodeURIComponent) : encodeURIComponent(cursor); + const cursor = Array.isArray(input) + ? input + .filter((item: PrimitiveValue): item is string | number | boolean => item !== null) + .map(item => encodeURIComponent(item)) + : encodeURIComponent(input); try { return Buffer.from(JSON.stringify(cursor)).toString("base64"); @@ -28,7 +32,7 @@ export const decodeCursor = (cursor?: string | null): PrimitiveValue[] | undefin try { const value = JSON.parse(Buffer.from(cursor, "base64").toString("ascii")); if (Array.isArray(value)) { - return value.map(decodeURIComponent); + return value.filter(item => item !== null).map(decodeURIComponent); } const decoded = decodeURIComponent(value); return decoded ? [decoded] : undefined; diff --git a/packages/api-elasticsearch/src/types.ts b/packages/api-elasticsearch/src/types.ts index 8b0ac7c5dde..a57bae522ea 100644 --- a/packages/api-elasticsearch/src/types.ts +++ b/packages/api-elasticsearch/src/types.ts @@ -1,6 +1,6 @@ -import { Client, ApiResponse } from "@elastic/elasticsearch"; -import { BoolQueryConfig as esBoolQueryConfig, Query as esQuery } from "elastic-ts"; -import { Context } from "@webiny/api/types"; +import { ApiResponse, Client } from "@elastic/elasticsearch"; +import { BoolQueryConfig, PrimitiveValue, Query as esQuery } from "elastic-ts"; +import { Context, GenericRecord } from "@webiny/api/types"; /** * Re-export some dep lib types. */ @@ -15,7 +15,7 @@ export interface ElasticsearchContext extends Context { * To simplify our plugins, we say that query contains arrays of objects, not single objects. * And that they all are defined as empty arrays at the start. */ -export interface ElasticsearchBoolQueryConfig extends esBoolQueryConfig { +export interface ElasticsearchBoolQueryConfig extends BoolQueryConfig { must: esQuery[]; filter: esQuery[]; should: esQuery[]; @@ -77,29 +77,40 @@ export interface ElasticsearchQueryBuilderArgsPlugin { * Elasticsearch responses. */ export interface ElasticsearchSearchResponseHit { + _index: string; + _type: string; + _id: string; + _score: number | null; _source: T; - sort: string; + sort: PrimitiveValue[]; } export interface ElasticsearchSearchResponseAggregationBucket { key: T; doc_count: number; } -export interface ElasticsearchSearchResponse { - body: { - hits: { - hits: ElasticsearchSearchResponseHit[]; - total: { - value: number; - }; - }; - aggregations: { - [key: string]: { - buckets: ElasticsearchSearchResponseAggregationBucket[]; - }; - }; + +export interface ElasticsearchSearchResponseBodyHits { + hits: ElasticsearchSearchResponseHit[]; + total: { + value: number; + }; +} + +export interface ElasticsearchSearchResponseBodyAggregations { + [key: string]: { + buckets: ElasticsearchSearchResponseAggregationBucket[]; }; } +export interface ElasticsearchSearchResponseBody { + hits: ElasticsearchSearchResponseBodyHits; + aggregations: ElasticsearchSearchResponseBodyAggregations; +} + +export interface ElasticsearchSearchResponse { + body: ElasticsearchSearchResponseBody; +} + export interface ElasticsearchIndexRequestBodyMappingsDynamicTemplate { [key: string]: { path_match?: string; diff --git a/packages/api-form-builder-so-ddb-es/src/index.ts b/packages/api-form-builder-so-ddb-es/src/index.ts index 1968a95b18d..33773290324 100644 --- a/packages/api-form-builder-so-ddb-es/src/index.ts +++ b/packages/api-form-builder-so-ddb-es/src/index.ts @@ -139,6 +139,27 @@ export const createFormBuilderStorageOperations: FormBuilderStorageOperationsFac return { beforeInit: async (context: FormBuilderContext) => { + context.db.registry.register({ + item: entities.form, + app: "fb", + tags: ["regular", "form", entities.form.name] + }); + context.db.registry.register({ + item: entities.esForm, + app: "fb", + tags: ["es", "form", entities.esForm.name] + }); + context.db.registry.register({ + item: entities.submission, + app: "fb", + tags: ["regular", "form-submission", entities.submission.name] + }); + context.db.registry.register({ + item: entities.esSubmission, + app: "fb", + tags: ["es", "form-submission", entities.esSubmission.name] + }); + const types: string[] = [ // Elasticsearch CompressionPlugin.type, diff --git a/packages/api-form-builder-so-ddb-es/src/types.ts b/packages/api-form-builder-so-ddb-es/src/types.ts index 8fbba21e172..8b9a96a8289 100644 --- a/packages/api-form-builder-so-ddb-es/src/types.ts +++ b/packages/api-form-builder-so-ddb-es/src/types.ts @@ -1,26 +1,24 @@ import { + FormBuilder, + FormBuilderContext as BaseFormBuilderContext, + FormBuilderFormStorageOperations as BaseFormBuilderFormStorageOperations, + FormBuilderSettingsStorageOperations as BaseFormBuilderSettingsStorageOperations, FormBuilderStorageOperations as BaseFormBuilderStorageOperations, - FormBuilderSystemStorageOperations as BaseFormBuilderSystemStorageOperations, FormBuilderSubmissionStorageOperations as BaseFormBuilderSubmissionStorageOperations, - FormBuilderSettingsStorageOperations as BaseFormBuilderSettingsStorageOperations, - FormBuilderFormStorageOperations as BaseFormBuilderFormStorageOperations, - FormBuilderContext + FormBuilderSystemStorageOperations as BaseFormBuilderSystemStorageOperations } from "@webiny/api-form-builder/types"; import { DynamoDBDocument } from "@webiny/aws-sdk/client-dynamodb"; -import { Entity, Table } from "@webiny/db-dynamodb/toolbox"; -import { AttributeDefinition } from "@webiny/db-dynamodb/toolbox"; +import { AttributeDefinition, Entity, Table } from "@webiny/db-dynamodb/toolbox"; import { Client } from "@elastic/elasticsearch"; import { PluginCollection } from "@webiny/plugins/types"; -export { FormBuilderContext }; - export type Attributes = Record; export enum ENTITIES { FORM = "FormBuilderForm", + ES_SUBMISSION = "FormBuilderSubmissionEs", ES_FORM = "FormBuilderFormEs", SUBMISSION = "FormBuilderSubmission", - ES_SUBMISSION = "FormBuilderSubmissionEs", SYSTEM = "FormBuilderSystem", SETTINGS = "FormBuilderSettings" } @@ -96,3 +94,9 @@ export interface FormBuilderStorageOperations export interface FormBuilderStorageOperationsFactory { (params: FormBuilderStorageOperationsFactoryParams): FormBuilderStorageOperations; } + +export interface FormBuilderContext extends BaseFormBuilderContext { + formBuilder: FormBuilder & { + storageOperations: FormBuilderStorageOperations; + }; +} diff --git a/packages/api-headless-cms-ddb-es/src/index.ts b/packages/api-headless-cms-ddb-es/src/index.ts index b4bf759c8c1..7c0dbad00e8 100644 --- a/packages/api-headless-cms-ddb-es/src/index.ts +++ b/packages/api-headless-cms-ddb-es/src/index.ts @@ -124,6 +124,16 @@ export const createStorageOperations: StorageOperationsFactory = params => { return { name: "dynamodb:elasticsearch", beforeInit: async context => { + context.db.registry.register({ + item: entities.entries, + app: "cms", + tags: ["regular", entities.entries.name] + }); + context.db.registry.register({ + item: entities.entriesEs, + app: "cms", + tags: ["es", entities.entriesEs.name] + }); /** * Attach the elasticsearch into context if it is not already attached. */ diff --git a/packages/api-headless-cms-ddb-es/src/types.ts b/packages/api-headless-cms-ddb-es/src/types.ts index 7618bf64df4..e7fae5a605f 100644 --- a/packages/api-headless-cms-ddb-es/src/types.ts +++ b/packages/api-headless-cms-ddb-es/src/types.ts @@ -7,14 +7,14 @@ import { CmsModelField, CmsModelFieldToGraphQLPlugin, CmsModelFieldType, + HeadlessCms, HeadlessCmsStorageOperations as BaseHeadlessCmsStorageOperations } from "@webiny/api-headless-cms/types"; -import { TableConstructor } from "@webiny/db-dynamodb/toolbox"; +import { AttributeDefinition, Entity, Table, TableConstructor } from "@webiny/db-dynamodb/toolbox"; import { DynamoDBDocument } from "@webiny/aws-sdk/client-dynamodb"; -import { AttributeDefinition } from "@webiny/db-dynamodb/toolbox"; import { Client } from "@elastic/elasticsearch"; -import { Entity, Table } from "@webiny/db-dynamodb/toolbox"; import { PluginsContainer } from "@webiny/plugins"; +import { ElasticsearchContext } from "@webiny/api-elasticsearch/types"; /** * A definition of the entry that is being prepared for the Elasticsearch. @@ -167,6 +167,12 @@ export interface StorageOperationsFactoryParams { plugins?: PluginCollection; } +export interface CmsContext extends BaseCmsContext, ElasticsearchContext { + cms: HeadlessCms & { + storageOperations: HeadlessCmsStorageOperations; + }; +} + export interface HeadlessCmsStorageOperations extends BaseHeadlessCmsStorageOperations { getTable: () => Table; getEsTable: () => Table; @@ -180,10 +186,6 @@ export interface StorageOperationsFactory { (params: StorageOperationsFactoryParams): HeadlessCmsStorageOperations; } -export interface CmsContext extends BaseCmsContext { - [key: string]: any; -} - export interface CmsEntryStorageOperations extends BaseCmsEntryStorageOperations { dataLoaders: DataLoadersHandlerInterface; } diff --git a/packages/api-page-builder-so-ddb-es/src/index.ts b/packages/api-page-builder-so-ddb-es/src/index.ts index ff9a54f69f2..73d8f53705d 100644 --- a/packages/api-page-builder-so-ddb-es/src/index.ts +++ b/packages/api-page-builder-so-ddb-es/src/index.ts @@ -208,6 +208,16 @@ export const createStorageOperations: StorageOperationsFactory = params => { return { beforeInit: async (context: PbContext) => { + context.db.registry.register({ + item: entities.pages, + app: "pb", + tags: ["regular", entities.pages.name] + }); + context.db.registry.register({ + item: entities.pagesEs, + app: "pb", + tags: ["es", entities.pagesEs.name] + }); const types: string[] = [ // Elasticsearch CompressionPlugin.type, diff --git a/packages/api-page-builder-so-ddb-es/src/types.ts b/packages/api-page-builder-so-ddb-es/src/types.ts index 6201f33b364..f0721d7cfbd 100644 --- a/packages/api-page-builder-so-ddb-es/src/types.ts +++ b/packages/api-page-builder-so-ddb-es/src/types.ts @@ -2,18 +2,15 @@ import { BlockCategoryStorageOperations as BaseBlockCategoryStorageOperations, CategoryStorageOperations as BaseCategoryStorageOperations, PageBlockStorageOperations as BasePageBlockStorageOperations, + PageBuilderContextObject, PageBuilderStorageOperations as BasePageBuilderStorageOperations, PageTemplateStorageOperations as BasePageTemplateStorageOperations, - PbContext + PbContext as BasePbContext } from "@webiny/api-page-builder/types"; -import { Entity, Table } from "@webiny/db-dynamodb/toolbox"; +import { AttributeDefinition, Entity, Table, TableConstructor } from "@webiny/db-dynamodb/toolbox"; import { DynamoDBDocument } from "@webiny/aws-sdk/client-dynamodb"; import { Client } from "@elastic/elasticsearch"; import { PluginCollection } from "@webiny/plugins/types"; -import { TableConstructor } from "@webiny/db-dynamodb/toolbox"; -import { AttributeDefinition } from "@webiny/db-dynamodb/toolbox"; - -export { PbContext }; export type Attributes = Record; @@ -52,6 +49,12 @@ export interface PageBuilderStorageOperations extends BasePageBuilderStorageOper >; } +export interface PbContext extends BasePbContext { + pageBuilder: PageBuilderContextObject & { + storageOperations: PageBuilderStorageOperations; + }; +} + export interface StorageOperationsFactoryParams { documentClient: DynamoDBDocument; elasticsearch: Client; diff --git a/packages/db-dynamodb/src/DynamoDbDriver.ts b/packages/db-dynamodb/src/DynamoDbDriver.ts index 45b5a13d2a5..7e3ccf28ec9 100644 --- a/packages/db-dynamodb/src/DynamoDbDriver.ts +++ b/packages/db-dynamodb/src/DynamoDbDriver.ts @@ -1,48 +1,19 @@ import { DynamoDBDocument } from "@webiny/aws-sdk/client-dynamodb"; -import { DbDriver, Result } from "@webiny/db"; +import { DbDriver } from "@webiny/db"; interface ConstructorArgs { documentClient: DynamoDBDocument; } -class DynamoDbDriver implements DbDriver { - batchProcesses: Record; - documentClient: DynamoDBDocument; +class DynamoDbDriver implements DbDriver { + public readonly documentClient: DynamoDBDocument; constructor({ documentClient }: ConstructorArgs) { - this.batchProcesses = {}; this.documentClient = documentClient; } getClient() { return this.documentClient; } - async create(): Promise { - return [true, {}]; - } - - async update(): Promise { - return [true, {}]; - } - - async delete(): Promise { - return [true, {}]; - } - - async read(): Promise> { - return [[], {}]; - } - - async createLog(): Promise { - return [true, {}]; - } - - async readLogs() { - return this.read(); - } - - getBatchProcess() { - // not empty - } } export default DynamoDbDriver; diff --git a/packages/db-dynamodb/src/types.ts b/packages/db-dynamodb/src/types.ts index 7462e21df3a..83d721d7b99 100644 --- a/packages/db-dynamodb/src/types.ts +++ b/packages/db-dynamodb/src/types.ts @@ -1,27 +1,3 @@ -export interface OperatorArgs { - expression: string; - attributeNames: Record; - attributeValues: Record; -} - -interface CanProcessArgs { - key: string; - value: any; - args: OperatorArgs; -} - -interface ProcessArgs { - key: string; - value: any; - args: OperatorArgs; - processStatement: any; -} - -export interface Operator { - canProcess: ({ key }: CanProcessArgs) => boolean; - process: ({ key, value, args }: ProcessArgs) => void; -} - /** * We use this definition to search for a value in any given field that was passed. * It works as an "OR" condition. diff --git a/packages/db-dynamodb/src/utils/batchRead.ts b/packages/db-dynamodb/src/utils/batchRead.ts index 1324297674c..74a51e5a220 100644 --- a/packages/db-dynamodb/src/utils/batchRead.ts +++ b/packages/db-dynamodb/src/utils/batchRead.ts @@ -1,6 +1,7 @@ import lodashChunk from "lodash/chunk"; import WebinyError from "@webiny/error"; import { TableDef } from "~/toolbox"; +import { GenericRecord } from "@webiny/api/types"; export interface BatchReadItem { Table?: TableDef; @@ -57,7 +58,7 @@ const batchReadAllChunk = async (params: BatchReadAllChunkParams): Prom * This helper function is meant to be used to batch read from one table. * It will fetch all results, as there is a next() method call built in. */ -export const batchReadAll = async ( +export const batchReadAll = async ( params: BatchReadParams, maxChunk = MAX_BATCH_ITEMS ): Promise => { @@ -75,7 +76,7 @@ export const batchReadAll = async ( const records: T[] = []; - const chunkItemsList: BatchReadItem[][] = lodashChunk(params.items, maxChunk); + const chunkItemsList = lodashChunk(params.items, maxChunk); for (const chunkItems of chunkItemsList) { const results = await batchReadAllChunk({ diff --git a/packages/db/package.json b/packages/db/package.json index 21347f1ea1a..54500711e6c 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -12,6 +12,9 @@ "access": "public", "directory": "dist" }, + "dependencies": { + "@webiny/api": "0.0.0" + }, "devDependencies": { "@babel/cli": "^7.23.9", "@babel/core": "^7.24.0", diff --git a/packages/db/src/DbRegistry.ts b/packages/db/src/DbRegistry.ts new file mode 100644 index 00000000000..3bbbf8fc7e1 --- /dev/null +++ b/packages/db/src/DbRegistry.ts @@ -0,0 +1,49 @@ +import { IRegistry, IRegistryItem, IRegistryRegisterParams } from "./types"; +import { GenericRecord } from "@webiny/api/types"; + +export class DbRegistry implements IRegistry { + private readonly items: GenericRecord = {}; + + public register(input: IRegistryRegisterParams): void { + const key = `${input.app}-${input.tags.sort().join("-")}`; + + if (this.items[key]) { + throw new Error( + `Item with app "${input.app}" and tags "${input.tags.join( + ", " + )}" is already registered.` + ); + } + this.items[key] = input; + } + + public getOneItem(cb: (item: IRegistryItem) => boolean): IRegistryItem { + const item = this.getItem(cb); + if (!item) { + throw new Error("Item not found."); + } + return item; + } + + public getItem(cb: (item: IRegistryItem) => boolean): IRegistryItem | null { + const items = this.getItems(cb); + if (items.length === 0) { + return null; + } else if (items.length > 1) { + throw new Error("More than one item found with the provided criteria."); + } + return items[0]; + } + + public getItems(cb: (item: IRegistryItem) => boolean): IRegistryItem[] { + const results: IRegistryItem[] = []; + for (const key in this.items) { + const item = this.items[key] as IRegistryItem; + if (cb(item)) { + results.push(item); + } + } + + return results; + } +} diff --git a/packages/db/src/index.ts b/packages/db/src/index.ts index e06d9da2d2b..01a56af9175 100644 --- a/packages/db/src/index.ts +++ b/packages/db/src/index.ts @@ -1,254 +1,26 @@ -/** - * TODO Remove when moved all packages to standalone storage opts. - */ -interface KeyField { - name: string; -} +import { DbRegistry } from "~/DbRegistry"; -export interface Key { - primary?: boolean; - unique?: boolean; - name: string; - fields: KeyField[]; -} +export * from "./types"; -export interface ArgsBatch { - instance: Batch; - operation: Operation; -} -export interface Args { - __batch?: ArgsBatch; - table?: string; - meta?: boolean; - limit?: number; - sort?: Record; - data?: Record; - query?: Record; - keys?: Key[]; +export interface DbDriver { + getClient(): T; } -export type Result = [T, Record]; - -export interface DbDriver { - create: (args: Args) => Promise>; - read: >(args: Args) => Promise>; - update: (args: Args) => Promise>; - delete: (args: Args) => Promise>; - - // Logging functions. - createLog: (args: { - operation: string; - data: Args; - table: string; - id: string; - }) => Promise>; - readLogs: >(args: { table: string }) => Promise>; -} - -export type OperationType = "create" | "read" | "update" | "delete"; -export type Operation = [OperationType, Args]; - -export type ConstructorArgs = { - driver: DbDriver; +export interface ConstructorArgs { + driver: DbDriver; table?: string; - logTable?: string; -}; - -// Generates a short and sortable ID, e.g. "1607677774994.tfz58m". -const shortId = () => { - const time = new Date().getTime(); - const uniqueId = Math.random().toString(36).slice(-6); - - return `${time}.${uniqueId}`; -}; - -interface LogDataBatch { - id: string; - type: string; -} -interface LogData - extends Pick { - batch: LogDataBatch | null; } -// Picks necessary data from received args, ready to be stored in the log table. -const getCreateLogData = (args: Args): LogData => { - const { table, meta, limit, sort, data, query, keys } = args; +class Db { + public driver: DbDriver; + public readonly table?: string; - return { - table, - meta, - limit, - sort, - data, - query, - keys, - batch: args.__batch - ? { - id: args.__batch.instance.id, - type: args.__batch.instance.type - } - : null - }; -}; + public readonly registry = new DbRegistry(); -class Db { - public driver: DbDriver; - public table: string; - public logTable?: string; - - constructor({ driver, table, logTable }: ConstructorArgs) { - this.driver = driver; - // @ts-expect-error + constructor({ driver, table }: ConstructorArgs) { this.table = table; - this.logTable = logTable; - } - - public async create(args: Args): Promise> { - const createArgs = { ...args, table: args.table || this.table }; - await this.createLog("create", createArgs); - return this.driver.create(createArgs); - } - - public async read>(args: Args): Promise> { - const readArgs = { ...args, table: args.table || this.table }; - await this.createLog("read", readArgs); - return this.driver.read(readArgs); - } - - public async update(args: Args): Promise> { - const updateArgs = { ...args, table: args.table || this.table }; - await this.createLog("update", updateArgs); - return this.driver.update(updateArgs); - } - - public async delete(args: Args): Promise> { - const deleteArgs = { ...args, table: args.table || this.table }; - await this.createLog("delete", deleteArgs); - return this.driver.delete(deleteArgs); - } - - // Logging functions. - public async createLog(operation: string, args: Args): Promise | null> { - if (!this.logTable) { - return null; - } - - const data = getCreateLogData(args); - return this.driver.createLog({ operation, data, table: this.logTable, id: shortId() }); - } - - public async readLogs>(): Promise | null> { - if (!this.logTable) { - return null; - } - - return this.driver.readLogs({ - table: this.logTable - }); - } - - public batch< - T0 = any, - T1 = any, - T2 = any, - T3 = any, - T4 = any, - T5 = any, - T6 = any, - T7 = any, - T8 = any, - T9 = any - >(): Batch { - return new Batch(this); - } -} - -class Batch< - T0 = any, - T1 = any, - T2 = any, - T3 = any, - T4 = any, - T5 = any, - T6 = any, - T7 = any, - T8 = any, - T9 = any -> { - db: Db; - type: "batch" | "transaction"; - id: string; - meta: Record; - operations: Operation[]; - - constructor(db: Db) { - this.db = db; - this.type = "batch"; - this.id = shortId(); - - this.meta = {}; - this.operations = []; - } - - push(...operations: Operation[]) { - for (let i = 0; i < operations.length; i++) { - const item = operations[i]; - this.operations.push(item); - } - return this; - } - - create(...args: Args[]) { - for (let i = 0; i < args.length; i++) { - this.push(["create", args[i]]); - } - return this; - } - - read(...args: Args[]) { - for (let i = 0; i < args.length; i++) { - this.push(["read", args[i]]); - } - return this; - } - - update(...args: Args[]) { - for (let i = 0; i < args.length; i++) { - this.push(["update", args[i]]); - } - return this; - } - - delete(...args: Args[]) { - for (let i = 0; i < args.length; i++) { - this.push(["delete", args[i]]); - } - return this; - } - - async execute(): Promise<[T0?, T1?, T2?, T3?, T4?, T5?, T6?, T7?, T8?, T9?]> { - /** - * TODO: figure out which exact type to use instead of any. - */ - const promises: Promise[] = []; - for (let i = 0; i < this.operations.length; i++) { - const [operation, args] = this.operations[i]; - promises.push( - this.db[operation]({ - ...args, - __batch: { - instance: this, - operation: this.operations[i] - } - }) - ); - } - - const result = Promise.all(promises); - - return result as Promise<[T0?, T1?, T2?, T3?, T4?, T5?, T6?, T7?, T8?, T9?]>; + this.driver = driver; } } -export { Batch, Db }; +export { Db }; diff --git a/packages/db/src/types.ts b/packages/db/src/types.ts new file mode 100644 index 00000000000..eac650c993b --- /dev/null +++ b/packages/db/src/types.ts @@ -0,0 +1,26 @@ +import { NonEmptyArray } from "@webiny/api/types"; + +export interface IRegistryRegisterParams { + item: T; + app: string; + tags: NonEmptyArray; +} + +export interface IRegistryItem { + item: T; + app: string; + tags: NonEmptyArray; +} + +export interface IRegistry { + register(params: IRegistryRegisterParams): void; + /** + * Throws an error if more than one item is found or there is no item found. + */ + getOneItem(cb: (item: IRegistryItem) => boolean): IRegistryItem; + /** + * Throws an error if more than one item is found. + */ + getItem(cb: (item: IRegistryItem) => boolean): IRegistryItem | null; + getItems(cb: (item: IRegistryItem) => boolean): IRegistryItem[]; +} diff --git a/packages/db/tsconfig.build.json b/packages/db/tsconfig.build.json index 5e7843d3c8c..097b4c0b400 100644 --- a/packages/db/tsconfig.build.json +++ b/packages/db/tsconfig.build.json @@ -1,7 +1,7 @@ { "extends": "../../tsconfig.build.json", "include": ["src"], - "references": [], + "references": [{ "path": "../api/tsconfig.build.json" }], "compilerOptions": { "rootDir": "./src", "outDir": "./dist", diff --git a/packages/db/tsconfig.json b/packages/db/tsconfig.json index 6ca26f3c929..306a82a42a7 100644 --- a/packages/db/tsconfig.json +++ b/packages/db/tsconfig.json @@ -1,12 +1,17 @@ { "extends": "../../tsconfig.json", "include": ["src", "__tests__"], - "references": [], + "references": [{ "path": "../api" }], "compilerOptions": { "rootDirs": ["./src", "./__tests__"], "outDir": "./dist", "declarationDir": "./dist", - "paths": { "~/*": ["./src/*"], "~tests/*": ["./__tests__/*"] }, + "paths": { + "~/*": ["./src/*"], + "~tests/*": ["./__tests__/*"], + "@webiny/api/*": ["../api/src/*"], + "@webiny/api": ["../api/src"] + }, "baseUrl": "." } } diff --git a/packages/handler-db/src/index.ts b/packages/handler-db/src/index.ts index af1cbeb82b5..eaafb0aed5f 100644 --- a/packages/handler-db/src/index.ts +++ b/packages/handler-db/src/index.ts @@ -1,17 +1,14 @@ -import { Db } from "@webiny/db"; +import { ConstructorArgs, Db } from "@webiny/db"; import { ContextPlugin } from "@webiny/api"; import { DbContext } from "./types"; -/** - * TODO: remove this package. - */ -export default (args: any) => { +export default (args: ConstructorArgs) => { return [ new ContextPlugin(context => { if (context.db) { return; } - context.db = new Db(args); + context.db = new Db(args); }) ]; }; diff --git a/packages/handler-db/src/types.ts b/packages/handler-db/src/types.ts index 6937b8bd4ca..bfa9b7fb0e6 100644 --- a/packages/handler-db/src/types.ts +++ b/packages/handler-db/src/types.ts @@ -2,5 +2,5 @@ import { Db } from "@webiny/db"; import { Context } from "@webiny/api/types"; export interface DbContext extends Context { - db: Db; + db: Db; } diff --git a/packages/tasks/src/runner/TaskManager.ts b/packages/tasks/src/runner/TaskManager.ts index 0cb6732b7ed..e9ee12c642b 100644 --- a/packages/tasks/src/runner/TaskManager.ts +++ b/packages/tasks/src/runner/TaskManager.ts @@ -16,15 +16,17 @@ import { } from "~/response/abstractions"; import { getErrorProperties } from "~/utils/getErrorProperties"; +type ITaskManagerRunner = Pick; + export class TaskManager implements ITaskManager { - private readonly runner: Pick; + private readonly runner: ITaskManagerRunner; private readonly context: Context; private readonly response: IResponse; private readonly taskResponse: ITaskResponse; private readonly store: ITaskManagerStorePrivate; public constructor( - runner: Pick, + runner: ITaskManagerRunner, context: Context, response: IResponse, taskResponse: ITaskResponse, @@ -134,7 +136,8 @@ export class TaskManager implements ITaskManager { ...params, parent: this.store.getTask() }); - } + }, + timer: this.runner.timer }); }); } catch (ex) { diff --git a/packages/tasks/src/runner/TaskRunner.ts b/packages/tasks/src/runner/TaskRunner.ts index 13db64c6741..f758aecf108 100644 --- a/packages/tasks/src/runner/TaskRunner.ts +++ b/packages/tasks/src/runner/TaskRunner.ts @@ -23,8 +23,8 @@ export class TaskRunner implements ITaskRunner { * Follow the same example for the rest of the properties. */ public readonly context: C; + public readonly timer: ITimer; private readonly validation: ITaskEventValidation; - private readonly timer: ITimer; /** * We take all required variables separately because they will get injected via DI - so less refactoring is required in the future. diff --git a/packages/tasks/src/runner/abstractions/TaskRunner.ts b/packages/tasks/src/runner/abstractions/TaskRunner.ts index da4f8bca9fb..3d399826041 100644 --- a/packages/tasks/src/runner/abstractions/TaskRunner.ts +++ b/packages/tasks/src/runner/abstractions/TaskRunner.ts @@ -1,6 +1,7 @@ import { Context } from "~/types"; import { ITaskEvent } from "~/handler/types"; import { IResponseResult } from "~/response/abstractions"; +import { ITimer } from "@webiny/handler-aws"; export interface IIsCloseToTimeoutCallable { (seconds?: number): boolean; @@ -9,5 +10,6 @@ export interface IIsCloseToTimeoutCallable { export interface ITaskRunner { context: C; isCloseToTimeout: IIsCloseToTimeoutCallable; + timer: ITimer; run(event: ITaskEvent): Promise; } diff --git a/packages/tasks/src/types.ts b/packages/tasks/src/types.ts index e3619ca6d8d..b7447934182 100644 --- a/packages/tasks/src/types.ts +++ b/packages/tasks/src/types.ts @@ -17,6 +17,7 @@ import { IIsCloseToTimeoutCallable, ITaskManagerStore } from "./runner/abstracti import { SecurityPermission } from "@webiny/api-security/types"; import { GenericRecord } from "@webiny/api/types"; import { IStepFunctionServiceFetchResult } from "~/service/StepFunctionServicePlugin"; +import { ITimer } from "@webiny/handler-aws"; import type zod from "zod"; @@ -334,6 +335,7 @@ export interface ITaskRunParams< trigger( params: Omit, "parent"> ): Promise>; + timer: ITimer; } export interface ITaskOnSuccessParams< diff --git a/yarn.lock b/yarn.lock index 2b89b21d867..cfbe168e186 100644 --- a/yarn.lock +++ b/yarn.lock @@ -13857,6 +13857,7 @@ __metadata: "@babel/preset-typescript": ^7.23.3 "@babel/runtime": ^7.24.0 "@webiny/api": 0.0.0 + "@webiny/api-dynamodb-to-elasticsearch": 0.0.0 "@webiny/api-elasticsearch": 0.0.0 "@webiny/api-headless-cms": 0.0.0 "@webiny/api-i18n": 0.0.0 @@ -13865,6 +13866,7 @@ __metadata: "@webiny/api-wcp": 0.0.0 "@webiny/aws-sdk": 0.0.0 "@webiny/cli": 0.0.0 + "@webiny/db": 0.0.0 "@webiny/db-dynamodb": 0.0.0 "@webiny/error": 0.0.0 "@webiny/handler": 0.0.0 @@ -17320,6 +17322,7 @@ __metadata: dependencies: "@babel/cli": ^7.23.9 "@babel/core": ^7.24.0 + "@webiny/api": 0.0.0 "@webiny/cli": 0.0.0 "@webiny/project-utils": 0.0.0 rimraf: ^5.0.5 From 6ba635c5ead342c54ab73a45ceb1ac377c11fc5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20Zori=C4=87?= Date: Mon, 11 Nov 2024 13:40:50 +0100 Subject: [PATCH 03/16] fix(app-headless-cms-common): push id in the list if no other field exists (#4370) --- .../src/createFieldsList.ts | 17 +++++------ .../src/entries.graphql.ts | 30 ------------------- 2 files changed, 7 insertions(+), 40 deletions(-) diff --git a/packages/app-headless-cms-common/src/createFieldsList.ts b/packages/app-headless-cms-common/src/createFieldsList.ts index 87a91ca75dd..f7228d2385d 100644 --- a/packages/app-headless-cms-common/src/createFieldsList.ts +++ b/packages/app-headless-cms-common/src/createFieldsList.ts @@ -1,4 +1,4 @@ -import { CmsModelField, CmsModelFieldTypePlugin, CmsModel } from "~/types"; +import { CmsModel, CmsModelField, CmsModelFieldTypePlugin } from "~/types"; import { plugins } from "@webiny/plugins"; interface CreateFieldsListParams { @@ -9,7 +9,7 @@ interface CreateFieldsListParams { export function createFieldsList({ model, - fields, + fields: inputFields, graphQLTypePrefix }: CreateFieldsListParams): string { const fieldPlugins: Record = plugins @@ -18,7 +18,7 @@ export function createFieldsList({ const typePrefix = graphQLTypePrefix ?? model.singularApiName; - const allFields = fields + const fields = inputFields .map(field => { if (!fieldPlugins[field.type]) { console.log(`Unknown field plugin for field type "${field.type}".`); @@ -46,14 +46,11 @@ export function createFieldsList({ return field.fieldId; }) .filter(Boolean); - /** - * If there are no fields for a given type, we add a dummy `_empty` field, which will also be present in the schema - * on the API side, to protect the schema from invalid types. + * If there are no fields, let's always load the `id` field. */ - if (!allFields.length) { - allFields.push("_empty"); + if (fields.length === 0) { + fields.push("id"); } - - return allFields.join("\n"); + return fields.join("\n"); } diff --git a/packages/app-headless-cms-common/src/entries.graphql.ts b/packages/app-headless-cms-common/src/entries.graphql.ts index 8512cb63058..6e5be56d9b7 100644 --- a/packages/app-headless-cms-common/src/entries.graphql.ts +++ b/packages/app-headless-cms-common/src/entries.graphql.ts @@ -112,36 +112,6 @@ const createEntrySystemFields = (model: CmsModel) => { type displayName } - revisionCreatedOn - revisionSavedOn - revisionModifiedOn - revisionFirstPublishedOn - revisionLastPublishedOn - revisionCreatedBy { - id - type - displayName - } - revisionSavedBy { - id - type - displayName - } - revisionModifiedBy { - id - type - displayName - } - revisionFirstPublishedBy { - id - type - displayName - } - revisionLastPublishedBy { - id - type - displayName - } ${optionalFields} `; }; From 0abd03e08b887957a38ab9b32f5e7c54869b0f25 Mon Sep 17 00:00:00 2001 From: Adrian Smijulj Date: Mon, 11 Nov 2024 14:15:22 +0100 Subject: [PATCH 04/16] fix: remove unneeded `listGroupSlugs` and `listTeamSlugs` methods (#4375) --- .../listPermissionsFromGroupsAndTeams.ts | 22 +------------------ 1 file changed, 1 insertion(+), 21 deletions(-) diff --git a/packages/api-security/src/utils/createGroupsTeamsAuthorizer/listPermissionsFromGroupsAndTeams.ts b/packages/api-security/src/utils/createGroupsTeamsAuthorizer/listPermissionsFromGroupsAndTeams.ts index 4cc5ba32003..6859668ffa3 100644 --- a/packages/api-security/src/utils/createGroupsTeamsAuthorizer/listPermissionsFromGroupsAndTeams.ts +++ b/packages/api-security/src/utils/createGroupsTeamsAuthorizer/listPermissionsFromGroupsAndTeams.ts @@ -12,21 +12,11 @@ export interface GroupsTeamsAuthorizerConfig Promise | GroupSlug; - /** - * List group slugs to load permissions from. - */ - listGroupSlugs?: (context: TContext) => Promise | GroupSlug[]; - - /** - * List team slugs to load groups and ultimately permissions from. - */ - listTeamSlugs?: (context: TContext) => Promise | TeamSlug[]; - /** * If a security group is not found, try loading it from a parent tenant (default: true). */ @@ -65,11 +55,6 @@ export const listPermissionsFromGroupsAndTeams = async < groupSlugs.push(loadedGroupSlug); } - if (config.listGroupSlugs) { - const loadedGroupSlugs = await config.listGroupSlugs(context); - groupSlugs.push(...loadedGroupSlugs); - } - if (identity.group) { groupSlugs.push(identity.group); } @@ -80,11 +65,6 @@ export const listPermissionsFromGroupsAndTeams = async < if (wcp.canUseTeams()) { // Load groups coming from teams. - if (config.listTeamSlugs) { - const loadedTeamSlugs = await config.listTeamSlugs(context); - teamSlugs.push(...loadedTeamSlugs); - } - if (identity.team) { teamSlugs.push(identity.team); } From 2844595e920989c4d4361307ca9e27db618b4f21 Mon Sep 17 00:00:00 2001 From: Adrian Smijulj Date: Mon, 11 Nov 2024 16:26:26 +0100 Subject: [PATCH 05/16] fix: use correct keys upon applying border radius (#4369) --- .../src/modifiers/styles/border.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/app-page-builder-elements/src/modifiers/styles/border.ts b/packages/app-page-builder-elements/src/modifiers/styles/border.ts index 3f731eb3ead..a02ab2bdb26 100644 --- a/packages/app-page-builder-elements/src/modifiers/styles/border.ts +++ b/packages/app-page-builder-elements/src/modifiers/styles/border.ts @@ -45,10 +45,10 @@ const border: ElementStylesModifier = ({ element, theme }) => { if (radius) { if (radius.advanced) { Object.assign(styles, { - borderRadiusTop: radius.top && parseInt(radius.top), - borderRadiusRight: radius.right && parseInt(radius.right), - borderRadiusBottom: radius.bottom && parseInt(radius.bottom), - borderRadiusLeft: radius.left && parseInt(radius.left) + borderTopLeftRadius: radius.topLeft && parseInt(radius.topLeft), + borderTopRightRadius: radius.topRight && parseInt(radius.topRight), + borderBottomLeftRadius: radius.bottomLeft && parseInt(radius.bottomLeft), + borderBottomRightRadius: radius.bottomRight && parseInt(radius.bottomRight) }); } else { Object.assign(styles, { borderRadius: parseInt(radius.all || "0") }); From ad8f4a2ff0c7a5b6db3bf81867c7a3d152180ff1 Mon Sep 17 00:00:00 2001 From: adrians5j Date: Tue, 12 Nov 2024 06:34:04 +0100 Subject: [PATCH 06/16] ci: auto-assign milestone [no ci] --- .github/workflows/pullRequests.yml | 20 ++++++++++++++++++++ .github/workflows/wac/pullRequests.wac.ts | 21 ++++++++++++++++++++- 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pullRequests.yml b/.github/workflows/pullRequests.yml index ed8aadad2c4..ebbad219880 100644 --- a/.github/workflows/pullRequests.yml +++ b/.github/workflows/pullRequests.yml @@ -43,6 +43,7 @@ jobs: run-cache-key: ${{ steps.run-cache-key.outputs.run-cache-key }} is-fork-pr: ${{ steps.is-fork-pr.outputs.is-fork-pr }} changed-packages: ${{ steps.detect-changed-packages.outputs.changed-packages }} + latest-webiny-version: ${{ steps.latest-webiny-version.outputs.latest-webiny-version }} steps: - uses: actions/setup-node@v4 with: @@ -79,6 +80,25 @@ jobs: .github/workflows/wac/utils/runNodeScripts/listChangedPackages.js '${{ steps.detect-changed-files.outputs.changed_files }}')" >> $GITHUB_OUTPUT + - name: Get latest Webiny version on NPM + id: latest-webiny-version + run: >- + echo "latest-webiny-version=npm view @webiny/cli version" >> + $GITHUB_OUTPUT + runs-on: ubuntu-latest + env: + NODE_OPTIONS: '--max_old_space_size=4096' + YARN_ENABLE_IMMUTABLE_INSTALLS: false + assignMilestone: + name: Assign milestone + needs: constants + steps: + - uses: actions/setup-node@v4 + with: + node-version: 20 + - uses: actions/checkout@v4 + - name: Print latest Webiny version + run: echo ${{ needs.constants.outputs.latest-webiny-version }} runs-on: ubuntu-latest env: NODE_OPTIONS: '--max_old_space_size=4096' diff --git a/.github/workflows/wac/pullRequests.wac.ts b/.github/workflows/wac/pullRequests.wac.ts index a1a6562f73e..fa2e4035fda 100644 --- a/.github/workflows/wac/pullRequests.wac.ts +++ b/.github/workflows/wac/pullRequests.wac.ts @@ -153,7 +153,8 @@ export const pullRequests = createWorkflow({ "global-cache-key": "${{ steps.global-cache-key.outputs.global-cache-key }}", "run-cache-key": "${{ steps.run-cache-key.outputs.run-cache-key }}", "is-fork-pr": "${{ steps.is-fork-pr.outputs.is-fork-pr }}", - "changed-packages": "${{ steps.detect-changed-packages.outputs.changed-packages }}" + "changed-packages": "${{ steps.detect-changed-packages.outputs.changed-packages }}", + "latest-webiny-version": "${{ steps.latest-webiny-version.outputs.latest-webiny-version }}", }, steps: [ { @@ -197,6 +198,24 @@ export const pullRequests = createWorkflow({ "${{ steps.detect-changed-files.outputs.changed_files }}", { outputAs: "changed-packages" } ) + }, + { + name: "Get latest Webiny version on NPM", + id: "latest-webiny-version", + run: addToOutputs( + "latest-webiny-version", + "npm view @webiny/cli version" + ), + } + ] + }), + assignMilestone: createJob({ + name: "Assign milestone", + needs: "constants", + steps: [ + { + name: "Print latest Webiny version", + run: "echo ${{ needs.constants.outputs.latest-webiny-version }}" } ] }), From 3b36f268a836cb6ad38757d2173def1625d10b7b Mon Sep 17 00:00:00 2001 From: adrians5j Date: Tue, 12 Nov 2024 09:07:30 +0100 Subject: [PATCH 07/16] ci: auto-assign milestone [no ci] --- .github/workflows/pullRequests.yml | 2 +- .github/workflows/wac/pullRequests.wac.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pullRequests.yml b/.github/workflows/pullRequests.yml index ebbad219880..69cf214a418 100644 --- a/.github/workflows/pullRequests.yml +++ b/.github/workflows/pullRequests.yml @@ -83,7 +83,7 @@ jobs: - name: Get latest Webiny version on NPM id: latest-webiny-version run: >- - echo "latest-webiny-version=npm view @webiny/cli version" >> + echo "latest-webiny-version=$(npm view @webiny/cli version)" >> $GITHUB_OUTPUT runs-on: ubuntu-latest env: diff --git a/.github/workflows/wac/pullRequests.wac.ts b/.github/workflows/wac/pullRequests.wac.ts index fa2e4035fda..dbaea3f088e 100644 --- a/.github/workflows/wac/pullRequests.wac.ts +++ b/.github/workflows/wac/pullRequests.wac.ts @@ -204,7 +204,7 @@ export const pullRequests = createWorkflow({ id: "latest-webiny-version", run: addToOutputs( "latest-webiny-version", - "npm view @webiny/cli version" + "$(npm view @webiny/cli version)" ), } ] From 050cdcfea879dfd2519b24d82ab14fd4224dbac4 Mon Sep 17 00:00:00 2001 From: adrians5j Date: Tue, 12 Nov 2024 09:57:03 +0100 Subject: [PATCH 08/16] chore: format code [no ci] --- .github/workflows/wac/pullRequests.wac.ts | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/wac/pullRequests.wac.ts b/.github/workflows/wac/pullRequests.wac.ts index dbaea3f088e..f8a736048df 100644 --- a/.github/workflows/wac/pullRequests.wac.ts +++ b/.github/workflows/wac/pullRequests.wac.ts @@ -154,7 +154,8 @@ export const pullRequests = createWorkflow({ "run-cache-key": "${{ steps.run-cache-key.outputs.run-cache-key }}", "is-fork-pr": "${{ steps.is-fork-pr.outputs.is-fork-pr }}", "changed-packages": "${{ steps.detect-changed-packages.outputs.changed-packages }}", - "latest-webiny-version": "${{ steps.latest-webiny-version.outputs.latest-webiny-version }}", + "latest-webiny-version": + "${{ steps.latest-webiny-version.outputs.latest-webiny-version }}" }, steps: [ { @@ -202,10 +203,7 @@ export const pullRequests = createWorkflow({ { name: "Get latest Webiny version on NPM", id: "latest-webiny-version", - run: addToOutputs( - "latest-webiny-version", - "$(npm view @webiny/cli version)" - ), + run: addToOutputs("latest-webiny-version", "$(npm view @webiny/cli version)") } ] }), From a9b97de5c19183f091b17e98075b02b2bba53564 Mon Sep 17 00:00:00 2001 From: adrians5j Date: Wed, 13 Nov 2024 09:38:39 +0100 Subject: [PATCH 09/16] ci: run `api-elasticsearch` tests with correct `--storage` params --- .github/workflows/pullRequests.yml | 6 ++++-- .github/workflows/pushDev.yml | 6 ++++-- .github/workflows/pushNext.yml | 6 ++++-- .github/workflows/wac/utils/listPackagesWithJestTests.ts | 8 ++++++-- 4 files changed, 18 insertions(+), 8 deletions(-) diff --git a/.github/workflows/pullRequests.yml b/.github/workflows/pullRequests.yml index 69cf214a418..daa191d2257 100644 --- a/.github/workflows/pullRequests.yml +++ b/.github/workflows/pullRequests.yml @@ -413,7 +413,8 @@ jobs: '[[{"cmd":"packages/api-aco --storage=ddb-es,ddb","storage":"ddb-es","packageName":"api-aco","id":"8f23ec33f547aa62236f5c71115688d6"},{"cmd":"packages/api-audit-logs --storage=ddb-es,ddb","storage":"ddb-es","packageName":"api-audit-logs","id":"a292444cd9100f78d8fc196274393ea8"},{"cmd":"packages/api-dynamodb-to-elasticsearch - --storage=ddb-es,ddb","storage":["ddb-es"],"packageName":"api-dynamodb-to-elasticsearch","id":"e2c325f0940ba5fb5a891a8cf74fca61"},{"cmd":"packages/api-elasticsearch","storage":["ddb-es","ddb-os"],"packageName":"api-elasticsearch","id":"430874606aeb8e8041b325955f9330e3"},{"cmd":"packages/api-elasticsearch-tasks + --storage=ddb-es,ddb","storage":["ddb-es"],"packageName":"api-dynamodb-to-elasticsearch","id":"e2c325f0940ba5fb5a891a8cf74fca61"},{"cmd":"packages/api-elasticsearch + --storage=ddb-es,ddb","storage":["ddb-es"],"packageName":"api-elasticsearch","id":"5963079c60b96202bbaf2a802ad14383"},{"cmd":"packages/api-elasticsearch-tasks --storage=ddb-es,ddb","storage":"ddb-es","packageName":"api-elasticsearch-tasks","id":"d81ad1d024a8746cc440e2e548770f8f"},{"cmd":"packages/api-file-manager --storage=ddb-es,ddb","storage":"ddb-es","packageName":"api-file-manager","id":"d6f293add4a252b96cbd770ab6e80557"},{"cmd":"packages/api-form-builder --storage=ddb-es,ddb","storage":"ddb-es","packageName":"api-form-builder","id":"3753bde0144d808eb15c755b7176386c"},{"cmd":"packages/api-form-builder-so-ddb-es @@ -541,7 +542,8 @@ jobs: '[[{"cmd":"packages/api-aco --storage=ddb-os,ddb","storage":"ddb-os","packageName":"api-aco","id":"e4b1b5ebc172f2657485e41c35ad1cd7"},{"cmd":"packages/api-audit-logs --storage=ddb-os,ddb","storage":"ddb-os","packageName":"api-audit-logs","id":"b36aac5f0e34dc4583e5422ae589f1ed"},{"cmd":"packages/api-dynamodb-to-elasticsearch - --storage=ddb-os,ddb","storage":["ddb-os"],"packageName":"api-dynamodb-to-elasticsearch","id":"6e0b282c3d135703e52b2c55822d4fb0"},{"cmd":"packages/api-elasticsearch","storage":["ddb-es","ddb-os"],"packageName":"api-elasticsearch","id":"430874606aeb8e8041b325955f9330e3"},{"cmd":"packages/api-elasticsearch-tasks + --storage=ddb-os,ddb","storage":["ddb-os"],"packageName":"api-dynamodb-to-elasticsearch","id":"6e0b282c3d135703e52b2c55822d4fb0"},{"cmd":"packages/api-elasticsearch + --storage=ddb-os,ddb","storage":["ddb-os"],"packageName":"api-elasticsearch","id":"b0f477d6b209f654714809b318be888e"},{"cmd":"packages/api-elasticsearch-tasks --storage=ddb-os,ddb","storage":"ddb-os","packageName":"api-elasticsearch-tasks","id":"580a9577fdbd4a241034a42e1a47dee5"},{"cmd":"packages/api-file-manager --storage=ddb-os,ddb","storage":"ddb-os","packageName":"api-file-manager","id":"346430a79981d3e214c87254a08e31b2"},{"cmd":"packages/api-form-builder --storage=ddb-os,ddb","storage":"ddb-os","packageName":"api-form-builder","id":"d386cddfd3c366ad9955193dcfe74363"},{"cmd":"packages/api-form-builder-so-ddb-es diff --git a/.github/workflows/pushDev.yml b/.github/workflows/pushDev.yml index 4d462a8a163..d640af619c5 100644 --- a/.github/workflows/pushDev.yml +++ b/.github/workflows/pushDev.yml @@ -277,7 +277,8 @@ jobs: ${{ fromJson('[{"cmd":"packages/api-aco --storage=ddb-es,ddb","storage":"ddb-es","packageName":"api-aco","id":"8f23ec33f547aa62236f5c71115688d6"},{"cmd":"packages/api-audit-logs --storage=ddb-es,ddb","storage":"ddb-es","packageName":"api-audit-logs","id":"a292444cd9100f78d8fc196274393ea8"},{"cmd":"packages/api-dynamodb-to-elasticsearch - --storage=ddb-es,ddb","storage":["ddb-es"],"packageName":"api-dynamodb-to-elasticsearch","id":"e2c325f0940ba5fb5a891a8cf74fca61"},{"cmd":"packages/api-elasticsearch","storage":["ddb-es","ddb-os"],"packageName":"api-elasticsearch","id":"430874606aeb8e8041b325955f9330e3"},{"cmd":"packages/api-elasticsearch-tasks + --storage=ddb-es,ddb","storage":["ddb-es"],"packageName":"api-dynamodb-to-elasticsearch","id":"e2c325f0940ba5fb5a891a8cf74fca61"},{"cmd":"packages/api-elasticsearch + --storage=ddb-es,ddb","storage":["ddb-es"],"packageName":"api-elasticsearch","id":"5963079c60b96202bbaf2a802ad14383"},{"cmd":"packages/api-elasticsearch-tasks --storage=ddb-es,ddb","storage":"ddb-es","packageName":"api-elasticsearch-tasks","id":"d81ad1d024a8746cc440e2e548770f8f"},{"cmd":"packages/api-file-manager --storage=ddb-es,ddb","storage":"ddb-es","packageName":"api-file-manager","id":"d6f293add4a252b96cbd770ab6e80557"},{"cmd":"packages/api-form-builder --storage=ddb-es,ddb","storage":"ddb-es","packageName":"api-form-builder","id":"3753bde0144d808eb15c755b7176386c"},{"cmd":"packages/api-form-builder-so-ddb-es @@ -372,7 +373,8 @@ jobs: ${{ fromJson('[{"cmd":"packages/api-aco --storage=ddb-os,ddb","storage":"ddb-os","packageName":"api-aco","id":"e4b1b5ebc172f2657485e41c35ad1cd7"},{"cmd":"packages/api-audit-logs --storage=ddb-os,ddb","storage":"ddb-os","packageName":"api-audit-logs","id":"b36aac5f0e34dc4583e5422ae589f1ed"},{"cmd":"packages/api-dynamodb-to-elasticsearch - --storage=ddb-os,ddb","storage":["ddb-os"],"packageName":"api-dynamodb-to-elasticsearch","id":"6e0b282c3d135703e52b2c55822d4fb0"},{"cmd":"packages/api-elasticsearch","storage":["ddb-es","ddb-os"],"packageName":"api-elasticsearch","id":"430874606aeb8e8041b325955f9330e3"},{"cmd":"packages/api-elasticsearch-tasks + --storage=ddb-os,ddb","storage":["ddb-os"],"packageName":"api-dynamodb-to-elasticsearch","id":"6e0b282c3d135703e52b2c55822d4fb0"},{"cmd":"packages/api-elasticsearch + --storage=ddb-os,ddb","storage":["ddb-os"],"packageName":"api-elasticsearch","id":"b0f477d6b209f654714809b318be888e"},{"cmd":"packages/api-elasticsearch-tasks --storage=ddb-os,ddb","storage":"ddb-os","packageName":"api-elasticsearch-tasks","id":"580a9577fdbd4a241034a42e1a47dee5"},{"cmd":"packages/api-file-manager --storage=ddb-os,ddb","storage":"ddb-os","packageName":"api-file-manager","id":"346430a79981d3e214c87254a08e31b2"},{"cmd":"packages/api-form-builder --storage=ddb-os,ddb","storage":"ddb-os","packageName":"api-form-builder","id":"d386cddfd3c366ad9955193dcfe74363"},{"cmd":"packages/api-form-builder-so-ddb-es diff --git a/.github/workflows/pushNext.yml b/.github/workflows/pushNext.yml index 24d2227030a..0c050c5bcb5 100644 --- a/.github/workflows/pushNext.yml +++ b/.github/workflows/pushNext.yml @@ -277,7 +277,8 @@ jobs: ${{ fromJson('[{"cmd":"packages/api-aco --storage=ddb-es,ddb","storage":"ddb-es","packageName":"api-aco","id":"8f23ec33f547aa62236f5c71115688d6"},{"cmd":"packages/api-audit-logs --storage=ddb-es,ddb","storage":"ddb-es","packageName":"api-audit-logs","id":"a292444cd9100f78d8fc196274393ea8"},{"cmd":"packages/api-dynamodb-to-elasticsearch - --storage=ddb-es,ddb","storage":["ddb-es"],"packageName":"api-dynamodb-to-elasticsearch","id":"e2c325f0940ba5fb5a891a8cf74fca61"},{"cmd":"packages/api-elasticsearch","storage":["ddb-es","ddb-os"],"packageName":"api-elasticsearch","id":"430874606aeb8e8041b325955f9330e3"},{"cmd":"packages/api-elasticsearch-tasks + --storage=ddb-es,ddb","storage":["ddb-es"],"packageName":"api-dynamodb-to-elasticsearch","id":"e2c325f0940ba5fb5a891a8cf74fca61"},{"cmd":"packages/api-elasticsearch + --storage=ddb-es,ddb","storage":["ddb-es"],"packageName":"api-elasticsearch","id":"5963079c60b96202bbaf2a802ad14383"},{"cmd":"packages/api-elasticsearch-tasks --storage=ddb-es,ddb","storage":"ddb-es","packageName":"api-elasticsearch-tasks","id":"d81ad1d024a8746cc440e2e548770f8f"},{"cmd":"packages/api-file-manager --storage=ddb-es,ddb","storage":"ddb-es","packageName":"api-file-manager","id":"d6f293add4a252b96cbd770ab6e80557"},{"cmd":"packages/api-form-builder --storage=ddb-es,ddb","storage":"ddb-es","packageName":"api-form-builder","id":"3753bde0144d808eb15c755b7176386c"},{"cmd":"packages/api-form-builder-so-ddb-es @@ -372,7 +373,8 @@ jobs: ${{ fromJson('[{"cmd":"packages/api-aco --storage=ddb-os,ddb","storage":"ddb-os","packageName":"api-aco","id":"e4b1b5ebc172f2657485e41c35ad1cd7"},{"cmd":"packages/api-audit-logs --storage=ddb-os,ddb","storage":"ddb-os","packageName":"api-audit-logs","id":"b36aac5f0e34dc4583e5422ae589f1ed"},{"cmd":"packages/api-dynamodb-to-elasticsearch - --storage=ddb-os,ddb","storage":["ddb-os"],"packageName":"api-dynamodb-to-elasticsearch","id":"6e0b282c3d135703e52b2c55822d4fb0"},{"cmd":"packages/api-elasticsearch","storage":["ddb-es","ddb-os"],"packageName":"api-elasticsearch","id":"430874606aeb8e8041b325955f9330e3"},{"cmd":"packages/api-elasticsearch-tasks + --storage=ddb-os,ddb","storage":["ddb-os"],"packageName":"api-dynamodb-to-elasticsearch","id":"6e0b282c3d135703e52b2c55822d4fb0"},{"cmd":"packages/api-elasticsearch + --storage=ddb-os,ddb","storage":["ddb-os"],"packageName":"api-elasticsearch","id":"b0f477d6b209f654714809b318be888e"},{"cmd":"packages/api-elasticsearch-tasks --storage=ddb-os,ddb","storage":"ddb-os","packageName":"api-elasticsearch-tasks","id":"580a9577fdbd4a241034a42e1a47dee5"},{"cmd":"packages/api-file-manager --storage=ddb-os,ddb","storage":"ddb-os","packageName":"api-file-manager","id":"346430a79981d3e214c87254a08e31b2"},{"cmd":"packages/api-form-builder --storage=ddb-os,ddb","storage":"ddb-os","packageName":"api-form-builder","id":"d386cddfd3c366ad9955193dcfe74363"},{"cmd":"packages/api-form-builder-so-ddb-es diff --git a/.github/workflows/wac/utils/listPackagesWithJestTests.ts b/.github/workflows/wac/utils/listPackagesWithJestTests.ts index 7b4fb256a56..36715bac50a 100644 --- a/.github/workflows/wac/utils/listPackagesWithJestTests.ts +++ b/.github/workflows/wac/utils/listPackagesWithJestTests.ts @@ -262,8 +262,12 @@ const CUSTOM_HANDLERS: Record Array> = { "api-elasticsearch": () => { return [ { - cmd: "packages/api-elasticsearch", - storage: ["ddb-es", "ddb-os"] + cmd: "packages/api-elasticsearch --storage=ddb-es,ddb", + storage: ["ddb-es"] + }, + { + cmd: "packages/api-elasticsearch --storage=ddb-os,ddb", + storage: ["ddb-os"] } ]; }, From d5ae34aba204a70bafe8e544413d0868eec5c75d Mon Sep 17 00:00:00 2001 From: adrians5j Date: Wed, 13 Nov 2024 09:46:23 +0100 Subject: [PATCH 10/16] ci: `staticCodeAnalysis` does not depend on `build` --- .github/workflows/pullRequests.yml | 1 - .github/workflows/wac/pullRequests.wac.ts | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/pullRequests.yml b/.github/workflows/pullRequests.yml index daa191d2257..347ed0389ce 100644 --- a/.github/workflows/pullRequests.yml +++ b/.github/workflows/pullRequests.yml @@ -138,7 +138,6 @@ jobs: staticCodeAnalysis: needs: - constants - - build name: Static code analysis steps: - uses: actions/setup-node@v4 diff --git a/.github/workflows/wac/pullRequests.wac.ts b/.github/workflows/wac/pullRequests.wac.ts index f8a736048df..af54ea95020 100644 --- a/.github/workflows/wac/pullRequests.wac.ts +++ b/.github/workflows/wac/pullRequests.wac.ts @@ -233,7 +233,7 @@ export const pullRequests = createWorkflow({ ] }), staticCodeAnalysis: createJob({ - needs: ["constants", "build"], + needs: ["constants"], name: "Static code analysis", checkout: { path: DIR_WEBINY_JS }, steps: [ From 9e3c010f1e346a6618b502a0515aca848e0f2290 Mon Sep 17 00:00:00 2001 From: adrians5j Date: Wed, 13 Nov 2024 12:11:42 +0100 Subject: [PATCH 11/16] ci: add single parentheses when echoing list of packages --- .github/workflows/pullRequests.yml | 16 ++++++++-------- .github/workflows/wac/pullRequests.wac.ts | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/pullRequests.yml b/.github/workflows/pullRequests.yml index 347ed0389ce..85911cf8036 100644 --- a/.github/workflows/pullRequests.yml +++ b/.github/workflows/pullRequests.yml @@ -222,8 +222,8 @@ jobs: - name: Packages to test with Jest id: list-packages run: >- - echo ${{ - steps.list-packages-to-jest-test.outputs.packages-to-jest-test }} + echo '${{ + steps.list-packages-to-jest-test.outputs.packages-to-jest-test }}' env: NODE_OPTIONS: '--max_old_space_size=4096' YARN_ENABLE_IMMUTABLE_INSTALLS: false @@ -340,8 +340,8 @@ jobs: - name: Packages to test with Jest id: list-packages run: >- - echo ${{ - steps.list-packages-to-jest-test.outputs.packages-to-jest-test }} + echo '${{ + steps.list-packages-to-jest-test.outputs.packages-to-jest-test }}' env: NODE_OPTIONS: '--max_old_space_size=4096' YARN_ENABLE_IMMUTABLE_INSTALLS: false @@ -456,8 +456,8 @@ jobs: - name: Packages to test with Jest id: list-packages run: >- - echo ${{ - steps.list-packages-to-jest-test.outputs.packages-to-jest-test }} + echo '${{ + steps.list-packages-to-jest-test.outputs.packages-to-jest-test }}' env: NODE_OPTIONS: '--max_old_space_size=4096' YARN_ENABLE_IMMUTABLE_INSTALLS: false @@ -585,8 +585,8 @@ jobs: - name: Packages to test with Jest id: list-packages run: >- - echo ${{ - steps.list-packages-to-jest-test.outputs.packages-to-jest-test }} + echo '${{ + steps.list-packages-to-jest-test.outputs.packages-to-jest-test }}' env: NODE_OPTIONS: '--max_old_space_size=4096' YARN_ENABLE_IMMUTABLE_INSTALLS: false diff --git a/.github/workflows/wac/pullRequests.wac.ts b/.github/workflows/wac/pullRequests.wac.ts index af54ea95020..60f304b84e4 100644 --- a/.github/workflows/wac/pullRequests.wac.ts +++ b/.github/workflows/wac/pullRequests.wac.ts @@ -73,7 +73,7 @@ const createJestTestsJobs = (storage: string | null) => { { name: "Packages to test with Jest", id: "list-packages", - run: "echo ${{ steps.list-packages-to-jest-test.outputs.packages-to-jest-test }}" + run: "echo '${{ steps.list-packages-to-jest-test.outputs.packages-to-jest-test }}'" } ] }); From 3c11ac98496a70a99eb161bf21a9b247e48a4f03 Mon Sep 17 00:00:00 2001 From: adrians5j Date: Fri, 15 Nov 2024 13:19:57 +0100 Subject: [PATCH 12/16] ci: auto-assign ms [no ci] --- .github/workflows/pullRequests.yml | 14 ++++++++++++ .github/workflows/wac/pullRequests.wac.ts | 22 +++++++++++++++++++ .../runNodeScripts/getMilestoneToAssign.js | 17 ++++++++++++++ 3 files changed, 53 insertions(+) create mode 100644 .github/workflows/wac/utils/runNodeScripts/getMilestoneToAssign.js diff --git a/.github/workflows/pullRequests.yml b/.github/workflows/pullRequests.yml index 85911cf8036..4df8b5fb9b0 100644 --- a/.github/workflows/pullRequests.yml +++ b/.github/workflows/pullRequests.yml @@ -92,6 +92,7 @@ jobs: assignMilestone: name: Assign milestone needs: constants + if: needs.constants.outputs.is-fork-pr != 'true' steps: - uses: actions/setup-node@v4 with: @@ -99,6 +100,19 @@ jobs: - uses: actions/checkout@v4 - name: Print latest Webiny version run: echo ${{ needs.constants.outputs.latest-webiny-version }} + - id: get-milestone-to-assign + name: Get milestone to assign + run: >- + echo "milestone=$(node + .github/workflows/wac/utils/runNodeScripts/getMilestoneToAssign.js + '{"latestWebinyVersion":"\"${{ + needs.constants.outputs.latest-webiny-version }}\"","baseBranch":"${{ + github.base_ref }}"}')" >> $GITHUB_OUTPUT + - uses: zoispag/action-assign-milestone@v1 + if: steps.get-milestone-to-assign.outputs.milestone + with: + repo-token: ${{ secrets.GH_TOKEN }} + milestone: ${{ steps.get-milestone-to-assign.outputs.milestone }} runs-on: ubuntu-latest env: NODE_OPTIONS: '--max_old_space_size=4096' diff --git a/.github/workflows/wac/pullRequests.wac.ts b/.github/workflows/wac/pullRequests.wac.ts index 60f304b84e4..9c206bb6121 100644 --- a/.github/workflows/wac/pullRequests.wac.ts +++ b/.github/workflows/wac/pullRequests.wac.ts @@ -210,10 +210,32 @@ export const pullRequests = createWorkflow({ assignMilestone: createJob({ name: "Assign milestone", needs: "constants", + if: "needs.constants.outputs.is-fork-pr != 'true'", steps: [ { name: "Print latest Webiny version", run: "echo ${{ needs.constants.outputs.latest-webiny-version }}" + }, + { + id: "get-milestone-to-assign", + name: "Get milestone to assign", + run: runNodeScript( + "getMilestoneToAssign", + JSON.stringify({ + latestWebinyVersion: + '"${{ needs.constants.outputs.latest-webiny-version }}"', + baseBranch: "${{ github.base_ref }}" + }), + { outputAs: "milestone" } + ) + }, + { + uses: "zoispag/action-assign-milestone@v1", + if: "steps.get-milestone-to-assign.outputs.milestone", + with: { + "repo-token": "${{ secrets.GH_TOKEN }}", + milestone: "${{ steps.get-milestone-to-assign.outputs.milestone }}" + } } ] }), diff --git a/.github/workflows/wac/utils/runNodeScripts/getMilestoneToAssign.js b/.github/workflows/wac/utils/runNodeScripts/getMilestoneToAssign.js new file mode 100644 index 00000000000..b1f7bba5ed3 --- /dev/null +++ b/.github/workflows/wac/utils/runNodeScripts/getMilestoneToAssign.js @@ -0,0 +1,17 @@ +// Returns the milestone to assign to the PR based on the base branch and the latest Webiny version. +const args = process.argv.slice(2); // Removes the first two elements +const [params] = args; +const { latestWebinyVersion, baseBranch } = JSON.parse(params); + +const [major, minor, patch] = latestWebinyVersion.split("."); + +switch (baseBranch) { + case "next": + console.log(`${major}.${parseInt(minor, 10) + 1}.0`); + break; + case "dev": + console.log(`${major}.${minor}.${parseInt(patch, 10) + 1}`); + break; + default: + console.log(""); +} From 9b56a77c42c1af3558c381cc9194b881b0e13550 Mon Sep 17 00:00:00 2001 From: adrians5j Date: Fri, 15 Nov 2024 13:22:55 +0100 Subject: [PATCH 13/16] ci: auto-assign ms [no ci] --- .github/workflows/pullRequests.yml | 4 ++-- .github/workflows/wac/pullRequests.wac.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pullRequests.yml b/.github/workflows/pullRequests.yml index 4df8b5fb9b0..d1d7eebad7c 100644 --- a/.github/workflows/pullRequests.yml +++ b/.github/workflows/pullRequests.yml @@ -105,8 +105,8 @@ jobs: run: >- echo "milestone=$(node .github/workflows/wac/utils/runNodeScripts/getMilestoneToAssign.js - '{"latestWebinyVersion":"\"${{ - needs.constants.outputs.latest-webiny-version }}\"","baseBranch":"${{ + '{"latestWebinyVersion":"${{ + needs.constants.outputs.latest-webiny-version }}","baseBranch":"${{ github.base_ref }}"}')" >> $GITHUB_OUTPUT - uses: zoispag/action-assign-milestone@v1 if: steps.get-milestone-to-assign.outputs.milestone diff --git a/.github/workflows/wac/pullRequests.wac.ts b/.github/workflows/wac/pullRequests.wac.ts index 9c206bb6121..723dcc6e574 100644 --- a/.github/workflows/wac/pullRequests.wac.ts +++ b/.github/workflows/wac/pullRequests.wac.ts @@ -223,7 +223,7 @@ export const pullRequests = createWorkflow({ "getMilestoneToAssign", JSON.stringify({ latestWebinyVersion: - '"${{ needs.constants.outputs.latest-webiny-version }}"', + "${{ needs.constants.outputs.latest-webiny-version }}", baseBranch: "${{ github.base_ref }}" }), { outputAs: "milestone" } From f0cd0250f069116d1ab22a35dccb8b97d0e438f9 Mon Sep 17 00:00:00 2001 From: adrians5j Date: Fri, 15 Nov 2024 13:24:54 +0100 Subject: [PATCH 14/16] ci: use correct token [no ci] --- .github/workflows/labeler.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index 39ad7740b23..95c5e39b417 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -14,5 +14,5 @@ jobs: fetch-depth: 0 - uses: actions/labeler@v5 with: - repo-token: ${{ secrets.GITHUB_TOKEN }} + repo-token: ${{ secrets.GH_TOKEN }} configuration-path: .github/labeler-config.yml \ No newline at end of file From fb60f824799206806bac2a79641b521a3ae11959 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20Zori=C4=87?= Date: Fri, 15 Nov 2024 13:58:09 +0100 Subject: [PATCH 15/16] test(api-headless-cms): check for published and unpublished references (#4390) --- .../publishedAndUnpublished.test.ts | 206 ++++++++++++++++++ .../contentAPI/republish.entries.test.ts | 11 +- .../__tests__/storageOperations/context.ts | 34 +++ .../storageOperations/entries.test.ts | 10 +- .../fieldUniqueValues.test.ts | 10 +- packages/api-headless-cms/package.json | 1 + packages/api-headless-cms/tsconfig.build.json | 3 +- packages/api-headless-cms/tsconfig.json | 7 +- .../cli-plugin-extensions/tsconfig.build.json | 21 +- packages/cli-plugin-extensions/tsconfig.json | 16 +- .../tsconfig.build.json | 13 +- yarn.lock | 1 + 12 files changed, 280 insertions(+), 53 deletions(-) create mode 100644 packages/api-headless-cms/__tests__/contentAPI/references/publishedAndUnpublished.test.ts create mode 100644 packages/api-headless-cms/__tests__/storageOperations/context.ts diff --git a/packages/api-headless-cms/__tests__/contentAPI/references/publishedAndUnpublished.test.ts b/packages/api-headless-cms/__tests__/contentAPI/references/publishedAndUnpublished.test.ts new file mode 100644 index 00000000000..ea2c214bcc1 --- /dev/null +++ b/packages/api-headless-cms/__tests__/contentAPI/references/publishedAndUnpublished.test.ts @@ -0,0 +1,206 @@ +import { useCategoryManageHandler } from "../../testHelpers/useCategoryManageHandler"; +import { useArticleManageHandler } from "../../testHelpers/useArticleManageHandler"; +import { useArticleReadHandler } from "../../testHelpers/useArticleReadHandler"; +import { useGraphQLHandler } from "../../testHelpers/useGraphQLHandler"; +import { setupContentModelGroup, setupContentModels } from "../../testHelpers/setup"; +import { GenericRecord } from "@webiny/api/types"; +import slugify from "slugify"; + +interface ICreateCategoryItemPrams { + manager: ReturnType; + publish: boolean; + data: GenericRecord; +} + +const createCategoryItem = async ({ manager, publish, data }: ICreateCategoryItemPrams) => { + const [response] = await manager.createCategory({ data }); + const category = response?.data?.createCategory?.data; + const error = response?.data?.createCategory?.error; + if (!category?.id || error) { + console.log(error.message); + console.log(JSON.stringify(error.data)); + throw new Error("Could not create category."); + } + if (!publish) { + return category; + } + const [publishResponse] = await manager.publishCategory({ + revision: category.id + }); + if (publishResponse?.data?.publishCategory?.error) { + console.log(publishResponse?.data?.publishCategory?.error?.message); + throw new Error("Could not publish category."); + } + return publishResponse.data.publishCategory.data; +}; + +interface ICreateArticleItemPrams { + manager: ReturnType; + publish: boolean; + data: GenericRecord; +} + +const createArticleItem = async ({ manager, publish, data }: ICreateArticleItemPrams) => { + const [response] = await manager.createArticle({ data }); + const article = response?.data?.createArticle?.data; + const error = response?.data?.createArticle?.error; + if (!article?.id || error) { + console.log(error.message); + console.log(JSON.stringify(error.data)); + throw new Error("Could not create article."); + } + if (!publish) { + return article; + } + const [publishResponse] = await manager.publishArticle({ + revision: article.id + }); + if (publishResponse?.data?.publishArticle?.error) { + console.log(publishResponse?.data?.publishArticle?.error?.message); + throw new Error("Could not publish article."); + } + return publishResponse.data.publishArticle.data; +}; + +interface ICategoryItem { + id: string; + entryId: string; + title: string; + slug: string; + published: boolean; +} + +const categoryNames = ["Tech", "Health", "Space", "Food", "Science", "Sports"]; + +describe("published and unpublished references", () => { + const manageOpts = { path: "manage/en-US" }; + const readOpts = { path: "read/en-US" }; + + const mainManager = useGraphQLHandler(manageOpts); + + it("should populate reference field with some published and some unpublished records", async () => { + const group = await setupContentModelGroup(mainManager); + await setupContentModels(mainManager, group, ["category", "article"]); + + const categoryManager = useCategoryManageHandler(manageOpts); + const articleManager = useArticleManageHandler(manageOpts); + const articleRead = useArticleReadHandler(readOpts); + + const categories: ICategoryItem[] = []; + + for (const index in categoryNames) { + const title = categoryNames[index]; + const published = Number(index) % 2 === 0; + const category = await createCategoryItem({ + manager: categoryManager, + data: { + title: title, + slug: slugify(title) + }, + publish: published + }); + categories.push({ + ...category, + published + }); + } + expect(categories.length).toBe(categoryNames.length); + + const firstUnpublishedCategoryId = categories.find(c => !c.published)!.id; + expect(firstUnpublishedCategoryId).toMatch(/^([a-zA-Z0-9]+)#0001$/); + /** + * Create an article and make sure all the categories are in it. + */ + const createdArticle = await createArticleItem({ + manager: articleManager, + data: { + title: "Tech article", + body: null, + category: { + id: firstUnpublishedCategoryId, + modelId: "category" + }, + categories: categories.map(c => { + return { + id: c.id, + modelId: "category" + }; + }) + }, + publish: false + }); + + const expectedAllCategories = categories.map(c => { + return { + id: c.id, + entryId: c.entryId, + modelId: "category" + }; + }); + const expectedPublishedCategories = categories + .filter(c => c.published) + .map(c => { + return { + id: c.id, + entryId: c.entryId, + modelId: "category" + }; + }); + expect(expectedAllCategories).toHaveLength(expectedPublishedCategories.length * 2); + + expect(createdArticle.categories).toEqual(expectedAllCategories); + + const [articleManageGetResponse] = await articleManager.getArticle({ + revision: createdArticle.id + }); + expect(articleManageGetResponse?.data?.getArticle?.data?.categories).toEqual( + expectedAllCategories + ); + expect(articleManageGetResponse?.data?.getArticle?.data?.category).toMatchObject({ + id: firstUnpublishedCategoryId + }); + /** + * Now we can publish the article and check that references are still there. + */ + const [publishResponse] = await articleManager.publishArticle({ + revision: createdArticle.id + }); + expect(publishResponse?.data?.publishArticle?.data?.categories).toEqual( + expectedAllCategories + ); + expect(publishResponse?.data?.publishArticle?.data?.category).toMatchObject({ + id: firstUnpublishedCategoryId + }); + /** + * Now we can read the article, from manage endpoint, and check that references are still there. + * + * There must be all the categories present. + */ + const [articleManageGetPublishedResponse] = await articleManager.getArticle({ + revision: createdArticle.id + }); + expect(articleManageGetPublishedResponse?.data?.getArticle?.data?.categories).toEqual( + expectedAllCategories + ); + expect(articleManageGetPublishedResponse?.data?.getArticle?.data?.category).toMatchObject({ + id: firstUnpublishedCategoryId + }); + /** + * And read from the read endpoint... + * + * There must be only published categories present. + */ + const [articleReadGetPublishedResponse] = await articleRead.getArticle({ + where: { + id: createdArticle.id + } + }); + expect(articleReadGetPublishedResponse?.data?.getArticle?.data?.categories).toMatchObject( + expectedPublishedCategories + ); + expect(articleReadGetPublishedResponse?.data?.getArticle?.data?.categories).toHaveLength( + expectedPublishedCategories.length + ); + expect(articleReadGetPublishedResponse?.data?.getArticle?.data?.category).toBeNull(); + }); +}); diff --git a/packages/api-headless-cms/__tests__/contentAPI/republish.entries.test.ts b/packages/api-headless-cms/__tests__/contentAPI/republish.entries.test.ts index e457e149ee1..b1a6c69aeac 100644 --- a/packages/api-headless-cms/__tests__/contentAPI/republish.entries.test.ts +++ b/packages/api-headless-cms/__tests__/contentAPI/republish.entries.test.ts @@ -1,10 +1,11 @@ import { mdbid } from "@webiny/utils"; import models from "./mocks/contentModels"; import { useGraphQLHandler } from "../testHelpers/useGraphQLHandler"; -import { CmsContext, CmsEntry, CmsGroup, CmsModel, StorageOperationsCmsModel } from "~/types"; +import { CmsEntry, CmsGroup, CmsModel, StorageOperationsCmsModel } from "~/types"; import { useCategoryManageHandler } from "../testHelpers/useCategoryManageHandler"; import { useCategoryReadHandler } from "../testHelpers/useCategoryReadHandler"; import { useProductManageHandler } from "../testHelpers/useProductManageHandler"; +import { createStorageOperationsContext } from "~tests/storageOperations/context"; const cliPackageJson = require("@webiny/cli/package.json"); const webinyVersion = cliPackageJson.version; @@ -288,9 +289,11 @@ describe("Republish entries", () => { const { storageOperations, plugins } = useCategoryManageHandler(manageOpts); - await storageOperations.beforeInit({ - plugins - } as CmsContext); + await storageOperations.beforeInit( + await createStorageOperationsContext({ + plugins + }) + ); const { entry: galaEntry } = createEntry(productModel, { title: "Gala", diff --git a/packages/api-headless-cms/__tests__/storageOperations/context.ts b/packages/api-headless-cms/__tests__/storageOperations/context.ts new file mode 100644 index 00000000000..f0fe3755da9 --- /dev/null +++ b/packages/api-headless-cms/__tests__/storageOperations/context.ts @@ -0,0 +1,34 @@ +import dbPlugins from "@webiny/handler-db"; +import { PluginsContainer } from "@webiny/plugins"; +import { getDocumentClient } from "@webiny/project-utils/testing/dynamodb"; +import { DynamoDbDriver } from "@webiny/db-dynamodb"; +import { CmsContext } from "~/types"; +import { Context } from "@webiny/api"; + +export interface ICreateStorageOperationsContextParams { + plugins?: PluginsContainer; +} + +export const createStorageOperationsContext = async ( + params: ICreateStorageOperationsContextParams +): Promise => { + const dbPluginsInitialized = dbPlugins({ + table: process.env.DB_TABLE, + driver: new DynamoDbDriver({ + documentClient: getDocumentClient() + }) + }); + const plugins = params.plugins || new PluginsContainer([]); + plugins.register(...dbPluginsInitialized); + + const context = new Context({ + plugins, + WEBINY_VERSION: "0.0.0" + }) as unknown as CmsContext; + + for (const db of dbPluginsInitialized) { + await db.apply(context); + } + + return context; +}; diff --git a/packages/api-headless-cms/__tests__/storageOperations/entries.test.ts b/packages/api-headless-cms/__tests__/storageOperations/entries.test.ts index 6920c5331c5..a0879ab63f3 100644 --- a/packages/api-headless-cms/__tests__/storageOperations/entries.test.ts +++ b/packages/api-headless-cms/__tests__/storageOperations/entries.test.ts @@ -1,6 +1,6 @@ import { createPersonEntries, createPersonModel, deletePersonModel } from "./helpers"; import { useGraphQLHandler } from "../testHelpers/useGraphQLHandler"; -import { CmsContext } from "~/types"; +import { createStorageOperationsContext } from "~tests/storageOperations/context"; jest.setTimeout(90000); @@ -15,9 +15,11 @@ describe("Entries storage operations", () => { * Some others might not need them... */ beforeAll(async () => { - await storageOperations.beforeInit({ - plugins - } as unknown as CmsContext); + await storageOperations.beforeInit( + await createStorageOperationsContext({ + plugins + }) + ); }); beforeEach(async () => { diff --git a/packages/api-headless-cms/__tests__/storageOperations/fieldUniqueValues.test.ts b/packages/api-headless-cms/__tests__/storageOperations/fieldUniqueValues.test.ts index 6e49c1b6206..4d358ef2a7f 100644 --- a/packages/api-headless-cms/__tests__/storageOperations/fieldUniqueValues.test.ts +++ b/packages/api-headless-cms/__tests__/storageOperations/fieldUniqueValues.test.ts @@ -1,10 +1,10 @@ -import { CmsContext } from "~/types"; import { createPersonEntries, createPersonModel, deletePersonModel } from "~tests/storageOperations/helpers"; import { useGraphQLHandler } from "~tests/testHelpers/useGraphQLHandler"; +import { createStorageOperationsContext } from "~tests/storageOperations/context"; describe("field unique values listing", () => { const { storageOperations, plugins } = useGraphQLHandler({ @@ -17,9 +17,11 @@ describe("field unique values listing", () => { * Some others might not need them... */ beforeAll(async () => { - await storageOperations.beforeInit({ - plugins - } as unknown as CmsContext); + await storageOperations.beforeInit( + await createStorageOperationsContext({ + plugins + }) + ); }); beforeEach(async () => { diff --git a/packages/api-headless-cms/package.json b/packages/api-headless-cms/package.json index 359939c82bb..8123558e380 100644 --- a/packages/api-headless-cms/package.json +++ b/packages/api-headless-cms/package.json @@ -56,6 +56,7 @@ "@webiny/api-wcp": "0.0.0", "@webiny/aws-sdk": "0.0.0", "@webiny/cli": "0.0.0", + "@webiny/db-dynamodb": "0.0.0", "@webiny/project-utils": "0.0.0", "apollo-graphql": "^0.9.5", "get-yarn-workspaces": "^1.0.2", diff --git a/packages/api-headless-cms/tsconfig.build.json b/packages/api-headless-cms/tsconfig.build.json index 777323f7d4c..3062670240e 100644 --- a/packages/api-headless-cms/tsconfig.build.json +++ b/packages/api-headless-cms/tsconfig.build.json @@ -17,7 +17,8 @@ { "path": "../utils/tsconfig.build.json" }, { "path": "../validation/tsconfig.build.json" }, { "path": "../api-wcp/tsconfig.build.json" }, - { "path": "../aws-sdk/tsconfig.build.json" } + { "path": "../aws-sdk/tsconfig.build.json" }, + { "path": "../db-dynamodb/tsconfig.build.json" } ], "compilerOptions": { "rootDir": "./src", diff --git a/packages/api-headless-cms/tsconfig.json b/packages/api-headless-cms/tsconfig.json index ec25eab3be4..06ba7176b3a 100644 --- a/packages/api-headless-cms/tsconfig.json +++ b/packages/api-headless-cms/tsconfig.json @@ -17,7 +17,8 @@ { "path": "../utils" }, { "path": "../validation" }, { "path": "../api-wcp" }, - { "path": "../aws-sdk" } + { "path": "../aws-sdk" }, + { "path": "../db-dynamodb" } ], "compilerOptions": { "rootDirs": ["./src", "./__tests__"], @@ -57,7 +58,9 @@ "@webiny/api-wcp/*": ["../api-wcp/src/*"], "@webiny/api-wcp": ["../api-wcp/src"], "@webiny/aws-sdk/*": ["../aws-sdk/src/*"], - "@webiny/aws-sdk": ["../aws-sdk/src"] + "@webiny/aws-sdk": ["../aws-sdk/src"], + "@webiny/db-dynamodb/*": ["../db-dynamodb/src/*"], + "@webiny/db-dynamodb": ["../db-dynamodb/src"] }, "baseUrl": "." } diff --git a/packages/cli-plugin-extensions/tsconfig.build.json b/packages/cli-plugin-extensions/tsconfig.build.json index 0c0500749d2..5fd85bfd03d 100644 --- a/packages/cli-plugin-extensions/tsconfig.build.json +++ b/packages/cli-plugin-extensions/tsconfig.build.json @@ -2,27 +2,16 @@ "extends": "../../tsconfig.build.json", "include": ["src"], "references": [ - { - "path": "../plugins/tsconfig.build.json" - }, - { - "path": "../aws-sdk/tsconfig.build.json" - }, - { - "path": "../cli-plugin-scaffold/tsconfig.build.json" - }, - { - "path": "../error/tsconfig.build.json" - } + { "path": "../aws-sdk/tsconfig.build.json" }, + { "path": "../cli-plugin-scaffold/tsconfig.build.json" }, + { "path": "../error/tsconfig.build.json" }, + { "path": "../plugins/tsconfig.build.json" } ], "compilerOptions": { "rootDir": "./src", "outDir": "./dist", "declarationDir": "./dist", - "paths": { - "~/*": ["./src/*"], - "~tests/*": ["./__tests__/*"] - }, + "paths": { "~/*": ["./src/*"], "~tests/*": ["./__tests__/*"] }, "baseUrl": "." } } diff --git a/packages/cli-plugin-extensions/tsconfig.json b/packages/cli-plugin-extensions/tsconfig.json index b21ff1a398f..564aad64d1e 100644 --- a/packages/cli-plugin-extensions/tsconfig.json +++ b/packages/cli-plugin-extensions/tsconfig.json @@ -2,18 +2,10 @@ "extends": "../../tsconfig.json", "include": ["src", "__tests__"], "references": [ - { - "path": "../aws-sdk" - }, - { - "path": "../cli-plugin-scaffold" - }, - { - "path": "../error" - }, - { - "path": "../plugins" - } + { "path": "../aws-sdk" }, + { "path": "../cli-plugin-scaffold" }, + { "path": "../error" }, + { "path": "../plugins" } ], "compilerOptions": { "rootDirs": ["./src", "./__tests__"], diff --git a/packages/cli-plugin-scaffold-extensions/tsconfig.build.json b/packages/cli-plugin-scaffold-extensions/tsconfig.build.json index fbaa95c4e0b..d0a312dc64a 100644 --- a/packages/cli-plugin-scaffold-extensions/tsconfig.build.json +++ b/packages/cli-plugin-scaffold-extensions/tsconfig.build.json @@ -2,21 +2,14 @@ "extends": "../../tsconfig.build.json", "include": ["src"], "references": [ - { - "path": "../cli-plugin-extensions/tsconfig.build.json" - }, - { - "path": "../cli-plugin-scaffold/tsconfig.build.json" - } + { "path": "../cli-plugin-extensions/tsconfig.build.json" }, + { "path": "../cli-plugin-scaffold/tsconfig.build.json" } ], "compilerOptions": { "rootDir": "./src", "outDir": "./dist", "declarationDir": "./dist", - "paths": { - "~/*": ["./src/*"], - "~tests/*": ["./__tests__/*"] - }, + "paths": { "~/*": ["./src/*"], "~tests/*": ["./__tests__/*"] }, "baseUrl": "." } } diff --git a/yarn.lock b/yarn.lock index cfbe168e186..e6b461baa7e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -14371,6 +14371,7 @@ __metadata: "@webiny/api-wcp": 0.0.0 "@webiny/aws-sdk": 0.0.0 "@webiny/cli": 0.0.0 + "@webiny/db-dynamodb": 0.0.0 "@webiny/error": 0.0.0 "@webiny/handler": 0.0.0 "@webiny/handler-aws": 0.0.0 From 98bb13c1eddc99625bac294b3ba1f111dbb4bb6c Mon Sep 17 00:00:00 2001 From: adrians5j Date: Fri, 15 Nov 2024 14:11:52 +0100 Subject: [PATCH 16/16] ci: rebuild workflow --- .github/workflows/pullRequests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pullRequests.yml b/.github/workflows/pullRequests.yml index d1d7eebad7c..b584e369c86 100644 --- a/.github/workflows/pullRequests.yml +++ b/.github/workflows/pullRequests.yml @@ -327,7 +327,8 @@ jobs: --storage=ddb","storage":"ddb","packageName":"api-headless-cms-aco","id":"718c110b004c59ed7d13cbcc875a6b64"},{"cmd":"packages/api-headless-cms-bulk-actions --storage=ddb","storage":"ddb","packageName":"api-headless-cms-bulk-actions","id":"00c0a57737502f28c304015d2d1ba442"},{"cmd":"packages/api-headless-cms-import-export --storage=ddb","storage":"ddb","packageName":"api-headless-cms-import-export","id":"e9052e7c40171aeb43ce089fdfbbe3c8"},{"cmd":"packages/api-i18n - --storage=ddb","storage":"ddb","packageName":"api-i18n","id":"943e15fe21c847b164f9413f8baf97b7"},{"cmd":"packages/api-mailer + --storage=ddb","storage":"ddb","packageName":"api-i18n","id":"943e15fe21c847b164f9413f8baf97b7"},{"cmd":"packages/api-log + --storage=ddb","storage":"ddb","packageName":"api-log","id":"9baae1f165e409fea40713e0cf2d300f"},{"cmd":"packages/api-mailer --storage=ddb","storage":"ddb","packageName":"api-mailer","id":"2cc1dc707a39e72f4e5d9a140677ca39"},{"cmd":"packages/api-page-builder --storage=ddb --shard=1/6","storage":"ddb","packageName":"api-page-builder","id":"b2a30dfaf230076ce7120c55eb581d32"},{"cmd":"packages/api-page-builder