From d79172aed85a8b144d2d83d12cb2fa02374ce852 Mon Sep 17 00:00:00 2001 From: Adrian Smijulj Date: Thu, 27 Jun 2024 22:45:41 +0200 Subject: [PATCH 01/70] fix: ensure published records are updated accordingly (#4184) --- .../src/operations/entry/index.ts | 62 +++++++++++++++---- .../src/operations/entry/index.ts | 17 +++++ ...tentEntriesOnByMetaFieldsOverrides.test.ts | 51 ++++++++++++--- 3 files changed, 111 insertions(+), 19 deletions(-) diff --git a/packages/api-headless-cms-ddb-es/src/operations/entry/index.ts b/packages/api-headless-cms-ddb-es/src/operations/entry/index.ts index 9742c42422a..c4a6d033449 100644 --- a/packages/api-headless-cms-ddb-es/src/operations/entry/index.ts +++ b/packages/api-headless-cms-ddb-es/src/operations/entry/index.ts @@ -43,7 +43,7 @@ import { createElasticsearchBody } from "./elasticsearch/body"; import { createLatestRecordType, createPublishedRecordType, createRecordType } from "./recordType"; import { StorageOperationsCmsModelPlugin } from "@webiny/api-headless-cms"; import { WriteRequest } from "@webiny/aws-sdk/client-dynamodb"; -import { batchReadAll, BatchReadItem, put } from "@webiny/db-dynamodb"; +import { batchReadAll, BatchReadItem } from "@webiny/db-dynamodb"; import { createTransformer } from "./transformations"; import { convertEntryKeysFromStorage } from "./transformations/convertEntryKeys"; import { @@ -276,6 +276,18 @@ export const createEntriesStorageOperations = ( SK: createLatestSortKey() }; + const publishedKeys = { + PK: createPartitionKey({ + id: entry.id, + locale: model.locale, + tenant: model.tenant + }), + SK: createPublishedSortKey() + }; + + // We'll need this flag below. + const isPublished = entry.status === "published"; + const esLatestData = await transformer.getElasticsearchLatestEntryData(); const items = [ @@ -294,6 +306,17 @@ export const createEntriesStorageOperations = ( const { index } = configurations.es({ model }); + + if (isPublished) { + items.push( + entity.putBatch({ + ...storageEntry, + TYPE: createPublishedRecordType(), + ...publishedKeys + }) + ); + } + try { await batchWriteAll({ table: entity.table, @@ -313,17 +336,34 @@ export const createEntriesStorageOperations = ( } ); } - /** - * Update the "latest" entry item in the Elasticsearch - */ + + const { index: esIndex } = configurations.es({ + model + }); + + const esItems: BatchWriteItem[] = [ + esEntity.putBatch({ + ...latestKeys, + index: esIndex, + data: esLatestData + }) + ]; + + if (isPublished) { + const esPublishedData = await transformer.getElasticsearchPublishedEntryData(); + esItems.push( + esEntity.putBatch({ + ...publishedKeys, + index: esIndex, + data: esPublishedData + }) + ); + } + try { - await put({ - entity: esEntity, - item: { - ...latestKeys, - index, - data: esLatestData - } + await batchWriteAll({ + table: esEntity.table, + items: esItems }); } catch (ex) { throw new WebinyError( diff --git a/packages/api-headless-cms-ddb/src/operations/entry/index.ts b/packages/api-headless-cms-ddb/src/operations/entry/index.ts index a9230cc2801..d2f54ef6df9 100644 --- a/packages/api-headless-cms-ddb/src/operations/entry/index.ts +++ b/packages/api-headless-cms-ddb/src/operations/entry/index.ts @@ -241,10 +241,12 @@ export const createEntriesStorageOperations = ( storageEntry: initialStorageEntry, model }); + /** * We need to: * - create the main entry item * - update the last entry item to a current one + * - update the published entry item to a current one (if the entry is published) */ const items = [ entity.putBatch({ @@ -264,6 +266,21 @@ export const createEntriesStorageOperations = ( GSI1_SK: createGSISortKey(storageEntry) }) ]; + + const isPublished = entry.status === "published"; + if (isPublished) { + items.push( + entity.putBatch({ + ...storageEntry, + PK: partitionKey, + SK: createPublishedSortKey(), + TYPE: createPublishedType(), + GSI1_PK: createGSIPartitionKey(model, "P"), + GSI1_SK: createGSISortKey(storageEntry) + }) + ); + } + try { await batchWriteAll({ table: entity.table, diff --git a/packages/api-headless-cms/__tests__/contentAPI/contentEntriesOnByMetaFieldsOverrides.test.ts b/packages/api-headless-cms/__tests__/contentAPI/contentEntriesOnByMetaFieldsOverrides.test.ts index 0b6683a48f3..ee008997469 100644 --- a/packages/api-headless-cms/__tests__/contentAPI/contentEntriesOnByMetaFieldsOverrides.test.ts +++ b/packages/api-headless-cms/__tests__/contentAPI/contentEntriesOnByMetaFieldsOverrides.test.ts @@ -2,19 +2,19 @@ import { useTestModelHandler } from "~tests/testHelpers/useTestModelHandler"; import { identityA, identityB, identityC, identityD } from "./security/utils"; describe("Content entries - Entry Meta Fields Overrides", () => { - const { manage: managerIdentityA } = useTestModelHandler({ + const { read: readIdentityA, manage: manageIdentityA } = useTestModelHandler({ identity: identityA }); beforeEach(async () => { - await managerIdentityA.setup(); + await manageIdentityA.setup(); }); test("users should be able to create and immediately publish an entry with custom publishing-related values", async () => { // 1. Initially, all meta fields should be populated, except the "modified" ones. const testDate = new Date("2020-01-01T00:00:00.000Z").toISOString(); - const { data: rev } = await managerIdentityA.createTestEntry({ + const { data: rev } = await manageIdentityA.createTestEntry({ data: { status: "published", revisionFirstPublishedOn: testDate, @@ -50,7 +50,7 @@ describe("Content entries - Entry Meta Fields Overrides", () => { const testDate2 = new Date("2021-01-01T00:00:00.000Z").toISOString(); const testDate3 = new Date("2022-01-01T00:00:00.000Z").toISOString(); - const { data: rev } = await managerIdentityA.createTestEntry({ + const { data: rev } = await manageIdentityA.createTestEntry({ data: { status: "published", revisionFirstPublishedOn: testDate1, @@ -65,7 +65,7 @@ describe("Content entries - Entry Meta Fields Overrides", () => { }); const { data: publishedRevWithCustomLastPublishedValues } = - await managerIdentityA.createTestEntryFrom({ + await manageIdentityA.createTestEntryFrom({ revision: rev.id, data: { status: "published", @@ -98,8 +98,8 @@ describe("Content entries - Entry Meta Fields Overrides", () => { rev.revisionFirstPublishedOn ).toBeTrue(); - const { data: publishedRevWithAllCustomValues } = - await managerIdentityA.createTestEntryFrom({ + const { data: publishedRevWithAllCustomValues } = await manageIdentityA.createTestEntryFrom( + { revision: publishedRevWithCustomLastPublishedValues.id, data: { status: "published", @@ -112,7 +112,8 @@ describe("Content entries - Entry Meta Fields Overrides", () => { firstPublishedBy: identityD, lastPublishedBy: identityD } - }); + } + ); expect(publishedRevWithAllCustomValues).toMatchObject({ createdOn: expect.toBeDateString(), @@ -128,5 +129,39 @@ describe("Content entries - Entry Meta Fields Overrides", () => { firstPublishedBy: identityD, lastPublishedBy: identityD }); + + // Ensure that the new published revision is the one that is returned when listing or getting the entry. + + // 1. Manage API. + const { data: getEntryManage } = await manageIdentityA.getTestEntry({ + entryId: rev.entryId + }); + + expect(getEntryManage).toMatchObject({ + meta: { + status: "published", + version: 3 + } + }); + + const { data: listEntriesManage } = await manageIdentityA.listTestEntries(); + + expect(listEntriesManage).toMatchObject([ + { + meta: { + status: "published", + version: 3 + } + } + ]); + + // 2. Read API (here we can't get versions directly, so we're just inspecting the revision ID). + const { data: getEntryRead } = await readIdentityA.getTestEntry({ + where: { entryId: rev.entryId } + }); + expect(getEntryRead.id).toEndWith("#0003"); + + const { data: listEntriesRead } = await readIdentityA.listTestEntries(); + expect(listEntriesRead[0].id).toEndWith("#0003"); }); }); From 34335609913a7aa93d6a484bfa2e5e847e6e600e Mon Sep 17 00:00:00 2001 From: Adrian Smijulj Date: Fri, 28 Jun 2024 08:02:38 +0200 Subject: [PATCH 02/70] fix: add modelId and used index name to the log (#4186) --- .../logIgnoredEsResponseError.ts | 23 ++++++++ .../shouldIgnoreEsResponseError.ts | 10 ++++ .../src/operations/entry/index.ts | 55 ++++++++----------- 3 files changed, 57 insertions(+), 31 deletions(-) create mode 100644 packages/api-headless-cms-ddb-es/src/operations/entry/elasticsearch/logIgnoredEsResponseError.ts create mode 100644 packages/api-headless-cms-ddb-es/src/operations/entry/elasticsearch/shouldIgnoreEsResponseError.ts diff --git a/packages/api-headless-cms-ddb-es/src/operations/entry/elasticsearch/logIgnoredEsResponseError.ts b/packages/api-headless-cms-ddb-es/src/operations/entry/elasticsearch/logIgnoredEsResponseError.ts new file mode 100644 index 00000000000..087d97705c1 --- /dev/null +++ b/packages/api-headless-cms-ddb-es/src/operations/entry/elasticsearch/logIgnoredEsResponseError.ts @@ -0,0 +1,23 @@ +import WebinyError from "@webiny/error"; +import { CmsModel } from "@webiny/api-headless-cms/types"; + +interface LogIgnoredElasticsearchExceptionParams { + error: WebinyError; + model: CmsModel; + indexName: string; +} + +export const logIgnoredEsResponseError = (params: LogIgnoredElasticsearchExceptionParams) => { + const { error, indexName, model } = params; + + console.log(`Ignoring Elasticsearch response error: ${error.message}`, { + modelId: model.modelId, + usedIndexName: indexName, + error: { + message: error.message, + code: error.code, + data: error.data, + stack: error.stack + } + }); +}; diff --git a/packages/api-headless-cms-ddb-es/src/operations/entry/elasticsearch/shouldIgnoreEsResponseError.ts b/packages/api-headless-cms-ddb-es/src/operations/entry/elasticsearch/shouldIgnoreEsResponseError.ts new file mode 100644 index 00000000000..9455112cf02 --- /dev/null +++ b/packages/api-headless-cms-ddb-es/src/operations/entry/elasticsearch/shouldIgnoreEsResponseError.ts @@ -0,0 +1,10 @@ +import WebinyError from "@webiny/error"; + +const IGNORED_ES_SEARCH_EXCEPTIONS = [ + "index_not_found_exception", + "search_phase_execution_exception" +]; + +export const shouldIgnoreEsResponseError = (error: WebinyError) => { + return IGNORED_ES_SEARCH_EXCEPTIONS.includes(error.message); +}; diff --git a/packages/api-headless-cms-ddb-es/src/operations/entry/index.ts b/packages/api-headless-cms-ddb-es/src/operations/entry/index.ts index c4a6d033449..3abc8464087 100644 --- a/packages/api-headless-cms-ddb-es/src/operations/entry/index.ts +++ b/packages/api-headless-cms-ddb-es/src/operations/entry/index.ts @@ -40,6 +40,8 @@ import { } from "@webiny/api-elasticsearch/types"; import { CmsEntryStorageOperations, CmsIndexEntry } from "~/types"; import { createElasticsearchBody } from "./elasticsearch/body"; +import { logIgnoredEsResponseError } from "./elasticsearch/logIgnoredEsResponseError"; +import { shouldIgnoreEsResponseError } from "./elasticsearch/shouldIgnoreEsResponseError"; import { createLatestRecordType, createPublishedRecordType, createRecordType } from "./recordType"; import { StorageOperationsCmsModelPlugin } from "@webiny/api-headless-cms"; import { WriteRequest } from "@webiny/aws-sdk/client-dynamodb"; @@ -63,24 +65,6 @@ export interface CreateEntriesStorageOperationsParams { plugins: PluginsContainer; } -const IGNORED_ES_SEARCH_EXCEPTIONS = [ - "index_not_found_exception", - "search_phase_execution_exception" -]; - -const shouldIgnoreElasticsearchException = (ex: WebinyError) => { - if (IGNORED_ES_SEARCH_EXCEPTIONS.includes(ex.message)) { - console.log(`Ignoring Elasticsearch exception: ${ex.message}`); - console.log({ - code: ex.code, - data: ex.data, - stack: ex.stack - }); - return true; - } - return false; -}; - export const createEntriesStorageOperations = ( params: CreateEntriesStorageOperationsParams ): CmsEntryStorageOperations => { @@ -303,10 +287,6 @@ export const createEntriesStorageOperations = ( }) ]; - const { index } = configurations.es({ - model - }); - if (isPublished) { items.push( entity.putBatch({ @@ -1096,12 +1076,18 @@ export const createEntriesStorageOperations = ( index, body }); - } catch (ex) { + } catch (error) { /** * We will silently ignore the `index_not_found_exception` error and return an empty result set. * This is because the index might not exist yet, and we don't want to throw an error. */ - if (shouldIgnoreElasticsearchException(ex)) { + if (shouldIgnoreEsResponseError(error)) { + logIgnoredEsResponseError({ + error, + model, + indexName: index + }); + return { hasMoreItems: false, totalCount: 0, @@ -1109,8 +1095,9 @@ export const createEntriesStorageOperations = ( items: [] }; } - throw new WebinyError(ex.message, ex.code || "ELASTICSEARCH_ERROR", { - error: ex, + + throw new WebinyError(error.message, error.code || "ELASTICSEARCH_ERROR", { + error, index, body, model @@ -1807,15 +1794,21 @@ export const createEntriesStorageOperations = ( index, body }); - } catch (ex) { - if (shouldIgnoreElasticsearchException(ex)) { + } catch (error) { + if (shouldIgnoreEsResponseError(error)) { + logIgnoredEsResponseError({ + error, + model, + indexName: index + }); return []; } + throw new WebinyError( - ex.message || "Error in the Elasticsearch query.", - ex.code || "ELASTICSEARCH_ERROR", + error.message || "Error in the Elasticsearch query.", + error.code || "ELASTICSEARCH_ERROR", { - error: ex, + error, index, model, body From d65c7e6cf2d7a6a7a37d5c37642e1dc9c04c8c11 Mon Sep 17 00:00:00 2001 From: Adrian Smijulj Date: Fri, 28 Jun 2024 08:02:54 +0200 Subject: [PATCH 03/70] fix: improve disabling and restoration of ES indexing settings (#4187) --- .../src/migrations/5.38.0/002/ddb-es/index.ts | 1 - .../src/migrations/5.39.0/001/ddb-es/index.ts | 1 - .../src/migrations/5.39.2/001/ddb-es/index.ts | 1 - .../src/migrations/5.39.6/001/ddb-es/index.ts | 1 - .../src/utils/elasticsearch/disableEsIndexing.ts | 1 - .../elasticsearch/fetchOriginalEsSettings.ts | 4 +--- .../elasticsearch/restoreOriginalEsSettings.ts | 16 ++++++++++++---- 7 files changed, 13 insertions(+), 12 deletions(-) diff --git a/packages/migrations/src/migrations/5.38.0/002/ddb-es/index.ts b/packages/migrations/src/migrations/5.38.0/002/ddb-es/index.ts index b2e365be06f..09c05f68d3d 100644 --- a/packages/migrations/src/migrations/5.38.0/002/ddb-es/index.ts +++ b/packages/migrations/src/migrations/5.38.0/002/ddb-es/index.ts @@ -35,7 +35,6 @@ interface LastEvaluatedKey { } interface IndexSettings { - number_of_replicas: number; refresh_interval: `${number}s`; } diff --git a/packages/migrations/src/migrations/5.39.0/001/ddb-es/index.ts b/packages/migrations/src/migrations/5.39.0/001/ddb-es/index.ts index 8ce9267f12a..eae8ee791f4 100644 --- a/packages/migrations/src/migrations/5.39.0/001/ddb-es/index.ts +++ b/packages/migrations/src/migrations/5.39.0/001/ddb-es/index.ts @@ -39,7 +39,6 @@ interface LastEvaluatedKey { } interface IndexSettings { - number_of_replicas: number; refresh_interval: `${number}s`; } diff --git a/packages/migrations/src/migrations/5.39.2/001/ddb-es/index.ts b/packages/migrations/src/migrations/5.39.2/001/ddb-es/index.ts index 93a61137087..3cb23aeebea 100644 --- a/packages/migrations/src/migrations/5.39.2/001/ddb-es/index.ts +++ b/packages/migrations/src/migrations/5.39.2/001/ddb-es/index.ts @@ -43,7 +43,6 @@ interface LastEvaluatedKey { } interface IndexSettings { - number_of_replicas: number; refresh_interval: `${number}s`; } diff --git a/packages/migrations/src/migrations/5.39.6/001/ddb-es/index.ts b/packages/migrations/src/migrations/5.39.6/001/ddb-es/index.ts index fdf0511cd89..392c76a254e 100644 --- a/packages/migrations/src/migrations/5.39.6/001/ddb-es/index.ts +++ b/packages/migrations/src/migrations/5.39.6/001/ddb-es/index.ts @@ -46,7 +46,6 @@ interface LastEvaluatedKey { } interface IndexSettings { - number_of_replicas: number; refresh_interval: `${number}s`; } diff --git a/packages/migrations/src/utils/elasticsearch/disableEsIndexing.ts b/packages/migrations/src/utils/elasticsearch/disableEsIndexing.ts index 13984d373fd..48d014bf695 100644 --- a/packages/migrations/src/utils/elasticsearch/disableEsIndexing.ts +++ b/packages/migrations/src/utils/elasticsearch/disableEsIndexing.ts @@ -18,7 +18,6 @@ export const disableElasticsearchIndexing = async ( elasticsearchClient: params.elasticsearchClient, index, settings: { - number_of_replicas: 0, refresh_interval: -1 } }); diff --git a/packages/migrations/src/utils/elasticsearch/fetchOriginalEsSettings.ts b/packages/migrations/src/utils/elasticsearch/fetchOriginalEsSettings.ts index 5a6f0ca6601..0c277910189 100644 --- a/packages/migrations/src/utils/elasticsearch/fetchOriginalEsSettings.ts +++ b/packages/migrations/src/utils/elasticsearch/fetchOriginalEsSettings.ts @@ -9,7 +9,6 @@ interface FetchOriginalElasticsearchSettingsParams { } interface IndexSettings { - number_of_replicas: number; refresh_interval: `${number}s`; } @@ -21,10 +20,9 @@ export const fetchOriginalElasticsearchSettings = async ( const settings = await esGetIndexSettings({ elasticsearchClient: params.elasticsearchClient, index, - fields: ["number_of_replicas", "refresh_interval"] + fields: ["refresh_interval"] }); return { - number_of_replicas: settings.number_of_replicas || 1, refresh_interval: settings.refresh_interval || "1s" }; } catch (ex) { diff --git a/packages/migrations/src/utils/elasticsearch/restoreOriginalEsSettings.ts b/packages/migrations/src/utils/elasticsearch/restoreOriginalEsSettings.ts index bfbe59bd6f3..2d321d6c740 100644 --- a/packages/migrations/src/utils/elasticsearch/restoreOriginalEsSettings.ts +++ b/packages/migrations/src/utils/elasticsearch/restoreOriginalEsSettings.ts @@ -3,8 +3,7 @@ import { Logger } from "@webiny/data-migration"; import { Client } from "@elastic/elasticsearch"; interface IndexSettings { - number_of_replicas: number; - refresh_interval: `${number}s`; + refresh_interval: `${number}s` | "-1"; } interface RestoreOriginalElasticsearchSettingsParams { @@ -30,13 +29,22 @@ export const restoreOriginalElasticsearchSettings = async ( if (!settings || typeof settings !== "object") { continue; } + + // We must ensure that the refresh interval is not set to a negative value. Why? + // We've had a case where a migration run has been manually stopped, and the index settings + // were never restored. Once a second run was started and this restore function + // was called, the refresh interval was set to `-1s`, which effectively disabled indexing. + let refreshInterval = settings.refresh_interval || `1s`; + if (refreshInterval === "-1") { + refreshInterval = "1s"; + } + try { await esPutIndexSettings({ elasticsearchClient: params.elasticsearchClient, index, settings: { - number_of_replicas: settings.number_of_replicas || 1, - refresh_interval: settings.refresh_interval || `1s` + refresh_interval: refreshInterval } }); } catch (ex) { From ded920f7f223f29ef1f7e9c9bd0b6c89df3604a4 Mon Sep 17 00:00:00 2001 From: Adrian Smijulj Date: Fri, 28 Jun 2024 08:03:11 +0200 Subject: [PATCH 04/70] fix: add support for WEBINY_MIGRATION_FORCE_EXECUTE_5_39_6_001 env var --- .../5.39.6/001/ddb-es/MetaFieldsMigration.ts | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/packages/migrations/src/migrations/5.39.6/001/ddb-es/MetaFieldsMigration.ts b/packages/migrations/src/migrations/5.39.6/001/ddb-es/MetaFieldsMigration.ts index 79faf5f4a9a..bb6fcf8366b 100644 --- a/packages/migrations/src/migrations/5.39.6/001/ddb-es/MetaFieldsMigration.ts +++ b/packages/migrations/src/migrations/5.39.6/001/ddb-es/MetaFieldsMigration.ts @@ -71,8 +71,16 @@ export class MetaFieldsMigration { }); if (dataMigrationRecordExists) { - this.logger.info("5.39.6-001 migration has already been executed. Exiting..."); - return; + const forceExecuteEnvVar = process.env["WEBINY_MIGRATION_FORCE_EXECUTE_5_39_6_001"]; + const forceExecute = forceExecuteEnvVar === "true"; + if (!forceExecute) { + this.logger.info("5.39.6-001 migration has already been executed. Exiting..."); + return; + } + + this.logger.info( + "5.39.6-001 migration has already been executed, but force execution was requested." + ); } this.logger.info("Starting 5.39.6-001 meta fields data migration..."); From 29d7905244e46c345c4caee3bcde34b778011225 Mon Sep 17 00:00:00 2001 From: adrians5j Date: Wed, 14 Aug 2024 10:56:46 +0200 Subject: [PATCH 05/70] fix: improve error logging for batch writes --- .../migrations/5.39.6/001/ddb-es/worker.ts | 113 ++++++++++++------ 1 file changed, 75 insertions(+), 38 deletions(-) diff --git a/packages/migrations/src/migrations/5.39.6/001/ddb-es/worker.ts b/packages/migrations/src/migrations/5.39.6/001/ddb-es/worker.ts index 9418f20702c..d7dfa4e39d3 100644 --- a/packages/migrations/src/migrations/5.39.6/001/ddb-es/worker.ts +++ b/packages/migrations/src/migrations/5.39.6/001/ddb-es/worker.ts @@ -353,6 +353,9 @@ const createInitialStatus = (): MigrationStatus => { } if (ddbItemsToBatchWrite.length) { + let ddbWriteError = false; + let ddbEsWriteError = false; + // Store data in primary DynamoDB table. const execute = () => { return batchWriteAll({ @@ -364,56 +367,90 @@ const createInitialStatus = (): MigrationStatus => { logger.trace( `Storing ${ddbItemsToBatchWrite.length} record(s) in primary DynamoDB table...` ); - await executeWithRetry(execute, { - onFailedAttempt: error => { - logger.warn( - `Batch write attempt #${error.attemptNumber} failed: ${error.message}` - ); - } - }); + + try { + await executeWithRetry(execute, { + onFailedAttempt: error => { + logger.warn( + `Batch write attempt #${error.attemptNumber} failed: ${error.message}` + ); + } + }); + } catch (e) { + ddbWriteError = true; + logger.error( + { + error: e, + ddbItemsToBatchWrite + }, + "After multiple retries, failed to batch-store records in primary DynamoDB table." + ); + } if (ddbEsItemsToBatchWrite.length) { logger.trace( `Storing ${ddbEsItemsToBatchWrite.length} record(s) in DDB-ES DynamoDB table...` ); - const results = await waitUntilHealthy.wait({ - async onUnhealthy(params) { - const shouldWaitReason = params.waitingReason.name; - logger.warn( - `Cluster is unhealthy (${shouldWaitReason}). Waiting for the cluster to become healthy...`, - params - ); - - if (status.stats.esHealthChecks.unhealthyReasons[shouldWaitReason]) { - status.stats.esHealthChecks.unhealthyReasons[shouldWaitReason]++; - } else { - status.stats.esHealthChecks.unhealthyReasons[shouldWaitReason] = 1; + try { + const results = await waitUntilHealthy.wait({ + async onUnhealthy(params) { + const shouldWaitReason = params.waitingReason.name; + + logger.warn( + `Cluster is unhealthy (${shouldWaitReason}). Waiting for the cluster to become healthy...`, + params + ); + + if ( + status.stats.esHealthChecks.unhealthyReasons[shouldWaitReason] + ) { + status.stats.esHealthChecks.unhealthyReasons[ + shouldWaitReason + ]++; + } else { + status.stats.esHealthChecks.unhealthyReasons[ + shouldWaitReason + ] = 1; + } } - } - }); + }); - status.stats.esHealthChecks.checksCount++; - status.stats.esHealthChecks.timeSpentWaiting += results.runningTime; + status.stats.esHealthChecks.checksCount++; + status.stats.esHealthChecks.timeSpentWaiting += results.runningTime; - // Store data in DDB-ES DynamoDB table. - const executeDdbEs = () => { - return batchWriteAll({ - table: ddbEsEntryEntity.table, - items: ddbEsItemsToBatchWrite - }); - }; + // Store data in DDB-ES DynamoDB table. + const executeDdbEs = () => { + return batchWriteAll({ + table: ddbEsEntryEntity.table, + items: ddbEsItemsToBatchWrite + }); + }; - await executeWithRetry(executeDdbEs, { - onFailedAttempt: error => { - logger.warn( - `[DDB-ES Table] Batch write attempt #${error.attemptNumber} failed: ${error.message}` - ); - } - }); + await executeWithRetry(executeDdbEs, { + onFailedAttempt: error => { + logger.warn( + `[DDB-ES Table] Batch write attempt #${error.attemptNumber} failed: ${error.message}` + ); + } + }); + } catch (e) { + ddbEsWriteError = true; + logger.error( + { + error: e, + ddbEsItemsToBatchWrite + }, + "After multiple retries, failed to batch-store records in DDB-ES DynamoDB table." + ); + } } - status.stats.recordsUpdated += ddbItemsToBatchWrite.length; + if (ddbEsWriteError || ddbWriteError) { + logger.warn('Not increasing the "recordsUpdated" count due to write errors.'); + } else { + status.stats.recordsUpdated += ddbItemsToBatchWrite.length; + } } // Update checkpoint after every batch. From 420603d5d2814ef317d2205ba02d8c504db0c2e7 Mon Sep 17 00:00:00 2001 From: adrians5j Date: Wed, 14 Aug 2024 10:58:49 +0200 Subject: [PATCH 06/70] fix: reduce maxChunk to 18 --- .../migrations/src/migrations/5.39.6/001/ddb-es/worker.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/migrations/src/migrations/5.39.6/001/ddb-es/worker.ts b/packages/migrations/src/migrations/5.39.6/001/ddb-es/worker.ts index d7dfa4e39d3..90928eeaabe 100644 --- a/packages/migrations/src/migrations/5.39.6/001/ddb-es/worker.ts +++ b/packages/migrations/src/migrations/5.39.6/001/ddb-es/worker.ts @@ -361,7 +361,7 @@ const createInitialStatus = (): MigrationStatus => { return batchWriteAll({ table: ddbEntryEntity.table, items: ddbItemsToBatchWrite - }); + }, 18); }; logger.trace( @@ -424,7 +424,7 @@ const createInitialStatus = (): MigrationStatus => { return batchWriteAll({ table: ddbEsEntryEntity.table, items: ddbEsItemsToBatchWrite - }); + }, 18); }; await executeWithRetry(executeDdbEs, { From 4bdf4faec5e256ad5cdf4cbf7ad29047b6ecfc18 Mon Sep 17 00:00:00 2001 From: adrians5j Date: Wed, 14 Aug 2024 11:01:06 +0200 Subject: [PATCH 07/70] fix: reduce maxChunk to 18 --- .../migrations/5.39.6/001/ddb-es/worker.ts | 24 ++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/packages/migrations/src/migrations/5.39.6/001/ddb-es/worker.ts b/packages/migrations/src/migrations/5.39.6/001/ddb-es/worker.ts index 90928eeaabe..e6cefa0d90a 100644 --- a/packages/migrations/src/migrations/5.39.6/001/ddb-es/worker.ts +++ b/packages/migrations/src/migrations/5.39.6/001/ddb-es/worker.ts @@ -103,6 +103,8 @@ const createInitialStatus = (): MigrationStatus => { }; }; +const BATCH_WRITE_MAX_CHUNK = 18; + (async () => { const logger = createPinoLogger( { @@ -358,10 +360,13 @@ const createInitialStatus = (): MigrationStatus => { // Store data in primary DynamoDB table. const execute = () => { - return batchWriteAll({ - table: ddbEntryEntity.table, - items: ddbItemsToBatchWrite - }, 18); + return batchWriteAll( + { + table: ddbEntryEntity.table, + items: ddbItemsToBatchWrite + }, + BATCH_WRITE_MAX_CHUNK + ); }; logger.trace( @@ -421,10 +426,13 @@ const createInitialStatus = (): MigrationStatus => { // Store data in DDB-ES DynamoDB table. const executeDdbEs = () => { - return batchWriteAll({ - table: ddbEsEntryEntity.table, - items: ddbEsItemsToBatchWrite - }, 18); + return batchWriteAll( + { + table: ddbEsEntryEntity.table, + items: ddbEsItemsToBatchWrite + }, + BATCH_WRITE_MAX_CHUNK + ); }; await executeWithRetry(executeDdbEs, { From 1023b01368cad85a3d2f062c658643c6a9cad5fb Mon Sep 17 00:00:00 2001 From: adrians5j Date: Wed, 14 Aug 2024 11:09:02 +0200 Subject: [PATCH 08/70] fix: reduce maxChunk to 20 --- packages/migrations/src/migrations/5.39.6/001/ddb-es/worker.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/migrations/src/migrations/5.39.6/001/ddb-es/worker.ts b/packages/migrations/src/migrations/5.39.6/001/ddb-es/worker.ts index e6cefa0d90a..b56ebe9cb3f 100644 --- a/packages/migrations/src/migrations/5.39.6/001/ddb-es/worker.ts +++ b/packages/migrations/src/migrations/5.39.6/001/ddb-es/worker.ts @@ -103,7 +103,7 @@ const createInitialStatus = (): MigrationStatus => { }; }; -const BATCH_WRITE_MAX_CHUNK = 18; +const BATCH_WRITE_MAX_CHUNK = 20; (async () => { const logger = createPinoLogger( From ea9221b8891b62797e288a71eb2c0acc08c8db19 Mon Sep 17 00:00:00 2001 From: adrians5j Date: Wed, 14 Aug 2024 13:05:36 +0200 Subject: [PATCH 09/70] fix: wrap whole process into try/catch and ensure process doesn't fail --- .../migrations/5.39.6/001/ddb-es/worker.ts | 555 +++++++++--------- 1 file changed, 289 insertions(+), 266 deletions(-) diff --git a/packages/migrations/src/migrations/5.39.6/001/ddb-es/worker.ts b/packages/migrations/src/migrations/5.39.6/001/ddb-es/worker.ts index b56ebe9cb3f..530001adaad 100644 --- a/packages/migrations/src/migrations/5.39.6/001/ddb-es/worker.ts +++ b/packages/migrations/src/migrations/5.39.6/001/ddb-es/worker.ts @@ -142,351 +142,374 @@ const BATCH_WRITE_MAX_CHUNK = 20; waitingTimeStep: argv.esHealthWaitingTimeStep }); - await ddbScanWithCallback( - { - entity: ddbEntryEntity, - options: { - segment: argv.segmentIndex, - segments: argv.totalSegments, - filters: [ - { - attr: "_et", - eq: "CmsEntries" - } - ], - startKey: status.lastEvaluatedKey || undefined, - limit: 100 - } - }, - async result => { - status.stats.iterationsCount++; - status.stats.recordsScanned += result.items.length; - - if (status.stats.iterationsCount % 5 === 0) { - // We log every 5th iteration. - logger.trace( - `[iteration #${status.stats.iterationsCount}] Reading ${result.items.length} record(s)...` - ); - } - - const ddbItemsToBatchWrite: BatchWriteItem[] = []; - const ddbEsItemsToBatchWrite: BatchWriteItem[] = []; - const ddbEsItemsToBatchRead: Record = {}; - - const fallbackDateTime = new Date().toISOString(); - - // Update records in primary DynamoDB table. Also do preparations for - // subsequent updates on DDB-ES DynamoDB table, and in Elasticsearch. - for (const item of result.items) { - const isFullyMigrated = - isMigratedEntry(item) && - hasValidTypeFieldValue(item) && - hasAllNonNullableValues(item); - - if (isFullyMigrated) { - status.stats.recordsSkipped++; - continue; - } - - // 1. Check if the data migration was ever performed. If not, let's perform it. - if (!isMigratedEntry(item)) { - // Get the oldest revision's `createdOn` value. We use that to set the entry-level `createdOn` value. - const createdOn = await getOldestRevisionCreatedOn({ - entry: item, - entryEntity: ddbEntryEntity - }); - - const firstLastPublishedOnByFields = await getFirstLastPublishedOnBy({ - entry: item, - entryEntity: ddbEntryEntity - }); - - assignNewMetaFields(item, { - createdOn, - ...firstLastPublishedOnByFields - }); - } - - // 2. We've noticed some of the records had an invalid `TYPE` field value - // in the database. This step addresses this issue. - if (!hasValidTypeFieldValue(item)) { - // Fixes the value of the `TYPE` field, if it's not valid. - fixTypeFieldValue(item); + try { + await ddbScanWithCallback( + { + entity: ddbEntryEntity, + options: { + segment: argv.segmentIndex, + segments: argv.totalSegments, + filters: [ + { + attr: "_et", + eq: "CmsEntries" + } + ], + startKey: status.lastEvaluatedKey || undefined, + limit: 100 } + }, + async result => { + status.stats.iterationsCount++; + status.stats.recordsScanned += result.items.length; - // 3. Finally, once both of the steps were performed, ensure that all - // new non-nullable meta fields have a value and nothing is missing. - if (!hasAllNonNullableValues(item)) { + if (status.stats.iterationsCount % 5 === 0) { + // We log every 5th iteration. logger.trace( - getNonNullableFieldsWithMissingValues(item), - `Detected an entry with missing values for non-nullable meta fields (${item.modelId}/${item.id}).` + `[iteration #${status.stats.iterationsCount}] Reading ${result.items.length} record(s)...` ); - - try { - const fallbackIdentity = await getFallbackIdentity({ - entity: ddbEntryEntity, - tenant: item.tenant - }); - - ensureAllNonNullableValues(item, { - dateTime: fallbackDateTime, - identity: fallbackIdentity - }); - - logger.trace( - `Successfully ensured all non-nullable meta fields have values (${item.modelId}/${item.id}). Will be saving into the database soon.` - ); - } catch (e) { - logger.debug( - `Failed to ensure all non-nullable meta fields have values (${item.modelId}/${item.id}): ${e.message}` - ); - } } - ddbItemsToBatchWrite.push(ddbEntryEntity.putBatch(item)); - - /** - * Prepare the loading of DynamoDB Elasticsearch part of the records. - */ - - const ddbEsLatestRecordKey = `${item.entryId}:L`; - if (ddbEsItemsToBatchRead[ddbEsLatestRecordKey]) { - continue; - } + const ddbItemsToBatchWrite: BatchWriteItem[] = []; + const ddbEsItemsToBatchWrite: BatchWriteItem[] = []; + const ddbEsItemsToBatchRead: Record = {}; - ddbEsItemsToBatchRead[ddbEsLatestRecordKey] = ddbEsEntryEntity.getBatch({ - PK: item.PK, - SK: "L" - }); + const fallbackDateTime = new Date().toISOString(); - const ddbEsPublishedRecordKey = `${item.entryId}:P`; - if (item.status === "published" || !!item.locked) { - ddbEsItemsToBatchRead[ddbEsPublishedRecordKey] = ddbEsEntryEntity.getBatch({ - PK: item.PK, - SK: "P" - }); - } - } + // Update records in primary DynamoDB table. Also do preparations for + // subsequent updates on DDB-ES DynamoDB table, and in Elasticsearch. + for (const item of result.items) { + const isFullyMigrated = + isMigratedEntry(item) && + hasValidTypeFieldValue(item) && + hasAllNonNullableValues(item); - if (Object.keys(ddbEsItemsToBatchRead).length > 0) { - /** - * Get all the records from DynamoDB Elasticsearch. - */ - const ddbEsRecords = await batchReadAll({ - table: ddbEsEntryEntity.table, - items: Object.values(ddbEsItemsToBatchRead) - }); - - for (const ddbEsRecord of ddbEsRecords) { - const decompressedData = await getDecompressedData(ddbEsRecord.data); - if (!decompressedData) { - logger.trace( - `[DDB-ES Table] Skipping record "${ddbEsRecord.PK}" as it is not a valid CMS entry...` - ); + if (isFullyMigrated) { + status.stats.recordsSkipped++; continue; } // 1. Check if the data migration was ever performed. If not, let's perform it. - if (!isMigratedEntry(decompressedData)) { + if (!isMigratedEntry(item)) { // Get the oldest revision's `createdOn` value. We use that to set the entry-level `createdOn` value. const createdOn = await getOldestRevisionCreatedOn({ - entry: { ...decompressedData, PK: ddbEsRecord.PK }, + entry: item, entryEntity: ddbEntryEntity }); const firstLastPublishedOnByFields = await getFirstLastPublishedOnBy({ - entry: { ...decompressedData, PK: ddbEsRecord.PK }, + entry: item, entryEntity: ddbEntryEntity }); - assignNewMetaFields(decompressedData, { + assignNewMetaFields(item, { createdOn, ...firstLastPublishedOnByFields }); } - // 2. Ensure new non-nullable meta fields have a value and nothing is missing. - if (!hasAllNonNullableValues(decompressedData)) { + // 2. We've noticed some of the records had an invalid `TYPE` field value + // in the database. This step addresses this issue. + if (!hasValidTypeFieldValue(item)) { + // Fixes the value of the `TYPE` field, if it's not valid. + fixTypeFieldValue(item); + } + + // 3. Finally, once both of the steps were performed, ensure that all + // new non-nullable meta fields have a value and nothing is missing. + if (!hasAllNonNullableValues(item)) { logger.trace( - getNonNullableFieldsWithMissingValues(decompressedData), - [ - `[DDB-ES Table] Detected an entry with missing values for non-nullable meta fields`, - `(${decompressedData.modelId}/${decompressedData.id}).` - ].join(" ") + getNonNullableFieldsWithMissingValues(item), + `Detected an entry with missing values for non-nullable meta fields (${item.modelId}/${item.id}).` ); try { const fallbackIdentity = await getFallbackIdentity({ entity: ddbEntryEntity, - tenant: decompressedData.tenant + tenant: item.tenant }); - ensureAllNonNullableValues(decompressedData, { + ensureAllNonNullableValues(item, { dateTime: fallbackDateTime, identity: fallbackIdentity }); logger.trace( - [ - `[DDB-ES Table] Successfully ensured all non-nullable meta fields`, - `have values (${decompressedData.modelId}/${decompressedData.id}).`, - "Will be saving the changes soon." - ].join(" ") + `Successfully ensured all non-nullable meta fields have values (${item.modelId}/${item.id}). Will be saving into the database soon.` ); } catch (e) { - logger.error( - [ - "[DDB-ES Table] Failed to ensure all non-nullable meta fields have values", - `(${decompressedData.modelId}/${decompressedData.id}): ${e.message}` - ].join(" ") + logger.debug( + `Failed to ensure all non-nullable meta fields have values (${item.modelId}/${item.id}): ${e.message}` ); } } - const compressedData = await getCompressedData(decompressedData); + ddbItemsToBatchWrite.push(ddbEntryEntity.putBatch(item)); - ddbEsItemsToBatchWrite.push( - ddbEsEntryEntity.putBatch({ - ...ddbEsRecord, - data: compressedData - }) - ); - } - } + /** + * Prepare the loading of DynamoDB Elasticsearch part of the records. + */ - if (ddbItemsToBatchWrite.length) { - let ddbWriteError = false; - let ddbEsWriteError = false; + const ddbEsLatestRecordKey = `${item.entryId}:L`; + if (ddbEsItemsToBatchRead[ddbEsLatestRecordKey]) { + continue; + } - // Store data in primary DynamoDB table. - const execute = () => { - return batchWriteAll( - { - table: ddbEntryEntity.table, - items: ddbItemsToBatchWrite - }, - BATCH_WRITE_MAX_CHUNK - ); - }; + ddbEsItemsToBatchRead[ddbEsLatestRecordKey] = ddbEsEntryEntity.getBatch({ + PK: item.PK, + SK: "L" + }); + + const ddbEsPublishedRecordKey = `${item.entryId}:P`; + if (item.status === "published" || !!item.locked) { + ddbEsItemsToBatchRead[ddbEsPublishedRecordKey] = ddbEsEntryEntity.getBatch({ + PK: item.PK, + SK: "P" + }); + } + } - logger.trace( - `Storing ${ddbItemsToBatchWrite.length} record(s) in primary DynamoDB table...` - ); + if (Object.keys(ddbEsItemsToBatchRead).length > 0) { + /** + * Get all the records from DynamoDB Elasticsearch. + */ + const ddbEsRecords = await batchReadAll({ + table: ddbEsEntryEntity.table, + items: Object.values(ddbEsItemsToBatchRead) + }); - try { - await executeWithRetry(execute, { - onFailedAttempt: error => { - logger.warn( - `Batch write attempt #${error.attemptNumber} failed: ${error.message}` + for (const ddbEsRecord of ddbEsRecords) { + const decompressedData = await getDecompressedData( + ddbEsRecord.data + ); + if (!decompressedData) { + logger.trace( + `[DDB-ES Table] Skipping record "${ddbEsRecord.PK}" as it is not a valid CMS entry...` ); + continue; } - }); - } catch (e) { - ddbWriteError = true; - logger.error( - { - error: e, - ddbItemsToBatchWrite - }, - "After multiple retries, failed to batch-store records in primary DynamoDB table." - ); - } - if (ddbEsItemsToBatchWrite.length) { - logger.trace( - `Storing ${ddbEsItemsToBatchWrite.length} record(s) in DDB-ES DynamoDB table...` - ); + // 1. Check if the data migration was ever performed. If not, let's perform it. + if (!isMigratedEntry(decompressedData)) { + // Get the oldest revision's `createdOn` value. We use that to set the entry-level `createdOn` value. + const createdOn = await getOldestRevisionCreatedOn({ + entry: { ...decompressedData, PK: ddbEsRecord.PK }, + entryEntity: ddbEntryEntity + }); - try { - const results = await waitUntilHealthy.wait({ - async onUnhealthy(params) { - const shouldWaitReason = params.waitingReason.name; + const firstLastPublishedOnByFields = await getFirstLastPublishedOnBy({ + entry: { ...decompressedData, PK: ddbEsRecord.PK }, + entryEntity: ddbEntryEntity + }); - logger.warn( - `Cluster is unhealthy (${shouldWaitReason}). Waiting for the cluster to become healthy...`, - params - ); + assignNewMetaFields(decompressedData, { + createdOn, + ...firstLastPublishedOnByFields + }); + } - if ( - status.stats.esHealthChecks.unhealthyReasons[shouldWaitReason] - ) { - status.stats.esHealthChecks.unhealthyReasons[ - shouldWaitReason - ]++; - } else { - status.stats.esHealthChecks.unhealthyReasons[ - shouldWaitReason - ] = 1; - } + // 2. Ensure new non-nullable meta fields have a value and nothing is missing. + if (!hasAllNonNullableValues(decompressedData)) { + logger.trace( + getNonNullableFieldsWithMissingValues(decompressedData), + [ + `[DDB-ES Table] Detected an entry with missing values for non-nullable meta fields`, + `(${decompressedData.modelId}/${decompressedData.id}).` + ].join(" ") + ); + + try { + const fallbackIdentity = await getFallbackIdentity({ + entity: ddbEntryEntity, + tenant: decompressedData.tenant + }); + + ensureAllNonNullableValues(decompressedData, { + dateTime: fallbackDateTime, + identity: fallbackIdentity + }); + + logger.trace( + [ + `[DDB-ES Table] Successfully ensured all non-nullable meta fields`, + `have values (${decompressedData.modelId}/${decompressedData.id}).`, + "Will be saving the changes soon." + ].join(" ") + ); + } catch (e) { + logger.error( + [ + "[DDB-ES Table] Failed to ensure all non-nullable meta fields have values", + `(${decompressedData.modelId}/${decompressedData.id}): ${e.message}` + ].join(" ") + ); } - }); + } - status.stats.esHealthChecks.checksCount++; - status.stats.esHealthChecks.timeSpentWaiting += results.runningTime; + const compressedData = await getCompressedData(decompressedData); - // Store data in DDB-ES DynamoDB table. - const executeDdbEs = () => { - return batchWriteAll( - { - table: ddbEsEntryEntity.table, - items: ddbEsItemsToBatchWrite - }, - BATCH_WRITE_MAX_CHUNK - ); - }; + ddbEsItemsToBatchWrite.push( + ddbEsEntryEntity.putBatch({ + ...ddbEsRecord, + data: compressedData + }) + ); + } + } + + if (ddbItemsToBatchWrite.length) { + let ddbWriteError = false; + let ddbEsWriteError = false; + + // Store data in primary DynamoDB table. + const execute = () => { + return batchWriteAll( + { + table: ddbEntryEntity.table, + items: ddbItemsToBatchWrite + }, + BATCH_WRITE_MAX_CHUNK + ); + }; - await executeWithRetry(executeDdbEs, { + logger.trace( + `Storing ${ddbItemsToBatchWrite.length} record(s) in primary DynamoDB table...` + ); + + try { + await executeWithRetry(execute, { onFailedAttempt: error => { logger.warn( - `[DDB-ES Table] Batch write attempt #${error.attemptNumber} failed: ${error.message}` + `Batch write attempt #${error.attemptNumber} failed: ${error.message}` ); } }); } catch (e) { - ddbEsWriteError = true; + ddbWriteError = true; logger.error( { error: e, - ddbEsItemsToBatchWrite + ddbItemsToBatchWrite }, - "After multiple retries, failed to batch-store records in DDB-ES DynamoDB table." + "After multiple retries, failed to batch-store records in primary DynamoDB table." ); } - } - if (ddbEsWriteError || ddbWriteError) { - logger.warn('Not increasing the "recordsUpdated" count due to write errors.'); - } else { - status.stats.recordsUpdated += ddbItemsToBatchWrite.length; - } - } + if (ddbEsItemsToBatchWrite.length) { + logger.trace( + `Storing ${ddbEsItemsToBatchWrite.length} record(s) in DDB-ES DynamoDB table...` + ); - // Update checkpoint after every batch. - let lastEvaluatedKey: LastEvaluatedKey = true; - if (result.lastEvaluatedKey) { - lastEvaluatedKey = result.lastEvaluatedKey as unknown as LastEvaluatedKeyObject; - } + try { + const results = await waitUntilHealthy.wait({ + async onUnhealthy(params) { + const shouldWaitReason = params.waitingReason.name; + + logger.warn( + `Cluster is unhealthy (${shouldWaitReason}). Waiting for the cluster to become healthy...`, + params + ); + + if ( + status.stats.esHealthChecks.unhealthyReasons[ + shouldWaitReason + ] + ) { + status.stats.esHealthChecks.unhealthyReasons[ + shouldWaitReason + ]++; + } else { + status.stats.esHealthChecks.unhealthyReasons[ + shouldWaitReason + ] = 1; + } + } + }); - status.lastEvaluatedKey = lastEvaluatedKey; + status.stats.esHealthChecks.checksCount++; + status.stats.esHealthChecks.timeSpentWaiting += results.runningTime; + + // Store data in DDB-ES DynamoDB table. + const executeDdbEs = () => { + return batchWriteAll( + { + table: ddbEsEntryEntity.table, + items: ddbEsItemsToBatchWrite + }, + BATCH_WRITE_MAX_CHUNK + ); + }; - if (lastEvaluatedKey === true) { - return false; - } + await executeWithRetry(executeDdbEs, { + onFailedAttempt: error => { + logger.warn( + `[DDB-ES Table] Batch write attempt #${error.attemptNumber} failed: ${error.message}` + ); + } + }); + } catch (e) { + ddbEsWriteError = true; + logger.error( + { + error: e, + ddbEsItemsToBatchWrite + }, + "After multiple retries, failed to batch-store records in DDB-ES DynamoDB table." + ); + } + } - // Continue further scanning. - return true; - } - ); + if (ddbEsWriteError || ddbWriteError) { + logger.warn( + 'Not increasing the "recordsUpdated" count due to write errors.' + ); + } else { + status.stats.recordsUpdated += ddbItemsToBatchWrite.length; + } + } - // Store status in tmp file. - logger.trace({ status }, "Segment processing completed. Saving status to tmp file..."); - const logFilePath = path.join( - os.tmpdir(), - `webiny-5-39-6-meta-fields-data-migration-log-${argv.runId}-${argv.segmentIndex}.log` - ); + // Update checkpoint after every batch. + let lastEvaluatedKey: LastEvaluatedKey = true; + if (result.lastEvaluatedKey) { + lastEvaluatedKey = result.lastEvaluatedKey as unknown as LastEvaluatedKeyObject; + } + + status.lastEvaluatedKey = lastEvaluatedKey; - // Save segment processing stats to a file. - fs.writeFileSync(logFilePath, JSON.stringify(status.stats, null, 2)); + if (lastEvaluatedKey === true) { + return false; + } - logger.trace(`Segment processing stats saved in ${logFilePath}.`); + // Continue further scanning. + return true; + } + ); + + // Store status in tmp file. + logger.trace({ status }, "Segment processing completed. Saving status to tmp file..."); + const logFilePath = path.join( + os.tmpdir(), + `webiny-5-39-6-meta-fields-data-migration-log-${argv.runId}-${argv.segmentIndex}.log` + ); + + // Save segment processing stats to a file. + fs.writeFileSync(logFilePath, JSON.stringify(status.stats, null, 2)); + + logger.trace(`Segment processing stats saved in ${logFilePath}.`); + } catch (error) { + // Store status in tmp file. + logger.error( + { status, error }, + "Segment processing failed to complete. Saving status to tmp file..." + ); + const logFilePath = path.join( + os.tmpdir(), + `webiny-5-39-6-meta-fields-data-migration-log-${argv.runId}-${argv.segmentIndex}.log` + ); + + // Save segment processing stats to a file. + fs.writeFileSync(logFilePath, JSON.stringify(status.stats, null, 2)); + + logger.trace(`Segment processing stats saved in ${logFilePath}.`); + } })(); From e55772784016d25e6c994c2b1286e730596015fe Mon Sep 17 00:00:00 2001 From: adrians5j Date: Fri, 16 Aug 2024 13:40:30 +0200 Subject: [PATCH 10/70] fix: retry read ops --- .../5.39.0/001/utils/getFallbackIdentity.ts | 23 ++++++++++++------- .../001/utils/getFirstLastPublishedOn.ts | 16 +++++++++---- .../001/utils/getOldestRevisionCreatedOn.ts | 16 +++++++++---- 3 files changed, 37 insertions(+), 18 deletions(-) diff --git a/packages/migrations/src/migrations/5.39.0/001/utils/getFallbackIdentity.ts b/packages/migrations/src/migrations/5.39.0/001/utils/getFallbackIdentity.ts index a06f016a298..f7ef6318ebe 100644 --- a/packages/migrations/src/migrations/5.39.0/001/utils/getFallbackIdentity.ts +++ b/packages/migrations/src/migrations/5.39.0/001/utils/getFallbackIdentity.ts @@ -1,6 +1,7 @@ import { CmsIdentity } from "@webiny/api-headless-cms/types"; import { queryAll } from "@webiny/db-dynamodb"; import { Entity } from "@webiny/db-dynamodb/toolbox"; +import { executeWithRetry, ExecuteWithRetryOptions } from "@webiny/utils"; const NON_EXISTING_DATA_MIGRATION_IDENTITY: CmsIdentity = { id: "data-migration", @@ -11,6 +12,7 @@ const NON_EXISTING_DATA_MIGRATION_IDENTITY: CmsIdentity = { interface GetFallbackIdentityParams { entity: Entity; tenant: string; + retryOptions?: ExecuteWithRetryOptions; } interface AdminUserRecord { @@ -25,19 +27,24 @@ const identitiesPerTenantCache: Record = {}; export const getFallbackIdentity = async ({ entity, - tenant + tenant, + retryOptions }: GetFallbackIdentityParams): Promise => { if (identitiesPerTenantCache[tenant]) { return identitiesPerTenantCache[tenant]; } - const allAdminUsersRecords = await queryAll({ - entity, - partitionKey: `T#${tenant}#ADMIN_USERS`, - options: { - index: "GSI1" - } - }); + const executeQueryAll = () => { + return queryAll({ + entity, + partitionKey: `T#${tenant}#ADMIN_USERS`, + options: { + index: "GSI1" + } + }); + }; + + const allAdminUsersRecords = await executeWithRetry(executeQueryAll, retryOptions); if (allAdminUsersRecords.length === 0) { // Hopefully it doesn't come to this, but we still need to consider it. diff --git a/packages/migrations/src/migrations/5.39.0/001/utils/getFirstLastPublishedOn.ts b/packages/migrations/src/migrations/5.39.0/001/utils/getFirstLastPublishedOn.ts index 6125aaf5221..bc7129cf3bf 100644 --- a/packages/migrations/src/migrations/5.39.0/001/utils/getFirstLastPublishedOn.ts +++ b/packages/migrations/src/migrations/5.39.0/001/utils/getFirstLastPublishedOn.ts @@ -1,5 +1,6 @@ import { createDdbEntryEntity } from "./../entities/createEntryEntity"; import { CmsEntry } from "../types"; +import { executeWithRetry, ExecuteWithRetryOptions } from "@webiny/utils"; const cachedEntryFirstLastPublishedOnBy: Record< string, @@ -13,6 +14,7 @@ interface CmsEntryWithPK extends CmsEntry { export interface getFirstLastPublishedOnParams { entry: CmsEntryWithPK; entryEntity: ReturnType; + retryOptions?: ExecuteWithRetryOptions; } export const getFirstLastPublishedOnBy = async (params: getFirstLastPublishedOnParams) => { @@ -29,11 +31,15 @@ export const getFirstLastPublishedOnBy = async (params: getFirstLastPublishedOnP lastPublishedBy: null }; - const result = await entryEntity.query(entry.PK, { - limit: 1, - eq: "P", - attributes: ["modifiedBy", "createdBy", "publishedOn"] - }); + const executeQuery = () => { + return entryEntity.query(entry.PK, { + limit: 1, + eq: "P", + attributes: ["modifiedBy", "createdBy", "publishedOn"] + }); + }; + + const result = await executeWithRetry(executeQuery, params.retryOptions); const publishedRecord = result.Items?.[0]; if (publishedRecord) { diff --git a/packages/migrations/src/migrations/5.39.0/001/utils/getOldestRevisionCreatedOn.ts b/packages/migrations/src/migrations/5.39.0/001/utils/getOldestRevisionCreatedOn.ts index 7214c2b1cbd..8a930406063 100644 --- a/packages/migrations/src/migrations/5.39.0/001/utils/getOldestRevisionCreatedOn.ts +++ b/packages/migrations/src/migrations/5.39.0/001/utils/getOldestRevisionCreatedOn.ts @@ -1,5 +1,6 @@ import { createDdbEntryEntity } from "./../entities/createEntryEntity"; import { CmsEntry } from "../types"; +import { executeWithRetry, ExecuteWithRetryOptions } from "@webiny/utils"; const cachedEntryCreatedOn: Record = {}; @@ -10,6 +11,7 @@ interface CmsEntryWithPK extends CmsEntry { export interface GetOldestRevisionCreatedOnParams { entry: CmsEntryWithPK; entryEntity: ReturnType; + retryOptions?: ExecuteWithRetryOptions; } export const getOldestRevisionCreatedOn = async (params: GetOldestRevisionCreatedOnParams) => { @@ -22,11 +24,15 @@ export const getOldestRevisionCreatedOn = async (params: GetOldestRevisionCreate if (entry.version === 1) { cachedEntryCreatedOn[entry.PK] = entry.createdOn; } else { - const result = await entryEntity.query(entry.PK, { - limit: 1, - beginsWith: "REV#", - attributes: ["createdOn"] - }); + const executeQuery = () => { + return entryEntity.query(entry.PK, { + limit: 1, + beginsWith: "REV#", + attributes: ["createdOn"] + }); + }; + + const result = await executeWithRetry(executeQuery, params.retryOptions); const oldestRevision = result.Items?.[0]; if (oldestRevision) { From 4ed822c61e2a262a0fc34d94ffb5c215e3e92934 Mon Sep 17 00:00:00 2001 From: adrians5j Date: Fri, 16 Aug 2024 13:41:14 +0200 Subject: [PATCH 11/70] fix: export `ExecuteWithRetryOptions` --- packages/utils/src/executeWithRetry.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/utils/src/executeWithRetry.ts b/packages/utils/src/executeWithRetry.ts index 42eba2d08bc..d6bfa8283e5 100644 --- a/packages/utils/src/executeWithRetry.ts +++ b/packages/utils/src/executeWithRetry.ts @@ -1,8 +1,10 @@ import pRetry from "p-retry"; +export type ExecuteWithRetryOptions = Parameters[1]; + export const executeWithRetry = ( execute: () => Promise, - options?: Parameters[1] + options?: ExecuteWithRetryOptions ) => { const retries = 20; return pRetry(execute, { From d95d4e641331b0d6e860278021266ae357c44905 Mon Sep 17 00:00:00 2001 From: adrians5j Date: Fri, 16 Aug 2024 13:56:06 +0200 Subject: [PATCH 12/70] fix: allow scanning with retries --- packages/db-dynamodb/package.json | 1 + packages/db-dynamodb/src/utils/scan.ts | 33 +++++++++++++++++++++--- packages/db-dynamodb/tsconfig.build.json | 3 ++- packages/db-dynamodb/tsconfig.json | 7 +++-- 4 files changed, 38 insertions(+), 6 deletions(-) diff --git a/packages/db-dynamodb/package.json b/packages/db-dynamodb/package.json index b944fc89a51..e06c8ff6e38 100644 --- a/packages/db-dynamodb/package.json +++ b/packages/db-dynamodb/package.json @@ -16,6 +16,7 @@ "@webiny/error": "0.0.0", "@webiny/handler-db": "0.0.0", "@webiny/plugins": "0.0.0", + "@webiny/utils": "0.0.0", "date-fns": "^2.22.1", "dot-prop": "^6.0.1", "dynamodb-toolbox": "^0.9.2", diff --git a/packages/db-dynamodb/src/utils/scan.ts b/packages/db-dynamodb/src/utils/scan.ts index d11df0a1db8..6feac2268f1 100644 --- a/packages/db-dynamodb/src/utils/scan.ts +++ b/packages/db-dynamodb/src/utils/scan.ts @@ -1,5 +1,6 @@ import { ScanInput, ScanOutput } from "@webiny/aws-sdk/client-dynamodb"; import { Entity, ScanOptions, Table } from "~/toolbox"; +import { executeWithRetry, ExecuteWithRetryOptions } from "@webiny/utils"; export type { ScanOptions }; @@ -93,11 +94,29 @@ export const scan = async (params: ScanParams): Promise> => { return convertResult(result) as ScanResponse; }; +interface ScanWithCallbackOptions { + retry?: true | ExecuteWithRetryOptions; +} + export const scanWithCallback = async ( params: ScanParams, - callback: (result: ScanResponse>) => Promise + callback: (result: ScanResponse>) => Promise, + options?: ScanWithCallbackOptions ): Promise => { - let result = await scan>(params); + // For backwards compatibility, we still allow for executing the scan without retries. + const usingRetry = Boolean(options?.retry); + const retryOptions = options?.retry === true ? {} : options?.retry; + + const executeScan = () => scan>(params); + const getInitialResult = () => { + if (usingRetry) { + return executeWithRetry(executeScan, retryOptions); + } + return executeScan(); + }; + + let result = await getInitialResult(); + if (!result.items?.length && !result.lastEvaluatedKey) { return; } @@ -111,7 +130,15 @@ export const scanWithCallback = async ( } while (result.next) { - result = await result.next(); + const executeNext = () => result.next!(); + const getNextResult = () => { + if (usingRetry) { + return executeWithRetry(executeNext, retryOptions); + } + return executeNext(); + }; + + result = await getNextResult(); // If the result of the callback was `false`, that means the // user's intention was to stop further table scanning. diff --git a/packages/db-dynamodb/tsconfig.build.json b/packages/db-dynamodb/tsconfig.build.json index 92d1a331dd8..d14ec0b973f 100644 --- a/packages/db-dynamodb/tsconfig.build.json +++ b/packages/db-dynamodb/tsconfig.build.json @@ -7,7 +7,8 @@ { "path": "../db/tsconfig.build.json" }, { "path": "../error/tsconfig.build.json" }, { "path": "../handler-db/tsconfig.build.json" }, - { "path": "../plugins/tsconfig.build.json" } + { "path": "../plugins/tsconfig.build.json" }, + { "path": "../utils/tsconfig.build.json" } ], "compilerOptions": { "rootDir": "./src", diff --git a/packages/db-dynamodb/tsconfig.json b/packages/db-dynamodb/tsconfig.json index fe73ebd8a43..13fd84d1241 100644 --- a/packages/db-dynamodb/tsconfig.json +++ b/packages/db-dynamodb/tsconfig.json @@ -7,7 +7,8 @@ { "path": "../db" }, { "path": "../error" }, { "path": "../handler-db" }, - { "path": "../plugins" } + { "path": "../plugins" }, + { "path": "../utils" } ], "compilerOptions": { "rootDirs": ["./src", "./__tests__"], @@ -27,7 +28,9 @@ "@webiny/handler-db/*": ["../handler-db/src/*"], "@webiny/handler-db": ["../handler-db/src"], "@webiny/plugins/*": ["../plugins/src/*"], - "@webiny/plugins": ["../plugins/src"] + "@webiny/plugins": ["../plugins/src"], + "@webiny/utils/*": ["../utils/src/*"], + "@webiny/utils": ["../utils/src"] }, "baseUrl": "." } From 307efff8b6963b1af24a1b81b833bc3f28f6900d Mon Sep 17 00:00:00 2001 From: adrians5j Date: Fri, 16 Aug 2024 13:57:10 +0200 Subject: [PATCH 13/70] fix: add retries to read ops --- .../migrations/5.39.6/001/ddb-es/worker.ts | 101 ++++++++++++++++-- 1 file changed, 91 insertions(+), 10 deletions(-) diff --git a/packages/migrations/src/migrations/5.39.6/001/ddb-es/worker.ts b/packages/migrations/src/migrations/5.39.6/001/ddb-es/worker.ts index 530001adaad..408ef414b62 100644 --- a/packages/migrations/src/migrations/5.39.6/001/ddb-es/worker.ts +++ b/packages/migrations/src/migrations/5.39.6/001/ddb-es/worker.ts @@ -103,7 +103,10 @@ const createInitialStatus = (): MigrationStatus => { }; }; -const BATCH_WRITE_MAX_CHUNK = 20; +let BATCH_WRITE_MAX_CHUNK = 20; +if (process.env.WEBINY_MIGRATION_5_39_6_001_BATCH_WRITE_MAX_CHUNK) { + BATCH_WRITE_MAX_CHUNK = parseInt(process.env.WEBINY_MIGRATION_5_39_6_001_BATCH_WRITE_MAX_CHUNK); +} (async () => { const logger = createPinoLogger( @@ -194,12 +197,28 @@ const BATCH_WRITE_MAX_CHUNK = 20; // Get the oldest revision's `createdOn` value. We use that to set the entry-level `createdOn` value. const createdOn = await getOldestRevisionCreatedOn({ entry: item, - entryEntity: ddbEntryEntity + entryEntity: ddbEntryEntity, + retryOptions: { + onFailedAttempt: error => { + logger.warn( + { error, item }, + `getOldestRevisionCreatedOn attempt #${error.attemptNumber} failed: ${error.message}` + ); + } + } }); const firstLastPublishedOnByFields = await getFirstLastPublishedOnBy({ entry: item, - entryEntity: ddbEntryEntity + entryEntity: ddbEntryEntity, + retryOptions: { + onFailedAttempt: error => { + logger.warn( + { error, item }, + `getFirstLastPublishedOnBy attempt #${error.attemptNumber} failed: ${error.message}` + ); + } + } }); assignNewMetaFields(item, { @@ -226,7 +245,15 @@ const BATCH_WRITE_MAX_CHUNK = 20; try { const fallbackIdentity = await getFallbackIdentity({ entity: ddbEntryEntity, - tenant: item.tenant + tenant: item.tenant, + retryOptions: { + onFailedAttempt: error => { + logger.warn( + { error, item }, + `getFallbackIdentity attempt #${error.attemptNumber} failed: ${error.message}` + ); + } + } }); ensureAllNonNullableValues(item, { @@ -273,9 +300,20 @@ const BATCH_WRITE_MAX_CHUNK = 20; /** * Get all the records from DynamoDB Elasticsearch. */ - const ddbEsRecords = await batchReadAll({ - table: ddbEsEntryEntity.table, - items: Object.values(ddbEsItemsToBatchRead) + const executeBatchReadAll = () => { + return batchReadAll({ + table: ddbEsEntryEntity.table, + items: Object.values(ddbEsItemsToBatchRead) + }); + }; + + const ddbEsRecords = await executeWithRetry(executeBatchReadAll, { + onFailedAttempt: error => { + logger.warn( + { error, items: Object.values(ddbEsItemsToBatchRead) }, + `[DDB-ES Table] Batch (ddbEsItemsToBatchRead) read attempt #${error.attemptNumber} failed: ${error.message}` + ); + } }); for (const ddbEsRecord of ddbEsRecords) { @@ -294,12 +332,34 @@ const BATCH_WRITE_MAX_CHUNK = 20; // Get the oldest revision's `createdOn` value. We use that to set the entry-level `createdOn` value. const createdOn = await getOldestRevisionCreatedOn({ entry: { ...decompressedData, PK: ddbEsRecord.PK }, - entryEntity: ddbEntryEntity + entryEntity: ddbEntryEntity, + retryOptions: { + onFailedAttempt: error => { + logger.warn( + { + error, + item: { ...decompressedData, PK: ddbEsRecord.PK } + }, + `[DDB-ES Table] getOldestRevisionCreatedOn attempt #${error.attemptNumber} failed: ${error.message}` + ); + } + } }); const firstLastPublishedOnByFields = await getFirstLastPublishedOnBy({ entry: { ...decompressedData, PK: ddbEsRecord.PK }, - entryEntity: ddbEntryEntity + entryEntity: ddbEntryEntity, + retryOptions: { + onFailedAttempt: error => { + logger.warn( + { + error, + item: { ...decompressedData, PK: ddbEsRecord.PK } + }, + `[DDB-ES Table] getFirstLastPublishedOnBy attempt #${error.attemptNumber} failed: ${error.message}` + ); + } + } }); assignNewMetaFields(decompressedData, { @@ -321,7 +381,15 @@ const BATCH_WRITE_MAX_CHUNK = 20; try { const fallbackIdentity = await getFallbackIdentity({ entity: ddbEntryEntity, - tenant: decompressedData.tenant + tenant: decompressedData.tenant, + retryOptions: { + onFailedAttempt: error => { + logger.warn( + { error, item: ddbEntryEntity }, + `[DDB-ES Table] getFallbackIdentity attempt #${error.attemptNumber} failed: ${error.message}` + ); + } + } }); ensureAllNonNullableValues(decompressedData, { @@ -482,6 +550,19 @@ const BATCH_WRITE_MAX_CHUNK = 20; // Continue further scanning. return true; + }, + { + retry: { + onFailedAttempt: error => { + logger.warn( + { + lastEvaluatedKey: status.lastEvaluatedKey, + error + }, + `ddbScanWithCallback attempt #${error.attemptNumber} failed: ${error.message}` + ); + } + } } ); From 641d166c7558f7d86c0aefa46127e0b67c679512 Mon Sep 17 00:00:00 2001 From: adrians5j Date: Fri, 16 Aug 2024 14:03:09 +0200 Subject: [PATCH 14/70] fix: update dependencies --- yarn.lock | 1 + 1 file changed, 1 insertion(+) diff --git a/yarn.lock b/yarn.lock index fef1ae8fe5b..2abf5e95bef 100644 --- a/yarn.lock +++ b/yarn.lock @@ -18327,6 +18327,7 @@ __metadata: "@webiny/handler-db": 0.0.0 "@webiny/plugins": 0.0.0 "@webiny/project-utils": 0.0.0 + "@webiny/utils": 0.0.0 date-fns: ^2.22.1 dot-prop: ^6.0.1 dynamodb-toolbox: ^0.9.2 From 31f95d2b5e82d74b378480bcf6b7bf3320857949 Mon Sep 17 00:00:00 2001 From: Pavel Denisjuk Date: Tue, 20 Aug 2024 13:52:46 +0200 Subject: [PATCH 15/70] fix(app-file-manager): revert to using nested forms for extensions --- .../FileDetails/components/Extensions.tsx | 35 ++++++++++--------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/packages/app-file-manager/src/components/FileDetails/components/Extensions.tsx b/packages/app-file-manager/src/components/FileDetails/components/Extensions.tsx index a8439b78295..54801f37d09 100644 --- a/packages/app-file-manager/src/components/FileDetails/components/Extensions.tsx +++ b/packages/app-file-manager/src/components/FileDetails/components/Extensions.tsx @@ -4,7 +4,7 @@ import { CompositionScope } from "@webiny/app-admin"; import { CmsModel } from "@webiny/app-headless-cms/types"; import { ModelProvider } from "@webiny/app-headless-cms/admin/components/ModelProvider"; import { Fields } from "@webiny/app-headless-cms/admin/components/ContentEntryForm/Fields"; -import { Bind, BindComponentProps } from "@webiny/form"; +import { Bind, Form, useBind } from "@webiny/form"; const HideEmptyCells = styled.div` .mdc-layout-grid__cell:empty { @@ -16,13 +16,6 @@ interface ExtensionsProps { model: CmsModel; } -function BindWithPrefix(props: BindComponentProps) { - return ( - - {props.children} - - ); -} export const Extensions = ({ model }: ExtensionsProps) => { const extensionsField = useMemo(() => { return model.fields.find(f => f.fieldId === "extensions"); @@ -39,18 +32,26 @@ export const Extensions = ({ model }: ExtensionsProps) => { layout.push(...fields.map(field => [field.fieldId])); } + const { value, onChange } = useBind({ + name: "extensions" + }); + return ( - - - +
+ {() => ( + + + + )} +
); From 7bb3196af2304ad35d5a1df8d2fbd781ef6b2966 Mon Sep 17 00:00:00 2001 From: Pavel Denisjuk Date: Wed, 21 Aug 2024 12:47:19 +0200 Subject: [PATCH 16/70] fix(app-file-manager): use bind prefix context to render extensions --- .../FileDetails/components/Extensions.tsx | 29 ++++++++----------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/packages/app-file-manager/src/components/FileDetails/components/Extensions.tsx b/packages/app-file-manager/src/components/FileDetails/components/Extensions.tsx index a8439b78295..ca5b35e3626 100644 --- a/packages/app-file-manager/src/components/FileDetails/components/Extensions.tsx +++ b/packages/app-file-manager/src/components/FileDetails/components/Extensions.tsx @@ -4,7 +4,7 @@ import { CompositionScope } from "@webiny/app-admin"; import { CmsModel } from "@webiny/app-headless-cms/types"; import { ModelProvider } from "@webiny/app-headless-cms/admin/components/ModelProvider"; import { Fields } from "@webiny/app-headless-cms/admin/components/ContentEntryForm/Fields"; -import { Bind, BindComponentProps } from "@webiny/form"; +import { Bind, BindPrefix } from "@webiny/form"; const HideEmptyCells = styled.div` .mdc-layout-grid__cell:empty { @@ -16,13 +16,6 @@ interface ExtensionsProps { model: CmsModel; } -function BindWithPrefix(props: BindComponentProps) { - return ( - - {props.children} - - ); -} export const Extensions = ({ model }: ExtensionsProps) => { const extensionsField = useMemo(() => { return model.fields.find(f => f.fieldId === "extensions"); @@ -42,15 +35,17 @@ export const Extensions = ({ model }: ExtensionsProps) => { return ( - - - + + + + + ); From 77e49d3c546915e343aa766f84ee7edd1a7f5f04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20Zori=C4=87?= Date: Thu, 22 Aug 2024 11:59:10 +0200 Subject: [PATCH 17/70] fix(api-headless-cms): generate model schema [skip ci] (#4232) --- .../graphql/schema/createManageResolvers.ts | 7 -- .../src/graphql/schema/createManageSDL.ts | 4 +- .../graphql/schema/createPreviewResolvers.ts | 6 - .../src/graphql/schema/createReadResolvers.ts | 6 - .../src/graphql/schema/createReadSDL.ts | 5 +- .../src/graphql/schema/schemaPlugins.ts | 112 +++++++++--------- 6 files changed, 56 insertions(+), 84 deletions(-) diff --git a/packages/api-headless-cms/src/graphql/schema/createManageResolvers.ts b/packages/api-headless-cms/src/graphql/schema/createManageResolvers.ts index 9a603a17f3b..882f27e72ea 100644 --- a/packages/api-headless-cms/src/graphql/schema/createManageResolvers.ts +++ b/packages/api-headless-cms/src/graphql/schema/createManageResolvers.ts @@ -38,13 +38,6 @@ export const createManageResolvers: CreateManageResolvers = ({ model, fieldTypePlugins }) => { - if (model.fields.length === 0) { - return { - Query: {}, - Mutation: {} - }; - } - const createFieldResolvers = createFieldResolversFactory({ endpointType: "manage", models, diff --git a/packages/api-headless-cms/src/graphql/schema/createManageSDL.ts b/packages/api-headless-cms/src/graphql/schema/createManageSDL.ts index 3e36da3db22..e08d847b527 100644 --- a/packages/api-headless-cms/src/graphql/schema/createManageSDL.ts +++ b/packages/api-headless-cms/src/graphql/schema/createManageSDL.ts @@ -30,9 +30,7 @@ export const createManageSDL: CreateManageSDL = ({ fields: model.fields, fieldTypePlugins }); - if (inputFields.length === 0) { - return ""; - } + const listFilterFieldsRender = renderListFilterFields({ model, fields: model.fields, diff --git a/packages/api-headless-cms/src/graphql/schema/createPreviewResolvers.ts b/packages/api-headless-cms/src/graphql/schema/createPreviewResolvers.ts index bd9265107b0..a27f1e9ddba 100644 --- a/packages/api-headless-cms/src/graphql/schema/createPreviewResolvers.ts +++ b/packages/api-headless-cms/src/graphql/schema/createPreviewResolvers.ts @@ -20,12 +20,6 @@ export const createPreviewResolvers: CreateReadResolvers = ({ model, fieldTypePlugins }) => { - if (model.fields.length === 0) { - return { - Query: {} - }; - } - const createFieldResolvers = createFieldResolversFactory({ endpointType: "read", models, diff --git a/packages/api-headless-cms/src/graphql/schema/createReadResolvers.ts b/packages/api-headless-cms/src/graphql/schema/createReadResolvers.ts index a050aa9371a..5a3de1c8cba 100644 --- a/packages/api-headless-cms/src/graphql/schema/createReadResolvers.ts +++ b/packages/api-headless-cms/src/graphql/schema/createReadResolvers.ts @@ -16,12 +16,6 @@ export interface CreateReadResolvers { } export const createReadResolvers: CreateReadResolvers = ({ models, model, fieldTypePlugins }) => { - if (model.fields.length === 0) { - return { - Query: {} - }; - } - const createFieldResolvers = createFieldResolversFactory({ endpointType: "read", models, diff --git a/packages/api-headless-cms/src/graphql/schema/createReadSDL.ts b/packages/api-headless-cms/src/graphql/schema/createReadSDL.ts index 58daebfda8f..5299d825756 100644 --- a/packages/api-headless-cms/src/graphql/schema/createReadSDL.ts +++ b/packages/api-headless-cms/src/graphql/schema/createReadSDL.ts @@ -33,9 +33,6 @@ export const createReadSDL: CreateReadSDL = ({ fieldTypePlugins }); - if (fieldsRender.length === 0) { - return ""; - } const listFilterFieldsRender = renderListFilterFields({ model, fields: model.fields, @@ -70,7 +67,7 @@ export const createReadSDL: CreateReadSDL = ({ entryId: String! ${hasModelIdField ? "" : "modelId: String!"} - ${onByMetaGqlFields} + ${onByMetaGqlFields} publishedOn: DateTime @deprecated(reason: "Field was removed with the 5.39.0 release. Use 'firstPublishedOn' or 'lastPublishedOn' field.") ownedBy: CmsIdentity @deprecated(reason: "Field was removed with the 5.39.0 release. Use 'createdBy' field.") diff --git a/packages/api-headless-cms/src/graphql/schema/schemaPlugins.ts b/packages/api-headless-cms/src/graphql/schema/schemaPlugins.ts index 8b1d2867f0f..68240346409 100644 --- a/packages/api-headless-cms/src/graphql/schema/schemaPlugins.ts +++ b/packages/api-headless-cms/src/graphql/schema/schemaPlugins.ts @@ -45,65 +45,61 @@ export const generateSchemaPlugins = async ( type }); - models - .filter(model => { - return model.fields.length > 0; - }) - .forEach(model => { - switch (type) { - case "manage": - { - const plugin = createCmsGraphQLSchemaPlugin({ - typeDefs: createManageSDL({ - models, - model, - fieldTypePlugins, - sorterPlugins - }), - resolvers: createManageResolvers({ - models, - model, - fieldTypePlugins, - context - }) - }); - plugin.name = `headless-cms.graphql.schema.manage.${model.modelId}`; - schemaPlugins.push(plugin); - } + models.forEach(model => { + switch (type) { + case "manage": + { + const plugin = createCmsGraphQLSchemaPlugin({ + typeDefs: createManageSDL({ + models, + model, + fieldTypePlugins, + sorterPlugins + }), + resolvers: createManageResolvers({ + models, + model, + fieldTypePlugins, + context + }) + }); + plugin.name = `headless-cms.graphql.schema.manage.${model.modelId}`; + schemaPlugins.push(plugin); + } - break; - case "preview": - case "read": - { - const plugin = createCmsGraphQLSchemaPlugin({ - typeDefs: createReadSDL({ - models, - model, - fieldTypePlugins, - sorterPlugins - }), - resolvers: cms.READ - ? createReadResolvers({ - models, - model, - fieldTypePlugins, - context - }) - : createPreviewResolvers({ - models, - model, - fieldTypePlugins, - context - }) - }); - plugin.name = `headless-cms.graphql.schema.${type}.${model.modelId}`; - schemaPlugins.push(plugin); - } - break; - default: - return; - } - }); + break; + case "preview": + case "read": + { + const plugin = createCmsGraphQLSchemaPlugin({ + typeDefs: createReadSDL({ + models, + model, + fieldTypePlugins, + sorterPlugins + }), + resolvers: cms.READ + ? createReadResolvers({ + models, + model, + fieldTypePlugins, + context + }) + : createPreviewResolvers({ + models, + model, + fieldTypePlugins, + context + }) + }); + plugin.name = `headless-cms.graphql.schema.${type}.${model.modelId}`; + schemaPlugins.push(plugin); + } + break; + default: + return; + } + }); return schemaPlugins.filter(pl => !!pl.schema.typeDefs); }; From 1e124e09e66da74b9e8bd7c5bc6ee234b8dddaf3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20Zori=C4=87?= Date: Thu, 22 Aug 2024 13:12:42 +0200 Subject: [PATCH 18/70] fix(api-headless-cms): single entry models (#4210) --- .../decorateIfModelAuthorizationEnabled.ts | 2 +- .../dynamoDb/transformValue/datetime.test.ts | 13 +- .../contentAPI/mocks/contentModels.ts | 28 +- .../contentAPI/singletonContentEntry.test.ts | 181 ++++++++++ .../useSingletonCategoryHandler.ts | 92 +++++ packages/api-headless-cms/src/constants.ts | 2 + .../src/crud/contentModel.crud.ts | 19 +- .../src/crud/contentModel/beforeDelete.ts | 37 ++ .../src/export/crud/sanitize.ts | 3 +- packages/api-headless-cms/src/export/types.ts | 1 + .../graphql/schema/createSingularResolvers.ts | 64 ++++ .../src/graphql/schema/createSingularSDL.ts | 100 ++++++ .../schema/resolvers/singular/resolveGet.ts | 21 ++ .../resolvers/singular/resolveUpdate.ts | 26 ++ .../src/graphql/schema/schemaPlugins.ts | 25 +- .../modelManager/DefaultCmsModelManager.ts | 36 +- .../src/modelManager/SingletonModelManager.ts | 66 ++++ .../src/modelManager/index.ts | 1 + packages/api-headless-cms/src/types/types.ts | 17 +- .../app-headless-cms-common/src/constants.ts | 1 + .../src/entries.graphql.ts | 329 +++++++++++------- packages/app-headless-cms-common/src/index.ts | 1 + packages/app-headless-cms/src/HeadlessCMS.tsx | 2 + .../ContentEntryForm/ContentEntryForm.tsx | 28 +- .../ContentEntryFormPreview.tsx | 3 +- .../ContentEntryFormProvider.tsx | 55 +-- .../SingletonHeader/SaveAction.tsx | 17 + .../SingletonHeader/SingletonHeader.tsx | 38 ++ .../SingletonHeader/index.tsx | 1 + .../src/admin/contexts/Cms/index.tsx | 91 ++++- .../src/admin/hooks/usePersistEntry.ts | 50 +++ .../components/NewReferencedEntryDialog.tsx | 6 +- .../src/admin/plugins/fields/ref.tsx | 20 +- .../views/contentEntries/ContentEntry.tsx | 11 +- .../ContentEntry/SingletonContentEntry.tsx | 39 +++ .../SingletonContentEntryContext.tsx | 78 +++++ .../SingletonContentEntryModule.tsx | 36 ++ .../hooks/useSingletonContentEntry.ts | 13 + .../contentModels/ContentModelsDataList.tsx | 22 +- .../contentModels/NewContentModelDialog.tsx | 296 +++++++++------- packages/db-dynamodb/package.json | 1 - .../definitions/NumberTransformPlugin.ts | 30 -- .../definitions/TimeTransformPlugin.ts | 17 +- yarn.lock | 1 - 44 files changed, 1535 insertions(+), 385 deletions(-) create mode 100644 packages/api-headless-cms/__tests__/contentAPI/singletonContentEntry.test.ts create mode 100644 packages/api-headless-cms/__tests__/testHelpers/useSingletonCategoryHandler.ts create mode 100644 packages/api-headless-cms/src/graphql/schema/createSingularResolvers.ts create mode 100644 packages/api-headless-cms/src/graphql/schema/createSingularSDL.ts create mode 100644 packages/api-headless-cms/src/graphql/schema/resolvers/singular/resolveGet.ts create mode 100644 packages/api-headless-cms/src/graphql/schema/resolvers/singular/resolveUpdate.ts create mode 100644 packages/api-headless-cms/src/modelManager/SingletonModelManager.ts create mode 100644 packages/app-headless-cms-common/src/constants.ts create mode 100644 packages/app-headless-cms/src/admin/components/ContentEntryForm/SingletonHeader/SaveAction.tsx create mode 100644 packages/app-headless-cms/src/admin/components/ContentEntryForm/SingletonHeader/SingletonHeader.tsx create mode 100644 packages/app-headless-cms/src/admin/components/ContentEntryForm/SingletonHeader/index.tsx create mode 100644 packages/app-headless-cms/src/admin/hooks/usePersistEntry.ts create mode 100644 packages/app-headless-cms/src/admin/views/contentEntries/ContentEntry/SingletonContentEntry.tsx create mode 100644 packages/app-headless-cms/src/admin/views/contentEntries/ContentEntry/SingletonContentEntryContext.tsx create mode 100644 packages/app-headless-cms/src/admin/views/contentEntries/SingletonContentEntryModule.tsx create mode 100644 packages/app-headless-cms/src/admin/views/contentEntries/hooks/useSingletonContentEntry.ts delete mode 100644 packages/db-dynamodb/src/plugins/definitions/NumberTransformPlugin.ts diff --git a/packages/api-aco/src/utils/decorators/decorateIfModelAuthorizationEnabled.ts b/packages/api-aco/src/utils/decorators/decorateIfModelAuthorizationEnabled.ts index 587315cbefc..0e44f25d5f3 100644 --- a/packages/api-aco/src/utils/decorators/decorateIfModelAuthorizationEnabled.ts +++ b/packages/api-aco/src/utils/decorators/decorateIfModelAuthorizationEnabled.ts @@ -17,7 +17,7 @@ type FilterModelMethods = { * E.g., `getEntryManager` has `model` typed as `CmsModel | string`. * Ideally, we would filter those out in the previous utility type, but I'm not sure how to achieve that. */ -type ModelMethods = Omit, "getEntryManager">; +type ModelMethods = Omit, "getEntryManager" | "getSingletonEntryManager">; /** * Decorator takes the decoratee as the _first_ parameter, and then forwards the rest of the parameters. diff --git a/packages/api-headless-cms-ddb/__tests__/plugins/dynamoDb/transformValue/datetime.test.ts b/packages/api-headless-cms-ddb/__tests__/plugins/dynamoDb/transformValue/datetime.test.ts index 46e16a38ec3..1c4463cf51c 100644 --- a/packages/api-headless-cms-ddb/__tests__/plugins/dynamoDb/transformValue/datetime.test.ts +++ b/packages/api-headless-cms-ddb/__tests__/plugins/dynamoDb/transformValue/datetime.test.ts @@ -39,6 +39,7 @@ describe("dynamodb transform datetime", () => { const incorrectTimeValues: [any][] = [ [{}], [[]], + [""], [ function () { return 1; @@ -50,14 +51,20 @@ describe("dynamodb transform datetime", () => { test.each(incorrectTimeValues)( "should throw an error when trying to transform time field but value is not a string or a number", value => { + expect.assertions(1); const plugin = createDatetimeTransformValuePlugin(); - expect(() => { - plugin.transform({ + try { + const result = plugin.transform({ field: createField("time"), value }); - }).toThrow("Field value must be a string because field is defined as time."); + expect(result).toEqual("SHOULD NOT HAPPEN"); + } catch (ex) { + expect(ex.message).toEqual( + "Field value must be a string because field is defined as time." + ); + } } ); }); diff --git a/packages/api-headless-cms/__tests__/contentAPI/mocks/contentModels.ts b/packages/api-headless-cms/__tests__/contentAPI/mocks/contentModels.ts index 94b5bab7d55..50510526ec1 100644 --- a/packages/api-headless-cms/__tests__/contentAPI/mocks/contentModels.ts +++ b/packages/api-headless-cms/__tests__/contentAPI/mocks/contentModels.ts @@ -1,6 +1,12 @@ import { createContentModelGroup } from "./contentModelGroup"; import { CmsModel } from "~/types"; -import { CmsModelInput, createCmsGroup, createCmsModel } from "~/plugins"; +import { + CmsGroupPlugin, + CmsModelInput, + CmsModelPlugin, + createCmsGroupPlugin, + createCmsModelPlugin +} from "~/plugins"; const { version: webinyVersion } = require("@webiny/cli/package.json"); @@ -1839,7 +1845,7 @@ export const getCmsModel = (modelId: string) => { export const createModelPlugins = (targets: string[]) => { return [ - createCmsGroup({ + createCmsGroupPlugin({ ...contentModelGroup }), ...targets.map(modelId => { @@ -1851,7 +1857,23 @@ export const createModelPlugins = (targets: string[]) => { ...(model as Omit), noValidate: true }; - return createCmsModel(newModel); + return createCmsModelPlugin(newModel); + }) + ]; +}; + +export const createPluginFromCmsModel = ( + model: Omit +): (CmsModelPlugin | CmsGroupPlugin)[] => { + return [ + createCmsGroupPlugin(contentModelGroup), + createCmsModelPlugin({ + ...model, + group: { + id: contentModelGroup.id, + name: contentModelGroup.name + }, + noValidate: true }) ]; }; diff --git a/packages/api-headless-cms/__tests__/contentAPI/singletonContentEntry.test.ts b/packages/api-headless-cms/__tests__/contentAPI/singletonContentEntry.test.ts new file mode 100644 index 00000000000..893e9515939 --- /dev/null +++ b/packages/api-headless-cms/__tests__/contentAPI/singletonContentEntry.test.ts @@ -0,0 +1,181 @@ +import { useSingletonCategoryHandler } from "~tests/testHelpers/useSingletonCategoryHandler"; +import { createPluginFromCmsModel, getCmsModel } from "~tests/contentAPI/mocks/contentModels"; +import { CMS_MODEL_SINGLETON_TAG } from "~/constants"; +// import { useHandler } from "~tests/testHelpers/useHandler"; + +describe("singleton model content entries", () => { + const plugins = createPluginFromCmsModel({ + ...getCmsModel("category"), + tags: [CMS_MODEL_SINGLETON_TAG] + }); + + // const { handler: contextHandler } = useHandler({ + // plugins + // }); + + const manager = useSingletonCategoryHandler({ + path: `manage/en-US`, + plugins + }); + const reader = useSingletonCategoryHandler({ + path: `read/en-US`, + plugins + }); + + it("should fetch the singleton entry", async () => { + const [managerResponse] = await manager.getCategory(); + + expect(managerResponse).toEqual({ + data: { + getCategory: { + data: { + createdBy: { + displayName: "John Doe", + id: "id-12345678", + type: "admin" + }, + createdOn: expect.toBeDateString(), + deletedBy: null, + deletedOn: null, + entryId: expect.any(String), + firstPublishedOn: null, + id: expect.any(String), + lastPublishedOn: null, + modifiedBy: null, + modifiedOn: null, + restoredBy: null, + restoredOn: null, + savedBy: { + displayName: "John Doe", + id: "id-12345678", + type: "admin" + }, + savedOn: expect.toBeDateString(), + slug: null, + title: null + }, + error: null + } + } + }); + + const item = managerResponse.data.getCategory.data; + + const [readerResponse] = await reader.getCategory(); + expect(readerResponse).toEqual({ + data: { + getCategory: { + data: item, + error: null + } + } + }); + }); + + it("should update the singleton entry", async () => { + const data = { + title: "New title", + slug: "new-title" + }; + const [managerResponse] = await manager.updateCategory({ + data + }); + + expect(managerResponse).toMatchObject({ + data: { + updateCategory: { + data, + error: null + } + } + }); + + const [readerResponse] = await reader.getCategory(); + expect(readerResponse).toMatchObject({ + data: { + getCategory: { + data, + error: null + } + } + }); + }); + + it("should update the singleton entry multiple times in a row", async () => { + const [rootResponse] = await reader.getCategory(); + expect(rootResponse).toMatchObject({ + data: { + getCategory: { + data: { + id: expect.any(String) + }, + error: null + } + } + }); + + const id = rootResponse.data.getCategory.data.id; + + const data = { + title: "New title", + slug: "new-title" + }; + const [response1] = await manager.updateCategory({ + data + }); + expect(response1).toMatchObject({ + data: { + updateCategory: { + data: { + ...data, + id + }, + error: null + } + } + }); + + const [response2] = await manager.updateCategory({ + data + }); + expect(response2).toMatchObject({ + data: { + updateCategory: { + data: { + ...data, + id + }, + error: null + } + } + }); + + const [response3] = await manager.updateCategory({ + data + }); + expect(response3).toMatchObject({ + data: { + updateCategory: { + data: { + ...data, + id + }, + error: null + } + } + }); + + const [readerResponse] = await reader.getCategory(); + expect(readerResponse).toMatchObject({ + data: { + getCategory: { + data: { + ...data, + id + }, + error: null + } + } + }); + }); +}); diff --git a/packages/api-headless-cms/__tests__/testHelpers/useSingletonCategoryHandler.ts b/packages/api-headless-cms/__tests__/testHelpers/useSingletonCategoryHandler.ts new file mode 100644 index 00000000000..b0a1465d905 --- /dev/null +++ b/packages/api-headless-cms/__tests__/testHelpers/useSingletonCategoryHandler.ts @@ -0,0 +1,92 @@ +import { GraphQLHandlerParams, useGraphQLHandler } from "./useGraphQLHandler"; +import { CmsModel } from "~/types"; +import { getCmsModel } from "~tests/contentAPI/mocks/contentModels"; +import { GenericRecord } from "@webiny/api/types"; + +const identityFields = /* GraphQL */ ` + { + id + displayName + type + } +`; + +const categoryFields = ` + id + entryId + createdOn + modifiedOn + savedOn + firstPublishedOn + lastPublishedOn + deletedOn + restoredOn + createdBy ${identityFields} + modifiedBy ${identityFields} + savedBy ${identityFields} + deletedBy ${identityFields} + restoredBy ${identityFields} + # user fields + title + slug +`; + +const errorFields = ` + error { + code + message + data + } +`; + +const getCategoryQuery = (model: CmsModel) => { + return /* GraphQL */ ` + query GetCategory { + getCategory: get${model.singularApiName} { + data { + ${categoryFields} + } + ${errorFields} + } + } + `; +}; + +const updateCategoryMutation = (model: CmsModel) => { + return /* GraphQL */ ` + mutation UpdateCategory($data: ${model.singularApiName}Input!) { + updateCategory: update${model.singularApiName}(data: $data) { + data { + ${categoryFields} + } + ${errorFields} + } + } + `; +}; + +export const useSingletonCategoryHandler = (params: GraphQLHandlerParams) => { + const model = getCmsModel("category"); + const contentHandler = useGraphQLHandler(params); + + return { + ...contentHandler, + async getCategory(headers: GenericRecord = {}) { + return await contentHandler.invoke({ + body: { + query: getCategoryQuery(model) + }, + headers + }); + }, + async updateCategory(variables: Record, headers: Record = {}) { + return await contentHandler.invoke({ + body: { + query: updateCategoryMutation(model), + variables + }, + headers + }); + } + }; +}; diff --git a/packages/api-headless-cms/src/constants.ts b/packages/api-headless-cms/src/constants.ts index 84d9886a5bc..ef271526a18 100644 --- a/packages/api-headless-cms/src/constants.ts +++ b/packages/api-headless-cms/src/constants.ts @@ -2,6 +2,8 @@ import { CmsIdentity } from "~/types"; export const ROOT_FOLDER = "root"; +export const CMS_MODEL_SINGLETON_TAG = "singleton"; + // Content entries - xOn and xBy meta fields. export const ENTRY_META_FIELDS = [ // Entry-level meta fields. diff --git a/packages/api-headless-cms/src/crud/contentModel.crud.ts b/packages/api-headless-cms/src/crud/contentModel.crud.ts index 276158caae5..e6cc6606105 100644 --- a/packages/api-headless-cms/src/crud/contentModel.crud.ts +++ b/packages/api-headless-cms/src/crud/contentModel.crud.ts @@ -1,6 +1,7 @@ import WebinyError from "@webiny/error"; import { CmsContext, + CmsEntryValues, CmsModel, CmsModelContext, CmsModelFieldToGraphQLPlugin, @@ -45,9 +46,10 @@ import { listModelsFromDatabase } from "~/crud/contentModel/listModelsFromDataba import { filterAsync } from "~/utils/filterAsync"; import { AccessControl } from "./AccessControl/AccessControl"; import { - CmsModelToAstConverter, - CmsModelFieldToAstConverterFromPlugins + CmsModelFieldToAstConverterFromPlugins, + CmsModelToAstConverter } from "~/utils/contentModelAst"; +import { SingletonModelManager } from "~/modelManager"; export interface CreateModelsCrudParams { getTenant: () => Tenant; @@ -217,6 +219,16 @@ export const createModelsCrud = (params: CreateModelsCrudParams): CmsModelContex return await updateManager(context, model); }; + const getSingletonEntryManager = async ( + input: CmsModel | string + ) => { + const model = typeof input === "string" ? await getModel(input) : input; + + const manager = await getEntryManager(model); + + return SingletonModelManager.create(manager); + }; + /** * Create */ @@ -691,6 +703,7 @@ export const createModelsCrud = (params: CreateModelsCrudParams): CmsModelContex ); }, getEntryManager, - getEntryManagers: () => managers + getEntryManagers: () => managers, + getSingletonEntryManager }; }; diff --git a/packages/api-headless-cms/src/crud/contentModel/beforeDelete.ts b/packages/api-headless-cms/src/crud/contentModel/beforeDelete.ts index 5a1574f8d55..fa44538755e 100644 --- a/packages/api-headless-cms/src/crud/contentModel/beforeDelete.ts +++ b/packages/api-headless-cms/src/crud/contentModel/beforeDelete.ts @@ -2,6 +2,7 @@ import { Topic } from "@webiny/pubsub/types"; import { CmsContext, OnModelBeforeDeleteTopicParams } from "~/types"; import WebinyError from "@webiny/error"; import { CmsModelPlugin } from "~/plugins/CmsModelPlugin"; +import { CMS_MODEL_SINGLETON_TAG } from "~/constants"; interface AssignBeforeModelDeleteParams { onModelBeforeDelete: Topic; @@ -27,6 +28,42 @@ export const assignModelBeforeDelete = (params: AssignBeforeModelDeleteParams) = ); } + const tags = Array.isArray(model.tags) ? model.tags : []; + /** + * If the model is a singleton, we need to delete all entries. + * There will be either 0 or 1 entries in latest or deleted, but let's put high limit, just in case... + */ + if (tags.includes(CMS_MODEL_SINGLETON_TAG)) { + const [latestEntries] = await context.cms.listLatestEntries(model, { + limit: 10000 + }); + + if (latestEntries.length > 0) { + for (const item of latestEntries) { + await context.cms.deleteEntry(model, item.id, { + permanently: true + }); + } + return; + } + + const [deletedEntries] = await context.cms.listDeletedEntries(model, { + limit: 10000 + }); + + if (deletedEntries.length === 0) { + return; + } + + for (const item of deletedEntries) { + await context.cms.deleteEntry(model, item.id, { + permanently: true + }); + } + + return; + } + try { const [latestEntries] = await context.cms.listLatestEntries(model, { limit: 1 }); diff --git a/packages/api-headless-cms/src/export/crud/sanitize.ts b/packages/api-headless-cms/src/export/crud/sanitize.ts index b0d607ac39a..7ae03cf3aae 100644 --- a/packages/api-headless-cms/src/export/crud/sanitize.ts +++ b/packages/api-headless-cms/src/export/crud/sanitize.ts @@ -27,6 +27,7 @@ export const sanitizeModel = ( layout: model.layout, titleFieldId: model.titleFieldId, descriptionFieldId: model.descriptionFieldId, - imageFieldId: model.imageFieldId + imageFieldId: model.imageFieldId, + tags: model.tags || [] }; }; diff --git a/packages/api-headless-cms/src/export/types.ts b/packages/api-headless-cms/src/export/types.ts index 62d7613ed24..22e3a705c8b 100644 --- a/packages/api-headless-cms/src/export/types.ts +++ b/packages/api-headless-cms/src/export/types.ts @@ -37,6 +37,7 @@ export interface SanitizedCmsModel | "pluralApiName" | "name" | "description" + | "tags" > { group: string; } diff --git a/packages/api-headless-cms/src/graphql/schema/createSingularResolvers.ts b/packages/api-headless-cms/src/graphql/schema/createSingularResolvers.ts new file mode 100644 index 00000000000..4985f2d790d --- /dev/null +++ b/packages/api-headless-cms/src/graphql/schema/createSingularResolvers.ts @@ -0,0 +1,64 @@ +import { ApiEndpoint, CmsContext, CmsFieldTypePlugins, CmsModel } from "~/types"; +import { resolveGet } from "./resolvers/singular/resolveGet"; +import { resolveUpdate } from "./resolvers/singular/resolveUpdate"; +import { normalizeGraphQlInput } from "./resolvers/manage/normalizeGraphQlInput"; +import { createFieldResolversFactory } from "./createFieldResolvers"; + +interface CreateSingularResolversParams { + models: CmsModel[]; + model: CmsModel; + context: CmsContext; + fieldTypePlugins: CmsFieldTypePlugins; + type: ApiEndpoint; +} + +interface CreateSingularResolvers { + // TODO @ts-refactor determine correct type. + (params: CreateSingularResolversParams): any; +} + +export const createSingularResolvers: CreateSingularResolvers = ({ + models, + model, + fieldTypePlugins, + type +}) => { + if (model.fields.length === 0) { + return { + Query: {}, + Mutation: {} + }; + } + + const createFieldResolvers = createFieldResolversFactory({ + endpointType: "manage", + models, + model, + fieldTypePlugins + }); + + const fieldResolvers = createFieldResolvers({ + graphQLType: model.singularApiName, + fields: model.fields, + isRoot: true + }); + + const resolverFactoryParams = { model, fieldTypePlugins }; + + const result = { + Query: { + [`get${model.singularApiName}`]: resolveGet(resolverFactoryParams) + }, + ...fieldResolvers + }; + if (type !== "manage") { + return result; + } + return { + ...result, + Mutation: { + [`update${model.singularApiName}`]: + normalizeGraphQlInput(resolveUpdate)(resolverFactoryParams) + } + }; +}; diff --git a/packages/api-headless-cms/src/graphql/schema/createSingularSDL.ts b/packages/api-headless-cms/src/graphql/schema/createSingularSDL.ts new file mode 100644 index 00000000000..f0728b0a5b0 --- /dev/null +++ b/packages/api-headless-cms/src/graphql/schema/createSingularSDL.ts @@ -0,0 +1,100 @@ +import { ApiEndpoint, CmsFieldTypePlugins, CmsModel } from "~/types"; +import { renderInputFields } from "~/utils/renderInputFields"; +import { renderFields } from "~/utils/renderFields"; +import { ENTRY_META_FIELDS, isDateTimeEntryMetaField } from "~/constants"; + +interface CreateSingularSDLParams { + models: CmsModel[]; + model: CmsModel; + fieldTypePlugins: CmsFieldTypePlugins; + type: ApiEndpoint; +} + +interface CreateSingularSDL { + (params: CreateSingularSDLParams): string; +} + +export const createSingularSDL: CreateSingularSDL = ({ + models, + model, + fieldTypePlugins, + type +}): string => { + const inputFields = renderInputFields({ + models, + model, + fields: model.fields, + fieldTypePlugins + }); + if (inputFields.length === 0) { + return ""; + } + + const fields = renderFields({ + models, + model, + fields: model.fields, + type: "manage", + fieldTypePlugins + }); + + const { singularApiName: singularName } = model; + + const inputGqlFields = inputFields.map(f => f.fields).join("\n"); + + const onByMetaInputGqlFields = ENTRY_META_FIELDS.map(field => { + const fieldType = isDateTimeEntryMetaField(field) ? "DateTime" : "CmsIdentityInput"; + + return `${field}: ${fieldType}`; + }).join("\n"); + + const onByMetaGqlFields = ENTRY_META_FIELDS.map(field => { + const fieldType = isDateTimeEntryMetaField(field) ? "DateTime" : "CmsIdentity"; + + return `${field}: ${fieldType}`; + }).join("\n"); + + // Had to remove /* GraphQL */ because prettier would not format the code correctly. + const read = ` + """${model.description || singularName}""" + type ${singularName} { + id: ID! + entryId: String! + + ${onByMetaGqlFields} + + ownedBy: CmsIdentity @deprecated(reason: "Field was removed with the 5.39.0 release. Use 'createdBy' field.") + + ${fields.map(f => f.fields).join("\n")} + } + + ${fields.map(f => f.typeDefs).join("\n")} + + type ${singularName}Response { + data: ${singularName} + error: CmsError + } + + extend type Query { + get${singularName}: ${singularName}Response + } + + `; + if (type !== "manage") { + return read; + } + return ` + ${read} + + ${inputFields.map(f => f.typeDefs).join("\n")} + + input ${singularName}Input { + ${onByMetaInputGqlFields} + ${inputGqlFields} + } + + extend type Mutation { + update${singularName}(data: ${singularName}Input!, options: UpdateCmsEntryOptionsInput): ${singularName}Response + } + `; +}; diff --git a/packages/api-headless-cms/src/graphql/schema/resolvers/singular/resolveGet.ts b/packages/api-headless-cms/src/graphql/schema/resolvers/singular/resolveGet.ts new file mode 100644 index 00000000000..7241d130f39 --- /dev/null +++ b/packages/api-headless-cms/src/graphql/schema/resolvers/singular/resolveGet.ts @@ -0,0 +1,21 @@ +import { ErrorResponse, Response } from "@webiny/handler-graphql/responses"; +import { CmsEntryResolverFactory as ResolverFactory } from "~/types"; + +interface ResolveGetArgs { + revision: string; +} + +type ResolveGet = ResolverFactory; + +export const resolveGet: ResolveGet = + ({ model }) => + async (_: unknown, __: unknown, context) => { + try { + const manager = await context.cms.getSingletonEntryManager(model); + const entry = await manager.get(); + + return new Response(entry); + } catch (e) { + return new ErrorResponse(e); + } + }; diff --git a/packages/api-headless-cms/src/graphql/schema/resolvers/singular/resolveUpdate.ts b/packages/api-headless-cms/src/graphql/schema/resolvers/singular/resolveUpdate.ts new file mode 100644 index 00000000000..5b8db7e6b35 --- /dev/null +++ b/packages/api-headless-cms/src/graphql/schema/resolvers/singular/resolveUpdate.ts @@ -0,0 +1,26 @@ +import { ErrorResponse, Response } from "@webiny/handler-graphql/responses"; +import { + CmsEntryResolverFactory as ResolverFactory, + UpdateCmsEntryInput, + UpdateCmsEntryOptionsInput +} from "~/types"; + +interface ResolveUpdateArgs { + data: UpdateCmsEntryInput; + options?: UpdateCmsEntryOptionsInput; +} + +type ResolveUpdate = ResolverFactory; + +export const resolveUpdate: ResolveUpdate = + ({ model }) => + async (_: unknown, args, context) => { + try { + const manager = await context.cms.getSingletonEntryManager(model.modelId); + const entry = await manager.update(args.data, args.options); + + return new Response(entry); + } catch (e) { + return new ErrorResponse(e); + } + }; diff --git a/packages/api-headless-cms/src/graphql/schema/schemaPlugins.ts b/packages/api-headless-cms/src/graphql/schema/schemaPlugins.ts index 68240346409..cac7f5e45c6 100644 --- a/packages/api-headless-cms/src/graphql/schema/schemaPlugins.ts +++ b/packages/api-headless-cms/src/graphql/schema/schemaPlugins.ts @@ -10,7 +10,10 @@ import { createCmsGraphQLSchemaPlugin, ICmsGraphQLSchemaPlugin } from "~/plugins"; -import { createFieldTypePluginRecords } from "~/graphql/schema/createFieldTypePluginRecords"; +import { createFieldTypePluginRecords } from "./createFieldTypePluginRecords"; +import { CMS_MODEL_SINGLETON_TAG } from "~/constants"; +import { createSingularSDL } from "./createSingularSDL"; +import { createSingularResolvers } from "./createSingularResolvers"; interface GenerateSchemaPluginsParams { context: CmsContext; @@ -46,6 +49,26 @@ export const generateSchemaPlugins = async ( }); models.forEach(model => { + if (model.tags?.includes(CMS_MODEL_SINGLETON_TAG)) { + const plugin = createCmsGraphQLSchemaPlugin({ + typeDefs: createSingularSDL({ + models, + model, + fieldTypePlugins, + type + }), + resolvers: createSingularResolvers({ + context, + models, + model, + fieldTypePlugins, + type + }) + }); + plugin.name = `headless-cms.graphql.schema.singular.${model.modelId}`; + schemaPlugins.push(plugin); + return; + } switch (type) { case "manage": { diff --git a/packages/api-headless-cms/src/modelManager/DefaultCmsModelManager.ts b/packages/api-headless-cms/src/modelManager/DefaultCmsModelManager.ts index 1df298e9bf3..8a7ac3aafc5 100644 --- a/packages/api-headless-cms/src/modelManager/DefaultCmsModelManager.ts +++ b/packages/api-headless-cms/src/modelManager/DefaultCmsModelManager.ts @@ -4,57 +4,63 @@ import { CmsContext, CmsEntryListParams, CreateCmsEntryInput, - UpdateCmsEntryInput + UpdateCmsEntryInput, + UpdateCmsEntryOptionsInput, + CreateCmsEntryOptionsInput } from "~/types"; import { parseIdentifier } from "@webiny/utils"; export class DefaultCmsModelManager implements CmsModelManager { private readonly _context: CmsContext; - private readonly _model: CmsModel; + public readonly model: CmsModel; public constructor(context: CmsContext, model: CmsModel) { this._context = context; - this._model = model; + this.model = model; } - public async create(data: CreateCmsEntryInput) { - return this._context.cms.createEntry(this._model, data); + public async create(data: CreateCmsEntryInput, options?: CreateCmsEntryOptionsInput) { + return this._context.cms.createEntry(this.model, data, options); } public async delete(id: string) { const { version } = parseIdentifier(id); if (version) { - return this._context.cms.deleteEntryRevision(this._model, id); + return this._context.cms.deleteEntryRevision(this.model, id); } - return this._context.cms.deleteEntry(this._model, id); + return this._context.cms.deleteEntry(this.model, id); } public async get(id: string) { - return this._context.cms.getEntryById(this._model, id); + return this._context.cms.getEntryById(this.model, id); } public async listPublished(params: CmsEntryListParams) { - return this._context.cms.listPublishedEntries(this._model, params); + return this._context.cms.listPublishedEntries(this.model, params); } public async listLatest(params: CmsEntryListParams) { - return this._context.cms.listLatestEntries(this._model, params); + return this._context.cms.listLatestEntries(this.model, params); } public async listDeleted(params: CmsEntryListParams) { - return this._context.cms.listDeletedEntries(this._model, params); + return this._context.cms.listDeletedEntries(this.model, params); } public async getPublishedByIds(ids: string[]) { - return this._context.cms.getPublishedEntriesByIds(this._model, ids); + return this._context.cms.getPublishedEntriesByIds(this.model, ids); } public async getLatestByIds(ids: string[]) { - return this._context.cms.getLatestEntriesByIds(this._model, ids); + return this._context.cms.getLatestEntriesByIds(this.model, ids); } - public async update(id: string, data: UpdateCmsEntryInput) { - return this._context.cms.updateEntry(this._model, id, data); + public async update( + id: string, + data: UpdateCmsEntryInput, + options?: UpdateCmsEntryOptionsInput + ) { + return this._context.cms.updateEntry(this.model, id, data, options); } } diff --git a/packages/api-headless-cms/src/modelManager/SingletonModelManager.ts b/packages/api-headless-cms/src/modelManager/SingletonModelManager.ts new file mode 100644 index 00000000000..07204152b65 --- /dev/null +++ b/packages/api-headless-cms/src/modelManager/SingletonModelManager.ts @@ -0,0 +1,66 @@ +import { + CmsEntry, + CmsEntryValues, + CmsModelManager, + UpdateCmsEntryInput, + UpdateCmsEntryOptionsInput +} from "~/types"; +import { WebinyError } from "@webiny/error"; +import { CMS_MODEL_SINGLETON_TAG } from "~/constants"; +import { createCacheKey } from "@webiny/utils"; + +export interface ISingletonModelManager { + update(data: UpdateCmsEntryInput, options?: UpdateCmsEntryOptionsInput): Promise>; + get(): Promise>; +} + +export class SingletonModelManager implements ISingletonModelManager { + private readonly manager: CmsModelManager; + + private constructor(manager: CmsModelManager) { + if (!manager.model.tags?.includes(CMS_MODEL_SINGLETON_TAG)) { + throw new WebinyError({ + message: "Model is not marked as singular.", + code: "MODEL_NOT_MARKED_AS_SINGULAR", + data: { + model: manager.model + } + }); + } + this.manager = manager; + } + + public async update( + data: UpdateCmsEntryInput, + options?: UpdateCmsEntryOptionsInput + ): Promise> { + const entry = await this.get(); + + return await this.manager.update(entry.id, data, options); + } + + public async get(): Promise> { + const id = createCacheKey(this.manager.model.modelId, { + algorithm: "sha256", + encoding: "hex" + }); + try { + return await this.manager.get(`${id}#0001`); + } catch { + return await this.manager.create( + { + id + }, + { + skipValidators: ["required"] + } + ); + } + } + + public static create( + manager: CmsModelManager + ): ISingletonModelManager { + return new SingletonModelManager(manager); + } +} diff --git a/packages/api-headless-cms/src/modelManager/index.ts b/packages/api-headless-cms/src/modelManager/index.ts index ff64efa8202..bcc722b4678 100644 --- a/packages/api-headless-cms/src/modelManager/index.ts +++ b/packages/api-headless-cms/src/modelManager/index.ts @@ -1,5 +1,6 @@ import { CmsModelManager, ModelManagerPlugin } from "~/types"; import { DefaultCmsModelManager } from "./DefaultCmsModelManager"; +export * from "./SingletonModelManager"; const plugin: ModelManagerPlugin = { type: "cms-content-model-manager", diff --git a/packages/api-headless-cms/src/types/types.ts b/packages/api-headless-cms/src/types/types.ts index 41e05096bef..9b31bcc50c2 100644 --- a/packages/api-headless-cms/src/types/types.ts +++ b/packages/api-headless-cms/src/types/types.ts @@ -16,6 +16,7 @@ import { CmsModelField, CmsModelFieldValidation, CmsModelUpdateInput } from "./m import { CmsModel, CmsModelCreateFromInput, CmsModelCreateInput } from "./model"; import { CmsGroup } from "./modelGroup"; import { CmsIdentity } from "./identity"; +import { ISingletonModelManager } from "~/modelManager"; export interface CmsError { message: string; @@ -693,6 +694,7 @@ export interface CmsEntryUniqueValue { * @category CmsModel */ export interface CmsModelManager { + model: CmsModel; /** * List only published entries in the content model. */ @@ -716,11 +718,18 @@ export interface CmsModelManager { /** * Create an entry. */ - create(data: CreateCmsEntryInput & I): Promise>; + create( + data: CreateCmsEntryInput & I, + options?: CreateCmsEntryOptionsInput + ): Promise>; /** * Update an entry. */ - update(id: string, data: UpdateCmsEntryInput): Promise>; + update( + id: string, + data: UpdateCmsEntryInput, + options?: UpdateCmsEntryOptionsInput + ): Promise>; /** * Delete an entry. */ @@ -877,6 +886,10 @@ export interface CmsModelContext { * @see CmsModelManager */ getEntryManager(model: CmsModel | string): Promise>; + /** + * A model manager for a model which has a single entry. + */ + getSingletonEntryManager(model: CmsModel | string): Promise>; /** * Get all content model managers mapped by modelId. * @see CmsModelManager diff --git a/packages/app-headless-cms-common/src/constants.ts b/packages/app-headless-cms-common/src/constants.ts new file mode 100644 index 00000000000..f764a149c27 --- /dev/null +++ b/packages/app-headless-cms-common/src/constants.ts @@ -0,0 +1 @@ +export const CMS_MODEL_SINGLETON_TAG = "singleton"; diff --git a/packages/app-headless-cms-common/src/entries.graphql.ts b/packages/app-headless-cms-common/src/entries.graphql.ts index 41c38d5a97c..1549601275a 100644 --- a/packages/app-headless-cms-common/src/entries.graphql.ts +++ b/packages/app-headless-cms-common/src/entries.graphql.ts @@ -5,133 +5,146 @@ import { CmsEditorContentModel, CmsErrorResponse, CmsMetaResponse, - CmsModelField + CmsModelField, + CmsModel } from "~/types"; import { createFieldsList } from "./createFieldsList"; import { getModelTitleFieldId } from "./getModelTitleFieldId"; import { FormValidationOptions } from "@webiny/form"; +import { CMS_MODEL_SINGLETON_TAG } from "./constants"; const CONTENT_META_FIELDS = /* GraphQL */ ` - meta { - title - description - image - version - locked - status - } + title + description + image + version + locked + status `; -const CONTENT_ENTRY_SYSTEM_FIELDS = /* GraphQL */ ` - id - entryId - createdOn - savedOn - modifiedOn, - deletedOn - firstPublishedOn - lastPublishedOn - createdBy { - id - type - displayName - } - savedBy { - id - type - displayName - } - modifiedBy { - id - type - displayName - } - deletedBy { - id - type - displayName - } - firstPublishedBy { - id - type - displayName - } - lastPublishedBy { - id - type - displayName - } - revisionCreatedOn - revisionSavedOn - revisionModifiedOn - revisionDeletedOn - revisionFirstPublishedOn - revisionLastPublishedOn - revisionCreatedBy { - id - type - displayName - } - revisionSavedBy { - id - type - displayName - } - revisionModifiedBy { - id - type - displayName - } - revisionDeletedBy { - id - type - displayName - } - revisionFirstPublishedBy { - id - type - displayName - } - revisionLastPublishedBy { - id - type - displayName - } - revisionCreatedOn - revisionSavedOn - revisionModifiedOn - revisionFirstPublishedOn - revisionLastPublishedOn - revisionCreatedBy { - id - type - displayName - } - revisionSavedBy { - id - type - displayName - } - revisionModifiedBy { - id - type - displayName - } - revisionFirstPublishedBy { - id - type - displayName +const createEntrySystemFields = (model: CmsModel) => { + const isSingletonModel = model.tags.includes(CMS_MODEL_SINGLETON_TAG); + + let optionalFields = ""; + if (!isSingletonModel) { + optionalFields = ` + wbyAco_location { + folderId + } + meta { + ${CONTENT_META_FIELDS} + } + `; } - revisionLastPublishedBy { + + return /* GraphQL */ ` id - type - displayName - } - wbyAco_location { - folderId - } - ${CONTENT_META_FIELDS} -`; + entryId + createdOn + savedOn + modifiedOn, + deletedOn + firstPublishedOn + lastPublishedOn + createdBy { + id + type + displayName + } + savedBy { + id + type + displayName + } + modifiedBy { + id + type + displayName + } + deletedBy { + id + type + displayName + } + firstPublishedBy { + id + type + displayName + } + lastPublishedBy { + id + type + displayName + } + revisionCreatedOn + revisionSavedOn + revisionModifiedOn + revisionDeletedOn + revisionFirstPublishedOn + revisionLastPublishedOn + revisionCreatedBy { + id + type + displayName + } + revisionSavedBy { + id + type + displayName + } + revisionModifiedBy { + id + type + displayName + } + revisionDeletedBy { + id + type + displayName + } + revisionFirstPublishedBy { + id + type + displayName + } + revisionLastPublishedBy { + id + type + displayName + } + revisionCreatedOn + revisionSavedOn + revisionModifiedOn + revisionFirstPublishedOn + revisionLastPublishedOn + revisionCreatedBy { + id + type + displayName + } + revisionSavedBy { + id + type + displayName + } + revisionModifiedBy { + id + type + displayName + } + revisionFirstPublishedBy { + id + type + displayName + } + revisionLastPublishedBy { + id + type + displayName + } + ${optionalFields} + `; +}; const ERROR_FIELD = /* GraphQL */ ` { @@ -165,7 +178,32 @@ export const createReadQuery = (model: CmsEditorContentModel) => { query CmsEntriesGet${model.singularApiName}($revision: ID, $entryId: ID) { content: get${model.singularApiName}(revision: $revision, entryId: $entryId) { data { - ${CONTENT_ENTRY_SYSTEM_FIELDS} + ${createEntrySystemFields(model)} + ${createFieldsList({ model, fields: model.fields })} + } + error ${ERROR_FIELD} + } + } + `; +}; + +/** + * ############################################ + * Get CMS Singleton Entry Query + */ +export interface CmsEntryGetSingletonQueryResponse { + content: { + data: CmsContentEntry; + error: CmsErrorResponse | null; + }; +} + +export const createReadSingletonQuery = (model: CmsEditorContentModel) => { + return gql` + query CmsEntryGetSingleton${model.singularApiName} { + content: get${model.singularApiName} { + data { + ${createEntrySystemFields(model)} ${createFieldsList({ model, fields: model.fields })} } error ${ERROR_FIELD} @@ -195,7 +233,7 @@ export const createRevisionsQuery = (model: CmsEditorContentModel) => { query CmsEntriesGet${model.singularApiName}Revisions($id: ID!) { revisions: get${model.singularApiName}Revisions(id: $id) { data { - ${CONTENT_ENTRY_SYSTEM_FIELDS} + ${createEntrySystemFields(model)} } error ${ERROR_FIELD} } @@ -230,7 +268,7 @@ export const createListQueryDataSelection = ( fields?: CmsModelField[] ) => { return ` - ${CONTENT_ENTRY_SYSTEM_FIELDS} + ${createEntrySystemFields(model)} ${fields ? createFieldsList({ model, fields }) : ""} ${!fields ? getModelTitleFieldId(model) : ""} `; @@ -315,7 +353,7 @@ export const createRestoreFromBinMutation = (model: CmsEditorContentModel) => { mutation CmsEntriesRestore${model.singularApiName}FromBin($revision: ID!) { content: restore${model.singularApiName}FromBin(revision: $revision) { data { - ${CONTENT_ENTRY_SYSTEM_FIELDS} + ${createEntrySystemFields(model)} ${createFieldsList({ model, fields: model.fields })} } error ${ERROR_FIELD} @@ -347,10 +385,12 @@ export const createCreateMutation = (model: CmsEditorContentModel) => { const createFields = createFieldsList({ model, fields: model.fields }); return gql` - mutation CmsEntriesCreate${model.singularApiName}($data: ${model.singularApiName}Input!, $options: CreateCmsEntryOptionsInput) { + mutation CmsEntriesCreate${model.singularApiName}($data: ${ + model.singularApiName + }Input!, $options: CreateCmsEntryOptionsInput) { content: create${model.singularApiName}(data: $data, options: $options) { data { - ${CONTENT_ENTRY_SYSTEM_FIELDS} + ${createEntrySystemFields(model)} ${createFields} } error ${ERROR_FIELD} @@ -388,7 +428,7 @@ export const createCreateFromMutation = (model: CmsEditorContentModel) => { model.singularApiName }From(revision: $revision, data: $data, options: $options) { data { - ${CONTENT_ENTRY_SYSTEM_FIELDS} + ${createEntrySystemFields(model)} ${createFieldsList({ model, fields: model.fields })} } error ${ERROR_FIELD} @@ -425,7 +465,7 @@ export const createUpdateMutation = (model: CmsEditorContentModel) => { model.singularApiName }(revision: $revision, data: $data, options: $options) { data { - ${CONTENT_ENTRY_SYSTEM_FIELDS} + ${createEntrySystemFields(model)} ${createFieldsList({ model, fields: model.fields })} } error ${ERROR_FIELD} @@ -434,6 +474,41 @@ export const createUpdateMutation = (model: CmsEditorContentModel) => { `; }; +/** + * ############################################ + * Update Singleton Mutation + */ +export interface CmsEntryUpdateSingletonMutationResponse { + content: { + data?: CmsContentEntry; + error?: CmsErrorResponse; + }; +} + +export interface CmsEntryUpdateSingletonMutationVariables { + /** + * We have any here because we do not know which fields does entry have + */ + data: Record; + options?: FormValidationOptions; +} + +export const createUpdateSingletonMutation = (model: CmsEditorContentModel) => { + return gql` + mutation CmsUpdate${model.singularApiName}($data: ${ + model.singularApiName + }Input!, $options: UpdateCmsEntryOptionsInput) { + content: update${model.singularApiName}(data: $data, options: $options) { + data { + ${createEntrySystemFields(model)} + ${createFieldsList({ model, fields: model.fields })} + } + error ${ERROR_FIELD} + } + } + `; +}; + /** * ############################################ * Publish Mutation @@ -454,7 +529,7 @@ export const createPublishMutation = (model: CmsEditorContentModel) => { mutation CmsPublish${model.singularApiName}($revision: ID!) { content: publish${model.singularApiName}(revision: $revision) { data { - ${CONTENT_ENTRY_SYSTEM_FIELDS} + ${createEntrySystemFields(model)} ${createFieldsList({ model, fields: model.fields })} } error ${ERROR_FIELD} @@ -482,7 +557,7 @@ export const createUnpublishMutation = (model: CmsEditorContentModel) => { mutation CmsUnpublish${model.singularApiName}($revision: ID!) { content: unpublish${model.singularApiName}(revision: $revision) { data { - ${CONTENT_ENTRY_SYSTEM_FIELDS} + ${createEntrySystemFields(model)} ${createFieldsList({ model, fields: model.fields })} } error ${ERROR_FIELD} diff --git a/packages/app-headless-cms-common/src/index.ts b/packages/app-headless-cms-common/src/index.ts index d2e39ac101e..3020716d964 100644 --- a/packages/app-headless-cms-common/src/index.ts +++ b/packages/app-headless-cms-common/src/index.ts @@ -2,3 +2,4 @@ export * from "./entries.graphql"; export * from "./getModelTitleFieldId"; export * from "./createFieldsList"; export * from "./prepareFormData"; +export * from "./constants"; diff --git a/packages/app-headless-cms/src/HeadlessCMS.tsx b/packages/app-headless-cms/src/HeadlessCMS.tsx index 7df7e45cc0c..3123ab9b252 100644 --- a/packages/app-headless-cms/src/HeadlessCMS.tsx +++ b/packages/app-headless-cms/src/HeadlessCMS.tsx @@ -9,6 +9,7 @@ import apiInformation from "~/admin/plugins/apiInformation"; import { ContentEntriesModule } from "~/admin/views/contentEntries/ContentEntriesModule"; import allPlugins from "~/allPlugins"; import { LexicalEditorCmsPlugin } from "~/admin/components/LexicalCmsEditor/LexicalEditorCmsPlugin"; +import { SingletonContentEntryModule } from "~/admin/views/contentEntries/SingletonContentEntryModule"; interface HeadlessCMSProvider { children: React.ReactNode; @@ -63,6 +64,7 @@ const HeadlessCMSExtension = ({ createApolloClient }: HeadlessCMSProps) => { return ( + diff --git a/packages/app-headless-cms/src/admin/components/ContentEntryForm/ContentEntryForm.tsx b/packages/app-headless-cms/src/admin/components/ContentEntryForm/ContentEntryForm.tsx index a911e28068c..13f5aa2a7d7 100644 --- a/packages/app-headless-cms/src/admin/components/ContentEntryForm/ContentEntryForm.tsx +++ b/packages/app-headless-cms/src/admin/components/ContentEntryForm/ContentEntryForm.tsx @@ -3,9 +3,12 @@ import styled from "@emotion/styled"; import { CmsContentEntry } from "~/types"; import { makeDecoratable } from "@webiny/app-admin"; import { ModelProvider, useModel } from "~/admin/components/ModelProvider"; -import { Header } from "~/admin/components/ContentEntryForm/Header"; import { useFormRenderer } from "~/admin/components/ContentEntryForm/useFormRenderer"; -import { ContentEntryFormContext, ContentEntryFormProvider } from "./ContentEntryFormProvider"; +import { + ContentEntryFormContext, + ContentEntryFormProvider, + PersistEntry +} from "./ContentEntryFormProvider"; import { CustomLayout } from "./CustomLayout"; import { DefaultLayout } from "./DefaultLayout"; import { useGoToRevision } from "~/admin/components/ContentEntryForm/useGoToRevision"; @@ -22,30 +25,25 @@ export interface ContentEntryFormProps { * @param entry */ onAfterCreate?: (entry: CmsContentEntry) => void; - header?: boolean; + /** + * This callback is executed when the form is valid, and it needs to persist the content entry. + */ + persistEntry: PersistEntry; + header?: React.ReactNode; /** * This prop is used to get a reference to `saveEntry` callback, so it can be triggered by components * outside the ContentEntryForm context. * TODO: introduce a `layout` prop to be able to mount arbitrary components around the entry form, within the context. */ setSaveEntry?: (cb: ContentEntryFormContext["saveEntry"]) => void; - /** - * This flag exists for a lack of better Apollo cache control, at the moment. - * We use this flag when we need to tell the system to add new entries to apollo cache. - * Why would you want to NOT add entries to cache? When using a `ref` field, which usually points to - * a different model than the main entry you're working on. Example: Book -> Author, you don't want - * an Author created via a `ref` field dialog to be added to the list of Books. - * TODO: revisit this, and look for a better solution. - */ - addEntryToListCache?: boolean; } export const ContentEntryForm = makeDecoratable( "ContentEntryForm", ({ entry, + persistEntry, onAfterCreate, - addEntryToListCache, setSaveEntry, header = true }: ContentEntryFormProps) => { @@ -75,11 +73,11 @@ export const ContentEntryForm = makeDecoratable( entry={entry} onAfterCreate={onAfterCreate || defaultOnAfterCreate} setSaveEntry={setSaveEntry} - addItemToListCache={addEntryToListCache} confirmNavigationIfDirty={true} + persistEntry={persistEntry} > - {header ?
: null} + {header ? header : null} {formRenderer ? ( diff --git a/packages/app-headless-cms/src/admin/components/ContentEntryForm/ContentEntryFormPreview.tsx b/packages/app-headless-cms/src/admin/components/ContentEntryForm/ContentEntryFormPreview.tsx index c89d3f995cb..35c3bf13a8e 100644 --- a/packages/app-headless-cms/src/admin/components/ContentEntryForm/ContentEntryFormPreview.tsx +++ b/packages/app-headless-cms/src/admin/components/ContentEntryForm/ContentEntryFormPreview.tsx @@ -1,7 +1,7 @@ import React from "react"; import styled from "@emotion/styled"; import { makeDecoratable } from "@webiny/app-admin"; -import { CmsEditorContentModel } from "~/types"; +import { CmsContentEntry, CmsEditorContentModel } from "~/types"; import { ModelProvider } from "~/admin/components/ModelProvider"; import { useFormRenderer } from "~/admin/components/ContentEntryForm/useFormRenderer"; import { CustomLayout } from "~/admin/components/ContentEntryForm/CustomLayout"; @@ -28,6 +28,7 @@ export const ContentEntryFormPreview = makeDecoratable( Promise.resolve({ entry } as { entry: CmsContentEntry })} confirmNavigationIfDirty={false} > diff --git a/packages/app-headless-cms/src/admin/components/ContentEntryForm/ContentEntryFormProvider.tsx b/packages/app-headless-cms/src/admin/components/ContentEntryForm/ContentEntryFormProvider.tsx index 6641be25add..427445b5e6f 100644 --- a/packages/app-headless-cms/src/admin/components/ContentEntryForm/ContentEntryFormProvider.tsx +++ b/packages/app-headless-cms/src/admin/components/ContentEntryForm/ContentEntryFormProvider.tsx @@ -5,8 +5,7 @@ import { Form, FormAPI, FormOnSubmit, FormValidation } from "@webiny/form"; import { CmsContentEntry, CmsModel } from "@webiny/app-headless-cms-common/types"; import { CompositionScope, useSnackbar } from "@webiny/app-admin"; import { prepareFormData } from "@webiny/app-headless-cms-common"; -import { useContentEntry } from "~/index"; -import { PartialCmsContentEntryWithId } from "~/admin/contexts/Cms"; +import { CreateEntryResponse, UpdateEntryRevisionResponse } from "~/admin/contexts/Cms"; const promptMessage = "There are some unsaved changes! Are you sure you want to navigate away and discard all changes?"; @@ -30,22 +29,23 @@ interface InvalidFieldError { error: string; } -interface PersistEntryParams { - entry: PartialCmsContentEntryWithId; - isLocked: boolean; -} - export interface SetSaveEntry { (cb: ContentEntryFormContext["saveEntry"]): void; } +export interface PersistEntry { + (entry: Partial, options?: SaveEntryOptions): Promise< + CreateEntryResponse | UpdateEntryRevisionResponse + >; +} + interface ContentEntryFormProviderProps { entry: Partial; model: CmsModel; + persistEntry: PersistEntry; confirmNavigationIfDirty: boolean; onAfterCreate?: (entry: CmsContentEntry) => void; setSaveEntry?: SetSaveEntry; - addItemToListCache?: boolean; children: React.ReactNode; } @@ -60,15 +60,14 @@ export const ContentEntryFormProvider = ({ model, entry, children, + persistEntry, onAfterCreate, setSaveEntry, - addItemToListCache, confirmNavigationIfDirty }: ContentEntryFormProviderProps) => { const ref = useRef | null>(null); const [invalidFields, setInvalidFields] = useState({}); const { showSnackbar } = useSnackbar(); - const contentEntry = useContentEntry(); const saveOptionsRef = useRef({ skipValidators: undefined }); const saveEntry = useCallback(async (options: SaveEntryOptions = {}) => { @@ -79,32 +78,6 @@ export const ContentEntryFormProvider = ({ }) as unknown as Promise; }, []); - const persistEntry = ({ entry, isLocked }: PersistEntryParams) => { - const options = { - skipValidators: saveOptionsRef.current.skipValidators, - addItemToListCache - }; - - if (!entry.id) { - return contentEntry.createEntry({ entry, options }); - } - - if (!isLocked) { - return contentEntry.updateEntryRevision({ - entry, - options: { skipValidators: options?.skipValidators } - }); - } - - const { id, ...input } = entry; - - return contentEntry.createEntryRevisionFrom({ - id, - input, - options: { skipValidators: options?.skipValidators } - }); - }; - const onFormSubmit: FormOnSubmit = async data => { const fieldsIds = model.fields.map(item => item.fieldId); const formData = pick(data, [...fieldsIds]); @@ -112,10 +85,12 @@ export const ContentEntryFormProvider = ({ const gqlData = prepareFormData(formData, model.fields) as Partial; const isNewEntry = data.id === undefined; - const { entry, error } = await persistEntry({ - entry: { id: data.id, ...gqlData }, - isLocked: data.meta?.locked === true - }); + const { entry, error } = await persistEntry( + { id: data.id, ...gqlData }, + { + skipValidators: saveOptionsRef.current.skipValidators + } + ); if (error) { showSnackbar(error.message); diff --git a/packages/app-headless-cms/src/admin/components/ContentEntryForm/SingletonHeader/SaveAction.tsx b/packages/app-headless-cms/src/admin/components/ContentEntryForm/SingletonHeader/SaveAction.tsx new file mode 100644 index 00000000000..d8d230d5edc --- /dev/null +++ b/packages/app-headless-cms/src/admin/components/ContentEntryForm/SingletonHeader/SaveAction.tsx @@ -0,0 +1,17 @@ +import React from "react"; +import { ContentEntryEditorConfig } from "~/ContentEntryEditorConfig"; +import { useContentEntryForm } from "~/admin/components/ContentEntryForm/useContentEntryForm"; + +const { Actions } = ContentEntryEditorConfig; + +export const SaveAction = () => { + const { useButtons } = Actions.ButtonAction; + const { ButtonPrimary } = useButtons(); + const { saveEntry } = useContentEntryForm(); + + return ( + + Save + + ); +}; diff --git a/packages/app-headless-cms/src/admin/components/ContentEntryForm/SingletonHeader/SingletonHeader.tsx b/packages/app-headless-cms/src/admin/components/ContentEntryForm/SingletonHeader/SingletonHeader.tsx new file mode 100644 index 00000000000..ecb97bdb08d --- /dev/null +++ b/packages/app-headless-cms/src/admin/components/ContentEntryForm/SingletonHeader/SingletonHeader.tsx @@ -0,0 +1,38 @@ +import React from "react"; +import { Buttons } from "@webiny/app-admin"; +import styled from "@emotion/styled"; +import { SaveAction } from "./SaveAction"; + +const ToolbarGrid = styled.div` + padding: 15px; + border-bottom: 1px solid var(--mdc-theme-on-background); + display: flex; + justify-content: space-between; + align-items: center; +`; + +const Actions = styled.div` + display: flex; + align-items: center; +`; + +const ModelName = styled.div` + font-family: var(--mdc-typography-font-family); + padding: 10px 0; + font-size: 24px; +`; + +export interface SingletonHeaderProps { + title: string; +} + +export const SingletonHeader = ({ title }: SingletonHeaderProps) => { + return ( + + {title} + + }]} /> + + + ); +}; diff --git a/packages/app-headless-cms/src/admin/components/ContentEntryForm/SingletonHeader/index.tsx b/packages/app-headless-cms/src/admin/components/ContentEntryForm/SingletonHeader/index.tsx new file mode 100644 index 00000000000..64d3a6148fc --- /dev/null +++ b/packages/app-headless-cms/src/admin/components/ContentEntryForm/SingletonHeader/index.tsx @@ -0,0 +1 @@ +export * from "./SingletonHeader"; diff --git a/packages/app-headless-cms/src/admin/contexts/Cms/index.tsx b/packages/app-headless-cms/src/admin/contexts/Cms/index.tsx index 322610e7f88..71e00be2245 100644 --- a/packages/app-headless-cms/src/admin/contexts/Cms/index.tsx +++ b/packages/app-headless-cms/src/admin/contexts/Cms/index.tsx @@ -25,7 +25,12 @@ import { CmsEntryCreateFromMutationResponse, CmsEntryCreateFromMutationVariables, CmsEntryGetQueryResponse, - CmsEntryGetQueryVariables + CmsEntryGetQueryVariables, + createReadSingletonQuery, + CmsEntryGetSingletonQueryResponse, + createUpdateSingletonMutation, + CmsEntryUpdateSingletonMutationResponse, + CmsEntryUpdateSingletonMutationVariables } from "@webiny/app-headless-cms-common"; import { getFetchPolicy } from "~/utils/getFetchPolicy"; @@ -56,7 +61,7 @@ export type UnpublishEntryRevisionResponse = OperationSuccess | OperationError; export interface CreateEntryParams { model: CmsModel; - entry: PartialCmsContentEntryWithId; + entry: Partial; options?: { skipValidators?: string[]; }; @@ -79,6 +84,14 @@ export interface UpdateEntryRevisionParams { }; } +export interface UpdateSingletonEntryParams { + model: CmsModel; + entry: PartialCmsContentEntryWithId; + options?: { + skipValidators?: string[]; + }; +} + export interface PublishEntryRevisionParams { model: CmsModel; id: string; @@ -98,15 +111,23 @@ export interface GetEntryParams { id: string; } +export interface GetSingletonEntryParams { + model: CmsModel; +} + export interface CmsContext { getApolloClient(locale: string): ApolloClient; createApolloClient: CmsProviderProps["createApolloClient"]; apolloClient: ApolloClient; getEntry: (params: GetEntryParams) => Promise; + getSingletonEntry: (params: GetSingletonEntryParams) => Promise; createEntry: (params: CreateEntryParams) => Promise; createEntryRevisionFrom: ( params: CreateEntryRevisionFromParams ) => Promise; + updateSingletonEntry: ( + params: UpdateSingletonEntryParams + ) => Promise; updateEntryRevision: ( params: UpdateEntryRevisionParams ) => Promise; @@ -177,7 +198,35 @@ export const CmsProvider = (props: CmsProviderProps) => { if (!response.data) { return { error: { - message: "Missing response data on Get Entry query.", + message: "Missing response data on getEntry query.", + code: "MISSING_RESPONSE_DATA", + data: {} + } + }; + } + + const { data, error } = response.data.content; + + if (error) { + return { error }; + } + + return { + entry: data as CmsContentEntry + }; + }, + getSingletonEntry: async ({ model }) => { + const query = createReadSingletonQuery(model); + + const response = await value.apolloClient.query({ + query, + fetchPolicy: getFetchPolicy(model) + }); + + if (!response.data) { + return { + error: { + message: "Missing response data on getSingletonEntry query.", code: "MISSING_RESPONSE_DATA", data: {} } @@ -299,6 +348,42 @@ export const CmsProvider = (props: CmsProviderProps) => { entry: data as CmsContentEntry }; }, + updateSingletonEntry: async ({ model, entry, options }) => { + const mutation = createUpdateSingletonMutation(model); + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const { id, ...input } = entry; + const response = await value.apolloClient.mutate< + CmsEntryUpdateSingletonMutationResponse, + CmsEntryUpdateSingletonMutationVariables + >({ + mutation, + variables: { + data: input, + options + }, + fetchPolicy: getFetchPolicy(model) + }); + + if (!response.data) { + return { + error: { + message: "Missing response data on updateSingletonEntry mutation.", + code: "MISSING_RESPONSE_DATA", + data: {} + } + }; + } + + const { data, error } = response.data.content; + + if (error) { + return { error }; + } + + return { + entry: data as CmsContentEntry + }; + }, publishEntryRevision: async ({ model, id }) => { const mutation = createPublishMutation(model); const response = await value.apolloClient.mutate< diff --git a/packages/app-headless-cms/src/admin/hooks/usePersistEntry.ts b/packages/app-headless-cms/src/admin/hooks/usePersistEntry.ts new file mode 100644 index 00000000000..f5cad481fe0 --- /dev/null +++ b/packages/app-headless-cms/src/admin/hooks/usePersistEntry.ts @@ -0,0 +1,50 @@ +import { useContentEntry } from "~/admin/views/contentEntries/hooks"; +import { CmsContentEntry } from "@webiny/app-headless-cms-common/types"; +import { PartialCmsContentEntryWithId } from "~/admin/contexts/Cms"; +import { useCallback } from "react"; + +interface UsePersistEntryOptions { + addItemToListCache?: boolean; +} + +interface PersistEntryOptions { + skipValidators?: string[]; +} + +export function usePersistEntry({ addItemToListCache }: UsePersistEntryOptions) { + const contentEntry = useContentEntry(); + + const persistEntry = useCallback( + (entry: Partial, persistOptions?: PersistEntryOptions) => { + const isLocked = entry.meta?.locked === true; + + if (!entry.id) { + return contentEntry.createEntry({ + entry, + options: { + skipValidators: persistOptions?.skipValidators, + addItemToListCache + } + }); + } + + if (!isLocked) { + return contentEntry.updateEntryRevision({ + entry: entry as PartialCmsContentEntryWithId, + options: { skipValidators: persistOptions?.skipValidators } + }); + } + + const { id, ...input } = entry; + + return contentEntry.createEntryRevisionFrom({ + id, + input, + options: { skipValidators: persistOptions?.skipValidators } + }); + }, + [addItemToListCache] + ); + + return { persistEntry }; +} diff --git a/packages/app-headless-cms/src/admin/plugins/fieldRenderers/ref/components/NewReferencedEntryDialog.tsx b/packages/app-headless-cms/src/admin/plugins/fieldRenderers/ref/components/NewReferencedEntryDialog.tsx index 3b77d0ba06b..8440f8bd78e 100644 --- a/packages/app-headless-cms/src/admin/plugins/fieldRenderers/ref/components/NewReferencedEntryDialog.tsx +++ b/packages/app-headless-cms/src/admin/plugins/fieldRenderers/ref/components/NewReferencedEntryDialog.tsx @@ -22,6 +22,7 @@ import styled from "@emotion/styled"; import { Elevation } from "@webiny/ui/Elevation"; import { SplitView, LeftPanel, RightPanel } from "@webiny/app-admin/components/SplitView"; import { CircularProgress } from "@webiny/ui/Progress"; +import { usePersistEntry } from "~/admin/hooks/usePersistEntry"; const t = i18n.ns("app-headless-cms/admin/fields/ref"); @@ -77,6 +78,7 @@ interface EntryFormProps { const EntryForm = ({ onCreate, setSaveEntry }: EntryFormProps) => { const { contentModel, loading } = useContentEntry(); + const { persistEntry } = usePersistEntry({ addItemToListCache: false }); const { currentFolderId, navigateToFolder } = useNavigateFolder(); return ( @@ -96,9 +98,9 @@ const EntryForm = ({ onCreate, setSaveEntry }: EntryFormProps) => { {loading ? : null} onCreate(entry)} entry={{}} - addEntryToListCache={false} + persistEntry={persistEntry} + onAfterCreate={entry => onCreate(entry)} setSaveEntry={setSaveEntry} /> diff --git a/packages/app-headless-cms/src/admin/plugins/fields/ref.tsx b/packages/app-headless-cms/src/admin/plugins/fields/ref.tsx index 34687e43524..b54022b1f57 100644 --- a/packages/app-headless-cms/src/admin/plugins/fields/ref.tsx +++ b/packages/app-headless-cms/src/admin/plugins/fields/ref.tsx @@ -5,12 +5,13 @@ import { validation, ValidationError } from "@webiny/validation"; import { Cell, Grid } from "@webiny/ui/Grid"; import { MultiAutoComplete } from "@webiny/ui/AutoComplete"; import { useSnackbar } from "@webiny/app-admin/hooks/useSnackbar"; -import { CmsModelFieldTypePlugin, CmsModel } from "~/types"; +import { CmsModel, CmsModelFieldTypePlugin } from "~/types"; import { ReactComponent as RefIcon } from "./icons/round-link-24px.svg"; import { i18n } from "@webiny/app/i18n"; import { Bind, BindComponentRenderProp, useForm } from "@webiny/form"; -import { useQuery, useModel } from "~/admin/hooks"; +import { useModel, useQuery } from "~/admin/hooks"; import { renderInfo } from "./ref/renderInfo"; +import { CMS_MODEL_SINGLETON_TAG } from "@webiny/app-headless-cms-common"; const t = i18n.ns("app-headless-cms/admin/fields"); @@ -34,9 +35,18 @@ const RefFieldSettings = () => { // Format options for the Autocomplete component. const options = useMemo(() => { const models = get(data, "listContentModels.data", []) as CmsModel[]; - return models.map(model => { - return { id: model.modelId, name: model.name }; - }); + return ( + models + /** + * Remove singleton models from the list of options. + */ + .filter(model => { + return !model.tags?.includes(CMS_MODEL_SINGLETON_TAG); + }) + .map(model => { + return { id: model.modelId, name: model.name }; + }) + ); }, [data]); const atLeastOneItem = useCallback(async (value: Pick) => { diff --git a/packages/app-headless-cms/src/admin/views/contentEntries/ContentEntry.tsx b/packages/app-headless-cms/src/admin/views/contentEntries/ContentEntry.tsx index d4bf9f68c25..5f625d758e3 100644 --- a/packages/app-headless-cms/src/admin/views/contentEntries/ContentEntry.tsx +++ b/packages/app-headless-cms/src/admin/views/contentEntries/ContentEntry.tsx @@ -4,10 +4,12 @@ import styled from "@emotion/styled"; import { Tab, Tabs } from "@webiny/ui/Tabs"; import { Elevation } from "@webiny/ui/Elevation"; import { CircularProgress } from "@webiny/ui/Progress"; +import { makeDecoratable } from "@webiny/app"; import { RevisionsList } from "./ContentEntry/RevisionsList/RevisionsList"; import { useContentEntry } from "./hooks/useContentEntry"; +import { Header } from "~/admin/components/ContentEntryForm/Header"; import { ContentEntryForm } from "~/admin/components/ContentEntryForm/ContentEntryForm"; -import { makeDecoratable } from "@webiny/app"; +import { usePersistEntry } from "~/admin/hooks/usePersistEntry"; const DetailsContainer = styled("div")({ height: "calc(100% - 10px)", @@ -44,6 +46,7 @@ declare global { export const ContentEntry = makeDecoratable("ContentEntry", () => { const { loading, entry, activeTab, setActiveTab } = useContentEntry(); + const { persistEntry } = usePersistEntry({ addItemToListCache: true }); return ( @@ -57,7 +60,11 @@ export const ContentEntry = makeDecoratable("ContentEntry", () => { {loading && } - + } + /> diff --git a/packages/app-headless-cms/src/admin/views/contentEntries/ContentEntry/SingletonContentEntry.tsx b/packages/app-headless-cms/src/admin/views/contentEntries/ContentEntry/SingletonContentEntry.tsx new file mode 100644 index 00000000000..354cdfbe380 --- /dev/null +++ b/packages/app-headless-cms/src/admin/views/contentEntries/ContentEntry/SingletonContentEntry.tsx @@ -0,0 +1,39 @@ +import React from "react"; +import styled from "@emotion/styled"; +import { Elevation as BaseElevation } from "@webiny/ui/Elevation"; +import { CircularProgress } from "@webiny/ui/Progress"; +import { ContentEntryForm } from "~/admin/components/ContentEntryForm/ContentEntryForm"; +import { makeDecoratable } from "@webiny/app"; +import { useSingletonContentEntry } from "~/admin/views/contentEntries/hooks/useSingletonContentEntry"; +import { PartialCmsContentEntryWithId } from "~/admin/contexts/Cms"; +import { SingletonHeader } from "~/admin/components/ContentEntryForm/SingletonHeader"; + +const Elevation = styled(BaseElevation)` + height: 100%; + flex: 0 0 50%; +`; + +const Container = styled.div` + display: flex; + padding: 25px; + justify-content: center; +`; + +export const SingletonContentEntry = makeDecoratable("SingletonContentEntry", () => { + const { loading, entry, updateEntry, contentModel } = useSingletonContentEntry(); + + return ( + + + {loading && } + } + entry={entry} + persistEntry={entry => + updateEntry({ entry: entry as PartialCmsContentEntryWithId }) + } + /> + + + ); +}); diff --git a/packages/app-headless-cms/src/admin/views/contentEntries/ContentEntry/SingletonContentEntryContext.tsx b/packages/app-headless-cms/src/admin/views/contentEntries/ContentEntry/SingletonContentEntryContext.tsx new file mode 100644 index 00000000000..89cbb0d6c5f --- /dev/null +++ b/packages/app-headless-cms/src/admin/views/contentEntries/ContentEntry/SingletonContentEntryContext.tsx @@ -0,0 +1,78 @@ +import React, { useEffect, useState } from "react"; +import { useSnackbar, useIsMounted } from "@webiny/app-admin"; +import { useCms } from "~/admin/hooks"; +import { useContentEntries } from "~/admin/views/contentEntries/hooks/useContentEntries"; +import { CmsContentEntry, CmsModel } from "~/types"; +import * as Cms from "~/admin/contexts/Cms"; + +export type UpdateEntryParams = Omit; + +export interface SingletonContentEntryCrud { + updateEntry: (params: UpdateEntryParams) => Promise; +} + +export interface SingletonContentEntryContext extends SingletonContentEntryCrud { + contentModel: CmsModel; + entry: CmsContentEntry; + loading: boolean; +} + +export const SingletonContentEntryContext = React.createContext< + SingletonContentEntryContext | undefined +>(undefined); + +export interface ContentEntryContextProviderProps { + children: React.ReactNode; +} + +export const SingletonContentEntryProvider = ({ children }: ContentEntryContextProviderProps) => { + const { isMounted } = useIsMounted(); + const [entry, setEntry] = useState(); + const { contentModel: model } = useContentEntries(); + const { showSnackbar } = useSnackbar(); + const cms = useCms(); + const [isLoading, setLoading] = useState(false); + + useEffect(() => { + (async () => { + setLoading(true); + const { entry, error } = await cms.getSingletonEntry({ model }); + setLoading(false); + + if (!isMounted()) { + return; + } + + if (!error) { + setEntry(entry); + return; + } + showSnackbar(error.message); + })(); + }, []); + + const updateEntry: SingletonContentEntryCrud["updateEntry"] = async params => { + setLoading(true); + const response = await cms.updateSingletonEntry({ model, ...params }); + setLoading(false); + if (response.entry) { + setEntry(response.entry); + } + return response; + }; + + const value: SingletonContentEntryContext = { + contentModel: model, + entry: (entry || {}) as CmsContentEntry, + loading: isLoading, + updateEntry + }; + + return ( + + {children} + + ); +}; + +SingletonContentEntryProvider.displayName = "SingletonContentEntryProvider"; diff --git a/packages/app-headless-cms/src/admin/views/contentEntries/SingletonContentEntryModule.tsx b/packages/app-headless-cms/src/admin/views/contentEntries/SingletonContentEntryModule.tsx new file mode 100644 index 00000000000..8ee9dd7ba7c --- /dev/null +++ b/packages/app-headless-cms/src/admin/views/contentEntries/SingletonContentEntryModule.tsx @@ -0,0 +1,36 @@ +import React from "react"; +import { ContentEntries } from "~/admin/views/contentEntries/ContentEntries"; +import { CMS_MODEL_SINGLETON_TAG } from "@webiny/app-headless-cms-common"; +import { ContentEntriesProvider } from "~/admin/views/contentEntries/ContentEntriesContext"; +import { DialogsProvider } from "@webiny/app-admin"; +import { SingletonContentEntryProvider } from "~/admin/views/contentEntries/ContentEntry/SingletonContentEntryContext"; +import { SingletonContentEntry } from "~/admin/views/contentEntries/ContentEntry/SingletonContentEntry"; +import { useModel } from "~/admin/components/ModelProvider"; + +const ContentEntriesDecorator = ContentEntries.createDecorator(Original => { + return function ContentEntries() { + const { model } = useModel(); + + if (model.tags.includes(CMS_MODEL_SINGLETON_TAG)) { + return ( + + + + + + + + ); + } + + return ; + }; +}); + +export const SingletonContentEntryModule = () => { + return ( + <> + + + ); +}; diff --git a/packages/app-headless-cms/src/admin/views/contentEntries/hooks/useSingletonContentEntry.ts b/packages/app-headless-cms/src/admin/views/contentEntries/hooks/useSingletonContentEntry.ts new file mode 100644 index 00000000000..299c1cd3afe --- /dev/null +++ b/packages/app-headless-cms/src/admin/views/contentEntries/hooks/useSingletonContentEntry.ts @@ -0,0 +1,13 @@ +import { useContext } from "react"; +import { makeDecoratable } from "@webiny/app-admin"; +import { SingletonContentEntryContext } from "../ContentEntry/SingletonContentEntryContext"; + +export const useSingletonContentEntry = makeDecoratable(() => { + const context = useContext(SingletonContentEntryContext); + if (!context) { + throw Error( + `useSingletonContentEntry() hook can only be used within the SingletonContentEntryContext provider.` + ); + } + return context; +}); diff --git a/packages/app-headless-cms/src/admin/views/contentModels/ContentModelsDataList.tsx b/packages/app-headless-cms/src/admin/views/contentModels/ContentModelsDataList.tsx index 3ea13dbeee3..e52925f20e2 100644 --- a/packages/app-headless-cms/src/admin/views/contentModels/ContentModelsDataList.tsx +++ b/packages/app-headless-cms/src/admin/views/contentModels/ContentModelsDataList.tsx @@ -34,6 +34,7 @@ import { OptionsMenu } from "./OptionsMenu"; import { ReactComponent as DownloadFileIcon } from "@webiny/app-admin/assets/icons/file_download.svg"; import { ReactComponent as UploadFileIcon } from "@webiny/app-admin/assets/icons/file_upload.svg"; import { useModelExport } from "./exporting/useModelExport"; +import { CMS_MODEL_SINGLETON_TAG } from "@webiny/app-headless-cms-common"; const t = i18n.namespace("FormsApp.ContentModelsDataList"); @@ -120,7 +121,20 @@ const ContentModelsDataList = ({ const client = useApolloClient(); const { showSnackbar } = useSnackbar(); const { showConfirmation } = useConfirmationDialog({ - dataTestId: "cms-delete-content-model-dialog" + dataTestId: "cms-delete-content-model-dialog", + title: "Delete a content model" + }); + + const { showConfirmation: showConfirmationSingleton } = useConfirmationDialog({ + dataTestId: "cms-delete-singleton-content-model-dialog", + title: "Delete a single entry model", + message: ( +

+ Deleting a single entry content model will also delete the entry in the model. +
+ Are you sure you want to delete this content model? +

+ ) }); const { models, loading, refresh } = useModels(); const { canDelete, canEdit } = usePermission(); @@ -144,7 +158,11 @@ const ContentModelsDataList = ({ ); const deleteRecord = async (item: CmsModel): Promise => { - showConfirmation(async () => { + const confirmation = item.tags.includes(CMS_MODEL_SINGLETON_TAG) + ? showConfirmationSingleton + : showConfirmation; + + confirmation(async () => { await client.mutate({ mutation: GQL.DELETE_CONTENT_MODEL, variables: { modelId: item.modelId }, diff --git a/packages/app-headless-cms/src/admin/views/contentModels/NewContentModelDialog.tsx b/packages/app-headless-cms/src/admin/views/contentModels/NewContentModelDialog.tsx index c540bdd452c..66ad6fecb33 100644 --- a/packages/app-headless-cms/src/admin/views/contentModels/NewContentModelDialog.tsx +++ b/packages/app-headless-cms/src/admin/views/contentModels/NewContentModelDialog.tsx @@ -6,11 +6,12 @@ import { Select } from "@webiny/ui/Select"; import { useSnackbar } from "@webiny/app-admin/hooks/useSnackbar"; import { CircularProgress } from "@webiny/ui/Progress"; import { validation } from "@webiny/validation"; -import { useQuery, useMutation, useApolloClient } from "../../hooks"; +import { useApolloClient, useMutation, useQuery } from "../../hooks"; import { i18n } from "@webiny/app/i18n"; import { ButtonPrimary } from "@webiny/ui/Button"; import * as UID from "@webiny/ui/Dialog"; -import { Grid, Cell } from "@webiny/ui/Grid"; +import { Cell, Grid } from "@webiny/ui/Grid"; +import { CMS_MODEL_SINGLETON_TAG } from "@webiny/app-headless-cms-common"; import { addModelToGroupCache, addModelToListCache } from "./cache"; import * as GQL from "../../viewsGraphql"; import { @@ -25,6 +26,7 @@ import { createApiNameValidator } from "~/admin/views/contentModels/helpers/apiN import { createNameValidator } from "~/admin/views/contentModels/helpers/nameValidator"; import { Checkbox } from "@webiny/ui/Checkbox"; import { IconPicker } from "~/admin/components/IconPicker"; +import { Switch } from "@webiny/ui/Switch"; const t = i18n.ns("app-headless-cms/admin/views/content-models/new-content-model-dialog"); @@ -37,6 +39,10 @@ interface CmsModelData { name: string; description: string; group: string; + singleton?: boolean; + singularApiName: string; + pluralApiName: string; + defaultFields: boolean; } const NewContentModelDialog = ({ open, onClose }: NewContentModelDialogProps) => { @@ -117,135 +123,183 @@ const NewContentModelDialog = ({ open, onClose }: NewContentModelDialogProps) => models ]); - const group = contentModelGroups.length > 0 ? contentModelGroups[0].value : null; + const group = useMemo(() => { + if (!contentModelGroups.length) { + return undefined; + } + return contentModelGroups[0]?.value; + }, [contentModelGroups]); - const onSubmit = async (data: CmsModelData) => { - setLoading(true); - await createContentModel({ - variables: { data } - }); - }; + const onSubmit = useCallback( + async (data: CmsModelData) => { + setLoading(true); + /** + * We need to make sure that tags are always an array. + * At the moment there is no tags on the CmsModelData type. + * If it is added at some point, the @ts-expect-error should be removed - it will cause TS error. + */ + // @ts-expect-error + const tags: string[] = Array.isArray(data.tags) ? data.tags : []; + /** + * If a model is a singleton, we add a special tag to it. + * + we need to put the pluralApiName to something that is not used. + */ + if (data.singleton) { + tags.push(CMS_MODEL_SINGLETON_TAG); + data.pluralApiName = `${data.singularApiName}Unused`; + } + delete data.singleton; + await createContentModel({ + variables: { + data: { + ...data, + tags + } + } + }); + }, + [loading, createContentModel] + ); return ( {open && ( -
+ data={{ group, singleton: false }} onSubmit={data => { - /** - * We are positive that data is CmsModelData. - */ - onSubmit(data as unknown as CmsModelData); + console.log("submitting", data); + onSubmit(data); }} > - {({ Bind, submit }) => ( - <> - {loading && } - {t`New Content Model`} - - - - - - - - - - - - - - - - - - - - + + + + - )} - - - - - - - - - - - { - submit(ev); - }} - > - + {t`Create Model`} - - - - )} + + + + + + + + + + + + + + + + )} + + + + + + + + + + + { + console.log("submitting click", data); + submit(ev); + }} + > + + {t`Create Model`} + + + + ); + }} )}
diff --git a/packages/db-dynamodb/package.json b/packages/db-dynamodb/package.json index dfa822a6d89..1692cfd92f4 100644 --- a/packages/db-dynamodb/package.json +++ b/packages/db-dynamodb/package.json @@ -20,7 +20,6 @@ "dot-prop": "^6.0.1", "dynamodb-toolbox": "^0.9.2", "fuse.js": "7.0.0", - "is-number": "^7.0.0", "lodash": "^4.17.21" }, "devDependencies": { diff --git a/packages/db-dynamodb/src/plugins/definitions/NumberTransformPlugin.ts b/packages/db-dynamodb/src/plugins/definitions/NumberTransformPlugin.ts deleted file mode 100644 index 8f88eadf180..00000000000 --- a/packages/db-dynamodb/src/plugins/definitions/NumberTransformPlugin.ts +++ /dev/null @@ -1,30 +0,0 @@ -import { - ValueTransformPlugin, - ValueTransformPluginParams, - ValueTransformPluginParamsTransformParams -} from "./ValueTransformPlugin"; -import WebinyError from "@webiny/error"; -import isNumber from "is-number"; - -const transformNumber = (params: ValueTransformPluginParamsTransformParams): number => { - const { value } = params; - const typeOf = typeof value; - /** - * Due to some internal JS stuff, we must check for a number like this. - */ - if (typeOf === "number" || isNumber(value) === true) { - return Number(value); - } - throw new WebinyError("Field value must be a number because.", "NUMBER_ERROR", { - value - }); -}; - -export class NumberTransformPlugin extends ValueTransformPlugin { - public constructor(params: Omit) { - super({ - transform: transformNumber, - ...params - }); - } -} diff --git a/packages/db-dynamodb/src/plugins/definitions/TimeTransformPlugin.ts b/packages/db-dynamodb/src/plugins/definitions/TimeTransformPlugin.ts index 5604fd235b9..0f2c28d7e54 100644 --- a/packages/db-dynamodb/src/plugins/definitions/TimeTransformPlugin.ts +++ b/packages/db-dynamodb/src/plugins/definitions/TimeTransformPlugin.ts @@ -4,19 +4,24 @@ import { ValueTransformPluginParamsTransformParams } from "./ValueTransformPlugin"; import WebinyError from "@webiny/error"; -import isNumber from "is-number"; const transformTime = (params: ValueTransformPluginParamsTransformParams): number => { const { value } = params; if (value === undefined || value === null) { - throw new WebinyError(`Time value is null or undefined`, "TIME_PARSE_ERROR", { + throw new WebinyError(`Time value is null or undefined.`, "TIME_PARSE_ERROR", { value }); + } else if (typeof value === "boolean" || value === "" || Array.isArray(value)) { + throw new WebinyError( + "Field value must be a string because field is defined as time.", + "TIME_PARSE_ERROR", + { + value + } + ); } - /** - * Due to some internal JS stuff, we must check for a number like this. - */ - if (typeof value === "number" || isNumber(value) === true) { + const converted = Number(`${value}`); + if (typeof value === "number" || isNaN(converted) === false) { return Number(value); } else if (typeof value !== "string") { throw new WebinyError( diff --git a/yarn.lock b/yarn.lock index 91afb243edd..6f585502cd7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -18930,7 +18930,6 @@ __metadata: dot-prop: ^6.0.1 dynamodb-toolbox: ^0.9.2 fuse.js: 7.0.0 - is-number: ^7.0.0 jest: ^29.7.0 jest-dynalite: ^3.2.0 lodash: ^4.17.21 From b988ac6b89173e2dce9185408acdc584b9d81004 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20Zori=C4=87?= Date: Tue, 27 Aug 2024 12:28:44 +0200 Subject: [PATCH 19/70] fix(api-headless-cms): filtering ref fields with null values (#4237) --- .../src/plugins/operator/equal.ts | 2 +- .../src/plugins/operator/not.ts | 2 +- .../filtering/plugins/refFilterPlugin.ts | 16 ++- packages/api-headless-cms-ddb/package.json | 1 + .../filtering/plugins/refFilterCreate.ts | 12 +- .../src/plugins/CmsEntryFieldFilterPlugin.ts | 16 +-- .../api-headless-cms-ddb/tsconfig.build.json | 1 + packages/api-headless-cms-ddb/tsconfig.json | 3 + .../__tests__/contentAPI/refField.test.ts | 129 +++++++++++++++++- packages/app-file-manager/tsconfig.build.json | 2 +- packages/app-file-manager/tsconfig.json | 6 +- .../tsconfig.json | 12 +- .../db-dynamodb/src/plugins/filters/eq.ts | 4 +- packages/pulumi-aws/tsconfig.build.json | 4 +- packages/pulumi-aws/tsconfig.json | 10 +- yarn.lock | 1 + 16 files changed, 183 insertions(+), 38 deletions(-) diff --git a/packages/api-elasticsearch/src/plugins/operator/equal.ts b/packages/api-elasticsearch/src/plugins/operator/equal.ts index 8aaed119139..879e8db291b 100644 --- a/packages/api-elasticsearch/src/plugins/operator/equal.ts +++ b/packages/api-elasticsearch/src/plugins/operator/equal.ts @@ -14,7 +14,7 @@ export class ElasticsearchQueryBuilderOperatorEqualPlugin extends ElasticsearchQ ): void { const { value, path, basePath } = params; - if (value === null) { + if (value === null || value === undefined) { query.must_not.push({ exists: { field: path diff --git a/packages/api-elasticsearch/src/plugins/operator/not.ts b/packages/api-elasticsearch/src/plugins/operator/not.ts index dc5a7523f23..1e918b8de5f 100644 --- a/packages/api-elasticsearch/src/plugins/operator/not.ts +++ b/packages/api-elasticsearch/src/plugins/operator/not.ts @@ -14,7 +14,7 @@ export class ElasticsearchQueryBuilderOperatorNotPlugin extends ElasticsearchQue ): void { const { value, path, basePath } = params; - if (value === null) { + if (value === null || value === undefined) { query.filter.push({ exists: { field: path diff --git a/packages/api-headless-cms-ddb-es/src/operations/entry/elasticsearch/filtering/plugins/refFilterPlugin.ts b/packages/api-headless-cms-ddb-es/src/operations/entry/elasticsearch/filtering/plugins/refFilterPlugin.ts index 1050e9adbf1..3ebea2f5527 100644 --- a/packages/api-headless-cms-ddb-es/src/operations/entry/elasticsearch/filtering/plugins/refFilterPlugin.ts +++ b/packages/api-headless-cms-ddb-es/src/operations/entry/elasticsearch/filtering/plugins/refFilterPlugin.ts @@ -3,10 +3,12 @@ import { CmsEntryFilterPlugin } from "~/plugins/CmsEntryFilterPlugin"; import { parseWhereKey } from "@webiny/api-elasticsearch"; export const createRefFilterPlugin = () => { - return new CmsEntryFilterPlugin({ + const plugin = new CmsEntryFilterPlugin({ fieldType: "ref", exec: params => { - const { applyFiltering, value: values, query, field } = params; + const { applyFiltering, query, field } = params; + + let values = params.value; /** * We must have an object when querying in the ref field. */ @@ -20,6 +22,12 @@ export const createRefFilterPlugin = () => { ); } + if (values === null || values === undefined) { + values = { + entryId: null + }; + } + for (const key in values) { const { operator } = parseWhereKey(key); const value = values[key]; @@ -37,4 +45,8 @@ export const createRefFilterPlugin = () => { } } }); + + plugin.name = `${plugin.type}.default.ref`; + + return plugin; }; diff --git a/packages/api-headless-cms-ddb/package.json b/packages/api-headless-cms-ddb/package.json index 21b54213125..96e94c4f676 100644 --- a/packages/api-headless-cms-ddb/package.json +++ b/packages/api-headless-cms-ddb/package.json @@ -23,6 +23,7 @@ "license": "MIT", "dependencies": { "@babel/runtime": "^7.24.0", + "@webiny/api": "0.0.0", "@webiny/api-headless-cms": "0.0.0", "@webiny/aws-sdk": "0.0.0", "@webiny/db-dynamodb": "0.0.0", diff --git a/packages/api-headless-cms-ddb/src/operations/entry/filtering/plugins/refFilterCreate.ts b/packages/api-headless-cms-ddb/src/operations/entry/filtering/plugins/refFilterCreate.ts index 37d29dc94d1..962cda6d476 100644 --- a/packages/api-headless-cms-ddb/src/operations/entry/filtering/plugins/refFilterCreate.ts +++ b/packages/api-headless-cms-ddb/src/operations/entry/filtering/plugins/refFilterCreate.ts @@ -5,17 +5,23 @@ import { } from "~/plugins/CmsEntryFieldFilterPlugin"; import { extractWhereParams } from "~/operations/entry/filtering/where"; import { transformValue } from "~/operations/entry/filtering/transform"; +import { GenericRecord } from "@webiny/api/types"; export const createRefFilterCreate = () => { - const plugin = new CmsEntryFieldFilterPlugin({ + const plugin = new CmsEntryFieldFilterPlugin({ fieldType: "ref", create: params => { - const { value, valueFilterPlugins, transformValuePlugins, field } = params; + const { valueFilterPlugins, transformValuePlugins, field } = params; + let value = params.value; + if (!value) { + value = { + entryId: null + }; + } const propertyFilters = Object.keys(value); if (propertyFilters.length === 0) { return null; } - const filters: CmsEntryFieldFilterPluginCreateResponse[] = []; for (const propertyFilter of propertyFilters) { diff --git a/packages/api-headless-cms-ddb/src/plugins/CmsEntryFieldFilterPlugin.ts b/packages/api-headless-cms-ddb/src/plugins/CmsEntryFieldFilterPlugin.ts index 81fa14a9c77..1ba4db03aef 100644 --- a/packages/api-headless-cms-ddb/src/plugins/CmsEntryFieldFilterPlugin.ts +++ b/packages/api-headless-cms-ddb/src/plugins/CmsEntryFieldFilterPlugin.ts @@ -8,16 +8,16 @@ import { CmsFieldFilterValueTransformPlugin } from "~/types"; * Internally we have default one + the one for the reference field - because it is actually an object when filtering. */ -interface CmsEntryFieldFilterPluginParams { +interface CmsEntryFieldFilterPluginParams { fieldType: string; create: ( - params: CmsEntryFieldFilterPluginCreateParams + params: CmsEntryFieldFilterPluginCreateParams ) => null | CmsEntryFieldFilterPluginCreateResponse | CmsEntryFieldFilterPluginCreateResponse[]; } -interface CmsEntryFieldFilterPluginCreateParams { +interface CmsEntryFieldFilterPluginCreateParams { key: string; - value: any; + value: T; field: Field; fields: Record; operation: string; @@ -39,21 +39,21 @@ export interface CmsEntryFieldFilterPluginCreateResponse { transformValue: (value: I) => O; } -export class CmsEntryFieldFilterPlugin extends Plugin { +export class CmsEntryFieldFilterPlugin extends Plugin { public static override readonly type: string = "cms.dynamodb.entry.field.filter"; public static readonly ALL: string = "*"; - private readonly config: CmsEntryFieldFilterPluginParams; + private readonly config: CmsEntryFieldFilterPluginParams; public readonly fieldType: string; - public constructor(config: CmsEntryFieldFilterPluginParams) { + public constructor(config: CmsEntryFieldFilterPluginParams) { super(); this.config = config; this.fieldType = this.config.fieldType; } - public create(params: CmsEntryFieldFilterPluginCreateParams) { + public create(params: CmsEntryFieldFilterPluginCreateParams) { return this.config.create(params); } } diff --git a/packages/api-headless-cms-ddb/tsconfig.build.json b/packages/api-headless-cms-ddb/tsconfig.build.json index bf17bbe28dd..b22550f95d2 100644 --- a/packages/api-headless-cms-ddb/tsconfig.build.json +++ b/packages/api-headless-cms-ddb/tsconfig.build.json @@ -2,6 +2,7 @@ "extends": "../../tsconfig.build.json", "include": ["src"], "references": [ + { "path": "../api/tsconfig.build.json" }, { "path": "../api-headless-cms/tsconfig.build.json" }, { "path": "../aws-sdk/tsconfig.build.json" }, { "path": "../db-dynamodb/tsconfig.build.json" }, diff --git a/packages/api-headless-cms-ddb/tsconfig.json b/packages/api-headless-cms-ddb/tsconfig.json index 9e355ca7a3f..ab91f5e37b0 100644 --- a/packages/api-headless-cms-ddb/tsconfig.json +++ b/packages/api-headless-cms-ddb/tsconfig.json @@ -2,6 +2,7 @@ "extends": "../../tsconfig.json", "include": ["src", "__tests__"], "references": [ + { "path": "../api" }, { "path": "../api-headless-cms" }, { "path": "../aws-sdk" }, { "path": "../db-dynamodb" }, @@ -17,6 +18,8 @@ "paths": { "~/*": ["./src/*"], "~tests/*": ["./__tests__/*"], + "@webiny/api/*": ["../api/src/*"], + "@webiny/api": ["../api/src"], "@webiny/api-headless-cms/*": ["../api-headless-cms/src/*"], "@webiny/api-headless-cms": ["../api-headless-cms/src"], "@webiny/aws-sdk/*": ["../aws-sdk/src/*"], diff --git a/packages/api-headless-cms/__tests__/contentAPI/refField.test.ts b/packages/api-headless-cms/__tests__/contentAPI/refField.test.ts index 0d15e59aa27..3a283574655 100644 --- a/packages/api-headless-cms/__tests__/contentAPI/refField.test.ts +++ b/packages/api-headless-cms/__tests__/contentAPI/refField.test.ts @@ -123,7 +123,7 @@ describe("refField", () => { return publishAuthorResponse.data.publishAuthor.data; }; - test("should create review connected to a product", async () => { + it("should create review connected to a product", async () => { await setupContentModels(mainHandler); const category = await createCategory(); @@ -441,4 +441,131 @@ describe("refField", () => { } }); }); + + it("should create a product which is not connected to category and list and filter by the category value", async () => { + await setupContentModels(mainHandler); + const { createProduct, listProducts } = useProductManageHandler({ + ...manageOpts + }); + + const [listEmptyResult] = await listProducts({ + where: { + category: null + } + }); + expect(listEmptyResult).toMatchObject({ + data: { + listProducts: { + data: [], + error: null, + meta: { + cursor: null, + hasMoreItems: false, + totalCount: 0 + } + } + } + }); + + const data = { + title: "Potato", + price: 100, + availableOn: "2020-12-25", + color: "white", + availableSizes: ["s", "m"], + image: "file.jpg", + category: null + }; + const [createResponse] = await createProduct({ + data + }); + + expect(createResponse).toMatchObject({ + data: { + createProduct: { + data: { + id: expect.any(String), + ...data, + category: null + }, + error: null + } + } + }); + + const [listResult] = await listProducts(); + + expect(listResult).toMatchObject({ + data: { + listProducts: { + data: [ + { + id: expect.any(String), + entryId: expect.any(String), + ...data, + category: null + } + ], + error: null, + meta: { + cursor: null, + hasMoreItems: false, + totalCount: 1 + } + } + } + }); + + const [whereNullResult] = await listProducts({ + where: { + category: null + } + }); + expect(whereNullResult).toMatchObject({ + data: { + listProducts: { + data: [ + { + id: expect.any(String), + entryId: expect.any(String), + ...data, + category: null + } + ], + error: null, + meta: { + cursor: null, + hasMoreItems: false, + totalCount: 1 + } + } + } + }); + + const [whereUndefinedResult] = await listProducts({ + where: { + category: undefined + } + }); + expect(whereUndefinedResult).toMatchObject({ + data: { + listProducts: { + data: [ + { + id: expect.any(String), + entryId: expect.any(String), + ...data, + category: null + } + ], + error: null, + meta: { + cursor: null, + hasMoreItems: false, + totalCount: 1 + } + } + } + }); + }); }); diff --git a/packages/app-file-manager/tsconfig.build.json b/packages/app-file-manager/tsconfig.build.json index 74817e22279..217862e1934 100644 --- a/packages/app-file-manager/tsconfig.build.json +++ b/packages/app-file-manager/tsconfig.build.json @@ -5,9 +5,9 @@ { "path": "../app/tsconfig.build.json" }, { "path": "../app-aco/tsconfig.build.json" }, { "path": "../app-admin/tsconfig.build.json" }, - { "path": "../app-i18n/tsconfig.build.json" }, { "path": "../app-headless-cms/tsconfig.build.json" }, { "path": "../app-headless-cms-common/tsconfig.build.json" }, + { "path": "../app-i18n/tsconfig.build.json" }, { "path": "../app-security/tsconfig.build.json" }, { "path": "../app-tenancy/tsconfig.build.json" }, { "path": "../error/tsconfig.build.json" }, diff --git a/packages/app-file-manager/tsconfig.json b/packages/app-file-manager/tsconfig.json index df85064dc1e..c6f1dd20495 100644 --- a/packages/app-file-manager/tsconfig.json +++ b/packages/app-file-manager/tsconfig.json @@ -5,9 +5,9 @@ { "path": "../app" }, { "path": "../app-aco" }, { "path": "../app-admin" }, - { "path": "../app-i18n" }, { "path": "../app-headless-cms" }, { "path": "../app-headless-cms-common" }, + { "path": "../app-i18n" }, { "path": "../app-security" }, { "path": "../app-tenancy" }, { "path": "../error" }, @@ -32,12 +32,12 @@ "@webiny/app-aco": ["../app-aco/src"], "@webiny/app-admin/*": ["../app-admin/src/*"], "@webiny/app-admin": ["../app-admin/src"], - "@webiny/app-i18n/*": ["../app-i18n/src/*"], - "@webiny/app-i18n": ["../app-i18n/src"], "@webiny/app-headless-cms/*": ["../app-headless-cms/src/*"], "@webiny/app-headless-cms": ["../app-headless-cms/src"], "@webiny/app-headless-cms-common/*": ["../app-headless-cms-common/src/*"], "@webiny/app-headless-cms-common": ["../app-headless-cms-common/src"], + "@webiny/app-i18n/*": ["../app-i18n/src/*"], + "@webiny/app-i18n": ["../app-i18n/src"], "@webiny/app-security/*": ["../app-security/src/*"], "@webiny/app-security": ["../app-security/src"], "@webiny/app-tenancy/*": ["../app-tenancy/src/*"], diff --git a/packages/cli-plugin-scaffold-extensions/tsconfig.json b/packages/cli-plugin-scaffold-extensions/tsconfig.json index 5cbebb5ff09..0a46a2edd0a 100644 --- a/packages/cli-plugin-scaffold-extensions/tsconfig.json +++ b/packages/cli-plugin-scaffold-extensions/tsconfig.json @@ -2,15 +2,9 @@ "extends": "../../tsconfig.json", "include": ["src", "__tests__"], "references": [ - { - "path": "../aws-sdk" - }, - { - "path": "../cli-plugin-scaffold" - }, - { - "path": "../error" - } + { "path": "../aws-sdk" }, + { "path": "../cli-plugin-scaffold" }, + { "path": "../error" } ], "compilerOptions": { "rootDirs": ["./src", "./__tests__"], diff --git a/packages/db-dynamodb/src/plugins/filters/eq.ts b/packages/db-dynamodb/src/plugins/filters/eq.ts index f055e9a498a..918061b32e8 100644 --- a/packages/db-dynamodb/src/plugins/filters/eq.ts +++ b/packages/db-dynamodb/src/plugins/filters/eq.ts @@ -12,10 +12,10 @@ const plugin = new ValueFilterPlugin({ }); } else if (Array.isArray(compareValue) === true) { return compareValue.every((v: string) => { - return value === v; + return value == v; }); } - return value === compareValue; + return value == compareValue; } }); diff --git a/packages/pulumi-aws/tsconfig.build.json b/packages/pulumi-aws/tsconfig.build.json index 95d2ea4294f..88d73a8f7d3 100644 --- a/packages/pulumi-aws/tsconfig.build.json +++ b/packages/pulumi-aws/tsconfig.build.json @@ -3,9 +3,9 @@ "include": ["src"], "references": [ { "path": "../aws-sdk/tsconfig.build.json" }, - { "path": "../feature-flags/tsconfig.build.json" }, { "path": "../pulumi/tsconfig.build.json" }, - { "path": "../api-page-builder/tsconfig.build.json" } + { "path": "../api-page-builder/tsconfig.build.json" }, + { "path": "../feature-flags/tsconfig.build.json" } ], "compilerOptions": { "rootDir": "./src", diff --git a/packages/pulumi-aws/tsconfig.json b/packages/pulumi-aws/tsconfig.json index 5d3c7a0055d..ce4d5c491fb 100644 --- a/packages/pulumi-aws/tsconfig.json +++ b/packages/pulumi-aws/tsconfig.json @@ -3,9 +3,9 @@ "include": ["src", "__tests__"], "references": [ { "path": "../aws-sdk" }, - { "path": "../feature-flags" }, { "path": "../pulumi" }, - { "path": "../api-page-builder" } + { "path": "../api-page-builder" }, + { "path": "../feature-flags" } ], "compilerOptions": { "rootDirs": ["./src", "./__tests__"], @@ -16,12 +16,12 @@ "~tests/*": ["./__tests__/*"], "@webiny/aws-sdk/*": ["../aws-sdk/src/*"], "@webiny/aws-sdk": ["../aws-sdk/src"], - "@webiny/feature-flags/*": ["../feature-flags/src/*"], - "@webiny/feature-flags": ["../feature-flags/src"], "@webiny/pulumi/*": ["../pulumi/src/*"], "@webiny/pulumi": ["../pulumi/src"], "@webiny/api-page-builder/*": ["../api-page-builder/src/*"], - "@webiny/api-page-builder": ["../api-page-builder/src"] + "@webiny/api-page-builder": ["../api-page-builder/src"], + "@webiny/feature-flags/*": ["../feature-flags/src/*"], + "@webiny/feature-flags": ["../feature-flags/src"] }, "baseUrl": "." } diff --git a/yarn.lock b/yarn.lock index 6f585502cd7..a7c77c339fa 100644 --- a/yarn.lock +++ b/yarn.lock @@ -15939,6 +15939,7 @@ __metadata: "@babel/preset-env": ^7.24.0 "@babel/runtime": ^7.24.0 "@types/jsonpack": ^1.1.0 + "@webiny/api": 0.0.0 "@webiny/api-headless-cms": 0.0.0 "@webiny/aws-sdk": 0.0.0 "@webiny/cli": 0.0.0 From fcc7c1ee850f4b271681dd03c7de7e1aaf3e6f86 Mon Sep 17 00:00:00 2001 From: Leonardo Giacone Date: Wed, 28 Aug 2024 17:06:02 +0200 Subject: [PATCH 20/70] fix(app-file-manager): stack tags when container width is limited (#4241) --- .../components/TagsList/TagsList.tsx | 9 ++---- .../components/TagsList/styled.tsx | 28 +++++++++++++------ 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/packages/app-file-manager/src/modules/FileManagerRenderer/FileManagerView/components/TagsList/TagsList.tsx b/packages/app-file-manager/src/modules/FileManagerRenderer/FileManagerView/components/TagsList/TagsList.tsx index 53dfd730879..ea6809ce790 100644 --- a/packages/app-file-manager/src/modules/FileManagerRenderer/FileManagerView/components/TagsList/TagsList.tsx +++ b/packages/app-file-manager/src/modules/FileManagerRenderer/FileManagerView/components/TagsList/TagsList.tsx @@ -3,10 +3,8 @@ import { Loader } from "@webiny/app-aco"; import { Empty } from "./Empty"; import { Tag } from "./Tag"; import { TagItem } from "@webiny/app-aco/types"; -import { Select } from "@webiny/ui/Select"; import { useFileManagerView } from "~/modules/FileManagerRenderer/FileManagerViewProvider"; -import { Typography } from "@webiny/ui/Typography"; -import { TagListWrapper } from "./styled"; +import { TagListWrapper, TagsFilterSelect, TagsTitle } from "./styled"; interface TagListProps { loading: boolean; @@ -59,15 +57,14 @@ export const TagsList = ({ return ( <> - Filter by tag + Filter by tag {tags.length > 1 ? ( - onChange(ev.target.value)} - /> - )} - - - - - {filteredPageTemplates.map(template => ( - { - setActiveTemplate(template); - }} - > - - {template.title} - {template.description} - - - ))} - - - onSelect()} - data-testid={"pb-new-page-dialog-use-blank-template-btn"} + + + + + } /> + + {({ value, onChange }) => ( + onChange(ev.target.value)} + /> + )} + + + + - Use a blank page template - - + {filteredPageTemplates.map(template => ( + { + setActiveTemplate(template); + }} + > + + {template.title} + + {template.description} + + + + ))} + + + onSelect()} + data-testid={"pb-new-page-dialog-use-blank-template-btn"} + > + Use a blank page template + + + {activeTemplate && ( diff --git a/packages/app-page-builder/src/templateEditor/config/Content/BlocksBrowser/StyledComponents.ts b/packages/app-page-builder/src/templateEditor/config/Content/BlocksBrowser/StyledComponents.ts index 367fc4c10bc..cedfa61fe52 100644 --- a/packages/app-page-builder/src/templateEditor/config/Content/BlocksBrowser/StyledComponents.ts +++ b/packages/app-page-builder/src/templateEditor/config/Content/BlocksBrowser/StyledComponents.ts @@ -15,16 +15,17 @@ export const List = styled("div")({ export const Input = styled("div")({ backgroundColor: "var(--mdc-theme-on-background)", position: "relative", - height: 30, - padding: 3, + height: 36, width: "100%", borderRadius: 2, "> input": { border: "none", fontSize: 16, - width: "calc(100% - 10px)", - height: "100%", - marginLeft: 50, + position: "absolute", + top: 0, + bottom: 0, + left: 36, + right: 0, backgroundColor: "transparent", outline: "none", color: "var(--mdc-theme-text-primary-on-background)" @@ -37,14 +38,13 @@ export const searchIcon = css({ position: "absolute", width: 24, height: 24, - left: 15, - top: 7 + left: 8, + top: 6 } }); export const wrapper = css({ height: "100vh", - //overflow: "scroll", backgroundColor: "var(--mdc-theme-background)" }); From 6860ae87e2361c49ab5bdcdf86e962d877c0530f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20Zori=C4=87?= Date: Wed, 25 Sep 2024 11:09:04 +0200 Subject: [PATCH 59/70] refactor(tasks): trigger step function with direct command instead of event (#4289) --- .../graphql/src/plugins/countDynamoDbTask.ts | 10 +- .../src/tasks/createEmptyTrashBinsTask.ts | 1 + .../ChildTaskCleanup/ChildTasksCleanup.ts | 4 +- .../__tests__/context/plugins.ts | 4 +- .../__tests__/helpers/useHandler.ts | 4 +- .../src/graphql/resolvers.ts | 19 ++++ .../src/graphql/security.ts | 16 +++ .../src/tasks/common/ChildTasksCleanup.ts | 4 +- .../tasks/pages/exportPagesControllerTask.ts | 1 + packages/aws-sdk/src/client-sfn/index.ts | 32 +++--- .../tasks/mockTaskTriggerTransportPlugin.ts | 25 +++-- .../project-utils/testing/tasks/runner.ts | 4 +- .../src/apps/api/ApiBackgroundTask.ts | 51 ++++++++-- .../pulumi-aws/src/apps/api/ApiGraphql.ts | 1 + .../src/apps/api/createApiPulumiApp.ts | 1 + packages/tasks/__tests__/crud/trigger.test.ts | 19 ---- packages/tasks/__tests__/graphql/logs.test.ts | 8 +- .../__tests__/helpers/useGraphQLHandler.ts | 2 + .../tasks/__tests__/helpers/useRawHandler.ts | 2 + .../tasks/__tests__/helpers/useTaskHandler.ts | 2 + .../mocks/taskTriggerTransportPlugin.ts | 26 +++++ packages/tasks/src/context.ts | 27 ++--- packages/tasks/src/crud/model.ts | 6 +- .../{trigger.tasks.ts => service.tasks.ts} | 60 ++++++----- .../EventBridgeEventTransportPlugin.ts | 35 +++---- .../transport/StepFunctionServicePlugin.ts | 99 +++++++++++++++++++ packages/tasks/src/crud/transport/index.ts | 9 ++ .../tasks/src/plugins/TaskServicePlugin.ts | 32 ++++++ .../src/plugins/TaskTriggerTransportPlugin.ts | 24 ----- packages/tasks/src/plugins/index.ts | 2 +- packages/tasks/src/runner/TaskManagerStore.ts | 6 +- packages/tasks/src/service/createService.ts | 33 +++++++ .../tasks/src/transport/createTransport.ts | 34 ------- packages/tasks/src/types.ts | 29 ++---- 34 files changed, 414 insertions(+), 218 deletions(-) create mode 100644 packages/api-headless-cms-import-export/src/graphql/security.ts create mode 100644 packages/tasks/__tests__/mocks/taskTriggerTransportPlugin.ts rename packages/tasks/src/crud/{trigger.tasks.ts => service.tasks.ts} (78%) create mode 100644 packages/tasks/src/crud/transport/StepFunctionServicePlugin.ts create mode 100644 packages/tasks/src/crud/transport/index.ts create mode 100644 packages/tasks/src/plugins/TaskServicePlugin.ts delete mode 100644 packages/tasks/src/plugins/TaskTriggerTransportPlugin.ts create mode 100644 packages/tasks/src/service/createService.ts delete mode 100644 packages/tasks/src/transport/createTransport.ts diff --git a/apps/api/graphql/src/plugins/countDynamoDbTask.ts b/apps/api/graphql/src/plugins/countDynamoDbTask.ts index 310aafc01ed..7a5bbdcd26b 100644 --- a/apps/api/graphql/src/plugins/countDynamoDbTask.ts +++ b/apps/api/graphql/src/plugins/countDynamoDbTask.ts @@ -1,13 +1,15 @@ import { createTaskDefinition } from "@webiny/tasks"; import { getDocumentClient } from "@webiny/aws-sdk/client-dynamodb"; +const COUNT_DDB_TASK_ID = "countDdb"; + export const createCountDynamoDbTask = () => { return createTaskDefinition({ - id: "countDdb", + id: COUNT_DDB_TASK_ID, title: "Count DynamoDB", description: "Counts DynamoDB items.", run: async params => { - const { response, isAborted, isCloseToTimeout } = params; + const { response, isAborted, isCloseToTimeout, context, store } = params; if (isAborted()) { return response.aborted(); } else if (isCloseToTimeout()) { @@ -19,6 +21,10 @@ export const createCountDynamoDbTask = () => { TableName: process.env.DB_TABLE }); + const service = await context.tasks.fetchServiceInfo(store.getTask()); + // just to see what we get out + console.log("service", service); + return response.done(`Count: ${results.Count}`); } }); diff --git a/packages/api-headless-cms-bulk-actions/src/tasks/createEmptyTrashBinsTask.ts b/packages/api-headless-cms-bulk-actions/src/tasks/createEmptyTrashBinsTask.ts index 3d14076a513..a7698bd4910 100644 --- a/packages/api-headless-cms-bulk-actions/src/tasks/createEmptyTrashBinsTask.ts +++ b/packages/api-headless-cms-bulk-actions/src/tasks/createEmptyTrashBinsTask.ts @@ -21,6 +21,7 @@ const calculateDateTimeString = () => { export const createEmptyTrashBinsTask = () => { return createTaskDefinition({ + isPrivate: true, id: "hcmsEntriesEmptyTrashBins", title: "Headless CMS - Empty all trash bins", description: diff --git a/packages/api-headless-cms-bulk-actions/src/useCases/internals/ChildTaskCleanup/ChildTasksCleanup.ts b/packages/api-headless-cms-bulk-actions/src/useCases/internals/ChildTaskCleanup/ChildTasksCleanup.ts index 0eb92a4cd32..3537d7cef64 100644 --- a/packages/api-headless-cms-bulk-actions/src/useCases/internals/ChildTaskCleanup/ChildTasksCleanup.ts +++ b/packages/api-headless-cms-bulk-actions/src/useCases/internals/ChildTaskCleanup/ChildTasksCleanup.ts @@ -1,4 +1,4 @@ -import { ITask, Context, ITaskLogItemType } from "@webiny/tasks"; +import { ITask, Context, TaskLogItemType } from "@webiny/tasks"; import { IUseCase } from "~/abstractions"; export interface IChildTasksCleanupExecuteParams { @@ -40,7 +40,7 @@ export class ChildTasksCleanup implements IUseCase item.type === ITaskLogItemType.ERROR)) { + if (log.items.some(item => item.type === TaskLogItemType.ERROR)) { continue; } await context.tasks.deleteLog(log.id); diff --git a/packages/api-headless-cms-es-tasks/__tests__/context/plugins.ts b/packages/api-headless-cms-es-tasks/__tests__/context/plugins.ts index 9526cb78e3e..2cc43a71123 100644 --- a/packages/api-headless-cms-es-tasks/__tests__/context/plugins.ts +++ b/packages/api-headless-cms-es-tasks/__tests__/context/plugins.ts @@ -15,7 +15,7 @@ import { getStorageOps } from "@webiny/project-utils/testing/environment"; import { createBackgroundTaskContext } from "@webiny/tasks"; import { HeadlessCmsStorageOperations } from "@webiny/api-headless-cms/types"; import { createHeadlessCmsEsTasks } from "~/index"; -import { createMockTaskTriggerTransportPlugin } from "@webiny/project-utils/testing/tasks/mockTaskTriggerTransportPlugin"; +import { createMockTaskServicePlugin } from "@webiny/project-utils/testing/tasks/mockTaskTriggerTransportPlugin"; export interface CreateHandlerCoreParams { setupTenancyAndSecurityGraphQL?: boolean; @@ -100,7 +100,7 @@ export const createHandlerCore = (params: CreateHandlerCoreParams = {}) => { plugins, graphQLHandlerPlugins(), createHeadlessCmsEsTasks(), - createMockTaskTriggerTransportPlugin(), + createMockTaskServicePlugin(), bottomPlugins ] }; diff --git a/packages/api-headless-cms-import-export/__tests__/helpers/useHandler.ts b/packages/api-headless-cms-import-export/__tests__/helpers/useHandler.ts index 9fff87025d3..739a9cc02c9 100644 --- a/packages/api-headless-cms-import-export/__tests__/helpers/useHandler.ts +++ b/packages/api-headless-cms-import-export/__tests__/helpers/useHandler.ts @@ -24,7 +24,7 @@ import { createHeadlessCmsImportExport } from "~/index"; import { createGetExportContentEntries } from "./graphql/getExportContentEntries"; import { createExportContentEntries } from "./graphql/exportContentEntries"; import { createAbortExportContentEntries } from "./graphql/abortExportContentEntries"; -import { createMockTaskTriggerTransportPlugin } from "@webiny/project-utils/testing/tasks"; +import { createMockTaskServicePlugin } from "@webiny/project-utils/testing/tasks"; import { createValidateImportFromUrl } from "./graphql/validateImportFromUrl"; import { createGetValidateImportFromUrl } from "./graphql/getValidateImportFromUrl"; import { createCmsPlugins } from "~tests/helpers/models"; @@ -70,7 +70,7 @@ export const useHandler = (params?: UseHandlerParam createRawEventHandler(async ({ context }) => { return context; }), - createMockTaskTriggerTransportPlugin(), + createMockTaskServicePlugin(), ...createCmsPlugins(), ...inputPlugins ]; diff --git a/packages/api-headless-cms-import-export/src/graphql/resolvers.ts b/packages/api-headless-cms-import-export/src/graphql/resolvers.ts index 6814e6cd886..b3d02992443 100644 --- a/packages/api-headless-cms-import-export/src/graphql/resolvers.ts +++ b/packages/api-headless-cms-import-export/src/graphql/resolvers.ts @@ -4,6 +4,7 @@ import { resolve, resolveList } from "@webiny/handler-graphql"; import zod from "zod"; import type { GenericRecord, NonEmptyArray } from "@webiny/api/types"; import { CmsEntryListSort, CmsEntryListWhere, CmsModel } from "@webiny/api-headless-cms/types"; +import { checkPermissions } from "./security"; const validateAbortExportContentEntries = zod.object({ id: zod.string() @@ -57,6 +58,8 @@ const createExportContentEntries = (models: NonEmptyArray) => { context: Context ) => { return resolve(async () => { + await checkPermissions(context); + const result = validateExportContentEntriesInput.safeParse(input); if (!result.success) { @@ -80,6 +83,8 @@ export const createResolvers = (models: NonEmptyArray) => { Query: { async getExportContentEntries(_: unknown, input: unknown, context: Context) { return resolve(async () => { + await checkPermissions(context); + const result = validateGetExportContentEntries.safeParse(input); if (!result.success) { @@ -91,6 +96,8 @@ export const createResolvers = (models: NonEmptyArray) => { }, async listExportContentEntries(_: unknown, input: unknown, context: Context) { return resolveList(async () => { + await checkPermissions(context); + const result = validateListExportContentEntries.safeParse(input); if (!result.success) { throw createZodError(result.error); @@ -100,6 +107,8 @@ export const createResolvers = (models: NonEmptyArray) => { }, async getValidateImportFromUrl(_: unknown, input: unknown, context: Context) { return resolve(async () => { + await checkPermissions(context); + const result = getValidateImportFromUrl.safeParse(input); if (!result.success) { @@ -111,6 +120,8 @@ export const createResolvers = (models: NonEmptyArray) => { }, async getImportFromUrl(_: unknown, input: unknown, context: Context) { return resolve(async () => { + await checkPermissions(context); + const result = getImportFromUrl.safeParse(input); if (!result.success) { @@ -125,6 +136,8 @@ export const createResolvers = (models: NonEmptyArray) => { ...createExportContentEntries(models), async abortExportContentEntries(_: unknown, input: unknown, context: Context) { return resolve(async () => { + await checkPermissions(context); + const result = validateAbortExportContentEntries.safeParse(input); if (!result.success) { @@ -136,6 +149,8 @@ export const createResolvers = (models: NonEmptyArray) => { }, async validateImportFromUrl(_: unknown, input: unknown, context: Context) { return resolve(async () => { + await checkPermissions(context); + const result = validateImportFromUrl.safeParse(input); if (!result.success) { throw createZodError(result.error); @@ -146,6 +161,8 @@ export const createResolvers = (models: NonEmptyArray) => { }, async importFromUrl(_: unknown, input: unknown, context: Context) { return resolve(async () => { + await checkPermissions(context); + const result = importFromUrlValidation.safeParse(input); if (!result.success) { throw createZodError(result.error); @@ -156,6 +173,8 @@ export const createResolvers = (models: NonEmptyArray) => { }, async abortImportFromUrl(_: unknown, input: unknown, context: Context) { return resolve(async () => { + await checkPermissions(context); + const result = abortImportFromUrl.safeParse(input); if (!result.success) { throw createZodError(result.error); diff --git a/packages/api-headless-cms-import-export/src/graphql/security.ts b/packages/api-headless-cms-import-export/src/graphql/security.ts new file mode 100644 index 00000000000..f811b1b9634 --- /dev/null +++ b/packages/api-headless-cms-import-export/src/graphql/security.ts @@ -0,0 +1,16 @@ +import { NotAuthorizedError } from "@webiny/api-security"; +import { Context } from "~/types"; + +export const checkPermissions = async (context: Pick): Promise => { + try { + const permission = await context.security.getPermission("*"); + + if (permission) { + return; + } + } catch (ex) { + console.log("Error while checking CMS Export / Import permissions."); + console.error(ex); + } + throw new NotAuthorizedError(); +}; diff --git a/packages/api-page-builder-import-export/src/tasks/common/ChildTasksCleanup.ts b/packages/api-page-builder-import-export/src/tasks/common/ChildTasksCleanup.ts index 3161cef0386..68e0327d7ae 100644 --- a/packages/api-page-builder-import-export/src/tasks/common/ChildTasksCleanup.ts +++ b/packages/api-page-builder-import-export/src/tasks/common/ChildTasksCleanup.ts @@ -2,7 +2,7 @@ * Cleanup of the child tasks. * This code will remove all the child tasks and their logs, which have no errors in them. */ -import { ITask, Context, ITaskLogItemType } from "@webiny/tasks"; +import { ITask, Context, TaskLogItemType } from "@webiny/tasks"; export interface IChildTasksCleanupExecuteParams { context: Context; @@ -37,7 +37,7 @@ export class ChildTasksCleanup { * First, we need to remove all the logs which have no errors. */ for (const log of childLogs) { - if (log.items.some(item => item.type === ITaskLogItemType.ERROR)) { + if (log.items.some(item => item.type === TaskLogItemType.ERROR)) { continue; } await context.tasks.deleteLog(log.id); diff --git a/packages/api-page-builder-import-export/src/tasks/pages/exportPagesControllerTask.ts b/packages/api-page-builder-import-export/src/tasks/pages/exportPagesControllerTask.ts index abe550e0bab..633226b4a65 100644 --- a/packages/api-page-builder-import-export/src/tasks/pages/exportPagesControllerTask.ts +++ b/packages/api-page-builder-import-export/src/tasks/pages/exportPagesControllerTask.ts @@ -14,6 +14,7 @@ export const createExportPagesControllerTask = () => { IExportPagesControllerInput, IExportPagesControllerOutput >({ + isPrivate: true, id: PageExportTask.Controller, title: "Page Builder - Export Pages - Controller", description: "Export pages from the Page Builder - controller.", diff --git a/packages/aws-sdk/src/client-sfn/index.ts b/packages/aws-sdk/src/client-sfn/index.ts index 309281fa0f6..269b3b660a7 100644 --- a/packages/aws-sdk/src/client-sfn/index.ts +++ b/packages/aws-sdk/src/client-sfn/index.ts @@ -15,6 +15,7 @@ import { StartExecutionCommand } from "@aws-sdk/client-sfn"; import { createCacheKey } from "@webiny/utils"; +import { GenericRecord } from "@webiny/cli/types"; export { SFNClient, @@ -37,21 +38,14 @@ export interface SFNClientConfig extends BaseSFNClientConfig { cache?: boolean; } -export type GenericData = string | number | boolean | null | undefined; - -export interface GenericStepFunctionData { - [key: string]: GenericData | GenericData[]; -} - -export interface TriggerStepFunctionParams< - T extends GenericStepFunctionData = GenericStepFunctionData -> extends Partial> { +export interface TriggerStepFunctionParams + extends Partial> { input: T; } const stepFunctionClientsCache = new Map(); -const getClient = (initial?: SFNClientConfig): SFNClient => { +export const createStepFunctionClient = (initial?: SFNClientConfig): SFNClient => { const config: SFNClientConfig = { region: process.env.AWS_REGION, ...initial @@ -74,9 +68,9 @@ const getClient = (initial?: SFNClientConfig): SFNClient => { }); }; -export const triggerStepFunctionFactory = (config?: SFNClientConfig) => { - const client = getClient(config); - return async ( +export const triggerStepFunctionFactory = (input?: SFNClient | SFNClientConfig) => { + const client = input instanceof SFNClient ? input : createStepFunctionClient(input); + return async ( params: TriggerStepFunctionParams ): Promise => { const cmd = new StartExecutionCommand({ @@ -89,8 +83,8 @@ export const triggerStepFunctionFactory = (config?: SFNClientConfig) => { }; }; -export const listExecutionsFactory = (config?: SFNClientConfig) => { - const client = getClient(config); +export const listExecutionsFactory = (input?: SFNClient | SFNClientConfig) => { + const client = input instanceof SFNClient ? input : createStepFunctionClient(input); return async (params: ListExecutionsCommandInput): Promise => { const cmd = new ListExecutionsCommand({ ...params, @@ -100,12 +94,14 @@ export const listExecutionsFactory = (config?: SFNClientConfig) => { }; }; -export const describeExecutionFactory = (config?: SFNClientConfig) => { - const client = getClient(config); +export const describeExecutionFactory = (input?: SFNClient | SFNClientConfig) => { + const client = input instanceof SFNClient ? input : createStepFunctionClient(input); return async ( params: DescribeExecutionCommandInput ): Promise => { - const cmd = new DescribeExecutionCommand(params); + const cmd = new DescribeExecutionCommand({ + ...params + }); return await client.send(cmd); }; }; diff --git a/packages/project-utils/testing/tasks/mockTaskTriggerTransportPlugin.ts b/packages/project-utils/testing/tasks/mockTaskTriggerTransportPlugin.ts index 6665bd4e49d..03d8a0adc78 100644 --- a/packages/project-utils/testing/tasks/mockTaskTriggerTransportPlugin.ts +++ b/packages/project-utils/testing/tasks/mockTaskTriggerTransportPlugin.ts @@ -1,12 +1,9 @@ -import { - ITaskTriggerTransport, - TaskTriggerTransportPlugin -} from "@webiny/tasks/plugins/TaskTriggerTransportPlugin"; +import { ITaskService, TaskServicePlugin } from "@webiny/tasks/plugins/TaskServicePlugin"; -class MockTaskTriggerTransportPlugin extends TaskTriggerTransportPlugin { - public override name = "tasks.mockTaskTriggerTransport"; +class MockTaskServicePlugin extends TaskServicePlugin { + public override name = "tasks.mockTaskService"; - override createTransport(): ITaskTriggerTransport { + public override createService(): ITaskService { return { async send() { return { @@ -14,11 +11,21 @@ class MockTaskTriggerTransportPlugin extends TaskTriggerTransportPlugin { $metadata: {}, FailedEntryCount: 0 }; + }, + async fetch(input: any) { + return { + fetched: true, + input + } as any; } }; } } -export const createMockTaskTriggerTransportPlugin = () => { - return [new MockTaskTriggerTransportPlugin()]; +export const createMockTaskServicePlugin = () => { + return [ + new MockTaskServicePlugin({ + default: true + }) + ]; }; diff --git a/packages/project-utils/testing/tasks/runner.ts b/packages/project-utils/testing/tasks/runner.ts index 3461e4ec5ea..ea146ca4e27 100644 --- a/packages/project-utils/testing/tasks/runner.ts +++ b/packages/project-utils/testing/tasks/runner.ts @@ -11,7 +11,7 @@ import { TaskRunner } from "@webiny/tasks/runner"; import { timerFactory } from "@webiny/handler-aws/utils"; import { TaskEventValidation } from "@webiny/tasks/runner/TaskEventValidation"; import { ResponseContinueResult } from "@webiny/tasks/response/ResponseContinueResult"; -import { createMockTaskTriggerTransportPlugin } from "./mockTaskTriggerTransportPlugin"; +import { createMockTaskServicePlugin } from "./mockTaskTriggerTransportPlugin"; export interface ICreateRunnerParamsOnContinueCallableParams { taskId: string; @@ -48,7 +48,7 @@ export const createRunner = < >( params: ICreateRunnerParams ) => { - params.context.plugins.register(createMockTaskTriggerTransportPlugin()); + params.context.plugins.register(createMockTaskServicePlugin()); const runner = new TaskRunner( params.context, timerFactory({ diff --git a/packages/pulumi-aws/src/apps/api/ApiBackgroundTask.ts b/packages/pulumi-aws/src/apps/api/ApiBackgroundTask.ts index e91506fc27d..4e05bc5aafc 100644 --- a/packages/pulumi-aws/src/apps/api/ApiBackgroundTask.ts +++ b/packages/pulumi-aws/src/apps/api/ApiBackgroundTask.ts @@ -6,6 +6,7 @@ import { createBackgroundTaskDefinition } from "./backgroundTask/definition"; import { createBackgroundTaskStepFunctionPolicy } from "~/apps/api/backgroundTask/policy"; import { createBackgroundTaskStepFunctionRole } from "./backgroundTask/role"; import { getLayerArn } from "@webiny/aws-layers"; +import { getAwsAccountId, getAwsRegion } from "~/apps/awsUtils"; export type ApiBackgroundTask = PulumiAppModule; @@ -14,6 +15,8 @@ export const ApiBackgroundTaskLambdaName = "background-task"; export const ApiBackgroundTask = createAppModule({ name: "ApiBackgroundTask", config(app: PulumiApp) { + const awsAccountId = getAwsAccountId(app); + const awsRegion = getAwsRegion(app); const core = app.getModule(CoreOutput); const graphql = app.getModule(ApiGraphql); const baseConfig = graphql.functions.graphql.config.clone(); @@ -44,14 +47,6 @@ export const ApiBackgroundTask = createAppModule({ const stepFunction = app.addResource(aws.sfn.StateMachine, { name: "background-task-sfn", config: { - // TODO logging to cloudwatch - /* - loggingConfiguration: { - level: "ALL", - includeExecutionData: true, - // insert real ARN - logDestination: ARN - */ roleArn: stepFunctionRole.output.arn, definition: pulumi.jsonStringify( createBackgroundTaskDefinition({ @@ -62,6 +57,42 @@ export const ApiBackgroundTask = createAppModule({ } }); + const policyToAccessStepFunction = app.addResource(aws.iam.Policy, { + name: "background-task-step-function-policy", + config: { + policy: { + Version: "2012-10-17", + Statement: [ + { + Action: ["states:StartExecution", "states:StopExecution"], + Effect: "Allow", + Resource: [ + stepFunction.output.arn.apply(arn => `${arn}`), + stepFunction.output.arn.apply(arn => `${arn}*`) + ] + }, + { + Action: ["states:DescribeExecution", "states:ListExecutions"], + Effect: "Allow", + Resource: [ + stepFunction.output.name.apply(name => { + return pulumi.interpolate`arn:aws:states:${awsRegion}:${awsAccountId}:execution:${name}:*`; + }) + ] + } + ] + } + } + }); + + app.addResource(aws.iam.RolePolicyAttachment, { + name: "background-task-step-function-policy-attachment-graphql", + config: { + policyArn: policyToAccessStepFunction.output.arn, + role: graphql.role.output.name + } + }); + const eventRole = app.addResource(aws.iam.Role, { name: "background-task-event-role", config: { @@ -132,10 +163,10 @@ export const ApiBackgroundTask = createAppModule({ return { backgroundTask, stepFunction, - eventRole, + stepFunctionRole, + stepFunctionPolicy, eventPolicy, eventRolePolicyAttachment, - eventRule, eventTarget }; } diff --git a/packages/pulumi-aws/src/apps/api/ApiGraphql.ts b/packages/pulumi-aws/src/apps/api/ApiGraphql.ts index fb0957b8eb1..55726a6b95f 100644 --- a/packages/pulumi-aws/src/apps/api/ApiGraphql.ts +++ b/packages/pulumi-aws/src/apps/api/ApiGraphql.ts @@ -40,6 +40,7 @@ export const ApiGraphql = createAppModule({ name: "api-lambda-role", policy: policy.output }); + policy.config.policy; const graphql = app.addResource(aws.lambda.Function, { name: "graphql", diff --git a/packages/pulumi-aws/src/apps/api/createApiPulumiApp.ts b/packages/pulumi-aws/src/apps/api/createApiPulumiApp.ts index 6dc9229cf86..60b59e4c8fb 100644 --- a/packages/pulumi-aws/src/apps/api/createApiPulumiApp.ts +++ b/packages/pulumi-aws/src/apps/api/createApiPulumiApp.ts @@ -314,6 +314,7 @@ export const createApiPulumiApp = (projectAppParams: CreateApiPulumiAppParams = app.addServiceManifest({ name: "api", manifest: { + bgTaskSfn: baseApp.resources.backgroundTask.stepFunction.output.arn, cloudfront: { distributionId: baseApp.resources.cloudfront.output.id } diff --git a/packages/tasks/__tests__/crud/trigger.test.ts b/packages/tasks/__tests__/crud/trigger.test.ts index 8fc238d2e3e..49bc994f3a7 100644 --- a/packages/tasks/__tests__/crud/trigger.test.ts +++ b/packages/tasks/__tests__/crud/trigger.test.ts @@ -3,25 +3,6 @@ import { createMockTaskDefinitions } from "~tests/mocks/definition"; import { createMockIdentity } from "~tests/mocks/identity"; import { TaskDataStatus } from "~/types"; -jest.mock("@webiny/aws-sdk/client-eventbridge", () => { - return { - EventBridgeClient: class EventBridgeClient { - async send(cmd: any) { - return { - input: cmd.input - }; - } - }, - PutEventsCommand: class PutEventsCommand { - public readonly input: any; - - constructor(input: any) { - this.input = input; - } - } - }; -}); - describe("trigger crud", () => { const handler = useRawHandler({ plugins: [...createMockTaskDefinitions()] diff --git a/packages/tasks/__tests__/graphql/logs.test.ts b/packages/tasks/__tests__/graphql/logs.test.ts index 0c0a2fb5809..8537d4d98d5 100644 --- a/packages/tasks/__tests__/graphql/logs.test.ts +++ b/packages/tasks/__tests__/graphql/logs.test.ts @@ -1,7 +1,7 @@ import { useGraphQLHandler } from "~tests/helpers/useGraphQLHandler"; import { createMockTaskDefinitions } from "~tests/mocks/definition"; import { useRawHandler } from "~tests/helpers/useRawHandler"; -import { ITaskLogItemType } from "~/types"; +import { TaskLogItemType } from "~/types"; import { createMockIdentity } from "~tests/mocks/identity"; describe("graphql - logs", () => { @@ -63,7 +63,7 @@ describe("graphql - logs", () => { const log2Item = { message: "Log 2 item message", - type: ITaskLogItemType.INFO, + type: TaskLogItemType.INFO, createdOn: new Date().toISOString() }; const log2 = await context.tasks.updateLog(log1.id, { @@ -72,7 +72,7 @@ describe("graphql - logs", () => { const log3Item = { message: "log 3 item message", - type: ITaskLogItemType.INFO, + type: TaskLogItemType.INFO, createdOn: new Date().toISOString() }; @@ -82,7 +82,7 @@ describe("graphql - logs", () => { const log4Item = { message: "log 4 item message", - type: ITaskLogItemType.ERROR, + type: TaskLogItemType.ERROR, createdOn: new Date().toISOString(), error: { message: "log 4 item error message", diff --git a/packages/tasks/__tests__/helpers/useGraphQLHandler.ts b/packages/tasks/__tests__/helpers/useGraphQLHandler.ts index 970f31aab53..58407b91bff 100644 --- a/packages/tasks/__tests__/helpers/useGraphQLHandler.ts +++ b/packages/tasks/__tests__/helpers/useGraphQLHandler.ts @@ -19,6 +19,7 @@ import apiKeyAuthorization from "@webiny/api-security/plugins/apiKeyAuthorizatio import { Context } from "~tests/types"; import { createListTasksQuery } from "~tests/helpers/graphql/tasks"; import { createListTaskLogsQuery } from "~tests/helpers/graphql/logs"; +import { createMockTaskServicePlugin } from "~tests/mocks/taskTriggerTransportPlugin"; export interface InvokeParams { httpMethod?: "POST" | "GET" | "OPTIONS"; @@ -95,6 +96,7 @@ export const useGraphQLHandler = (params?: UseHandlerParams) => { graphQLHandlerPlugins(), createBackgroundTaskContext(), createBackgroundTaskGraphQL(), + createMockTaskServicePlugin(), ...plugins ] }); diff --git a/packages/tasks/__tests__/helpers/useRawHandler.ts b/packages/tasks/__tests__/helpers/useRawHandler.ts index 9c09bb60750..2c71123e2e5 100644 --- a/packages/tasks/__tests__/helpers/useRawHandler.ts +++ b/packages/tasks/__tests__/helpers/useRawHandler.ts @@ -12,6 +12,7 @@ import { LambdaContext } from "@webiny/handler-aws/types"; import { Context } from "~tests/types"; import { PluginCollection } from "@webiny/plugins/types"; import { createBackgroundTaskContext } from "~/index"; +import { createMockTaskServicePlugin } from "~tests/mocks/taskTriggerTransportPlugin"; export interface UseHandlerParams { plugins?: PluginCollection; @@ -44,6 +45,7 @@ export const useRawHandler = (params?: UseHandlerPa createRawEventHandler(async ({ context }) => { return context; }), + createMockTaskServicePlugin(), ...plugins ] }); diff --git a/packages/tasks/__tests__/helpers/useTaskHandler.ts b/packages/tasks/__tests__/helpers/useTaskHandler.ts index 7af49329c60..b28702fe7ec 100644 --- a/packages/tasks/__tests__/helpers/useTaskHandler.ts +++ b/packages/tasks/__tests__/helpers/useTaskHandler.ts @@ -13,6 +13,7 @@ import { HeadlessCmsStorageOperations } from "@webiny/api-headless-cms/types"; import { PluginCollection } from "@webiny/plugins/types"; import { LambdaContext } from "@webiny/handler-aws/types"; import { ITaskRawEvent } from "~/handler/types"; +import { createMockTaskServicePlugin } from "~tests/mocks/taskTriggerTransportPlugin"; export interface UseTaskHandlerParams { plugins?: PluginCollection; @@ -45,6 +46,7 @@ export const useTaskHandler = (params?: UseTaskHandlerParams) => { createRawEventHandler(async ({ context }) => { return context; }), + createMockTaskServicePlugin(), ...plugins ] }); diff --git a/packages/tasks/__tests__/mocks/taskTriggerTransportPlugin.ts b/packages/tasks/__tests__/mocks/taskTriggerTransportPlugin.ts new file mode 100644 index 00000000000..0138fc04d16 --- /dev/null +++ b/packages/tasks/__tests__/mocks/taskTriggerTransportPlugin.ts @@ -0,0 +1,26 @@ +import { TaskServicePlugin } from "~/plugins"; + +class MockTaskServicePlugin extends TaskServicePlugin { + public override name = "mock-task-trigger-transport-plugin"; + public createService() { + return { + send: async () => { + return { + mockedSend: true + }; + }, + fetch: async (input: any) => { + return { + fetched: true, + input + } as any; + } + }; + } +} + +export const createMockTaskServicePlugin = (): TaskServicePlugin => { + return new MockTaskServicePlugin({ + default: true + }); +}; diff --git a/packages/tasks/src/context.ts b/packages/tasks/src/context.ts index 506de72bebe..90cc88f38f4 100644 --- a/packages/tasks/src/context.ts +++ b/packages/tasks/src/context.ts @@ -1,28 +1,19 @@ import { Plugin } from "@webiny/plugins"; import { ContextPlugin } from "@webiny/api"; -import { Context, ITaskConfig, ITasksContextConfigObject } from "~/types"; +import { Context } from "~/types"; import { createTaskModel } from "./crud/model"; import { createDefinitionCrud } from "./crud/definition.tasks"; -import { createTriggerTasksCrud } from "~/crud/trigger.tasks"; +import { createServiceCrud } from "~/crud/service.tasks"; import { createTaskCrud } from "./crud/crud.tasks"; import { createTestingRunTask } from "~/tasks/testingRunTask"; +import { createTransportPlugins } from "~/crud/transport"; -const createConfig = (config?: ITaskConfig): ITasksContextConfigObject => { - return { - config: { - eventBusName: config?.eventBusName || String(process.env.EVENT_BUS) - } - }; -}; - -const createTasksCrud = (input?: ITaskConfig) => { - const config = createConfig(input); +const createTasksCrud = () => { const plugin = new ContextPlugin(async context => { context.tasks = { - ...config, ...createDefinitionCrud(context), ...createTaskCrud(context), - ...createTriggerTasksCrud(context, config.config) + ...createServiceCrud(context) }; }); @@ -31,10 +22,10 @@ const createTasksCrud = (input?: ITaskConfig) => { return plugin; }; -const createTasksContext = (input?: ITaskConfig): Plugin[] => { - return [...createTaskModel(), createTasksCrud(input)]; +const createTasksContext = (): Plugin[] => { + return [...createTransportPlugins(), ...createTaskModel(), createTasksCrud()]; }; -export const createBackgroundTaskContext = (config?: ITaskConfig): Plugin[] => { - return [createTestingRunTask(), ...createTasksContext(config)]; +export const createBackgroundTaskContext = (): Plugin[] => { + return [createTestingRunTask(), ...createTasksContext()]; }; diff --git a/packages/tasks/src/crud/model.ts b/packages/tasks/src/crud/model.ts index 78d8ac8e1b2..31034b6fda3 100644 --- a/packages/tasks/src/crud/model.ts +++ b/packages/tasks/src/crud/model.ts @@ -1,5 +1,5 @@ import { createCmsModel, createPrivateModel } from "@webiny/api-headless-cms"; -import { ITaskLogItemType, TaskDataStatus } from "~/types"; +import { TaskLogItemType, TaskDataStatus } from "~/types"; export const WEBINY_TASK_MODEL_ID = "webinyTask"; export const WEBINY_TASK_LOG_MODEL_ID = "webinyTaskLog"; @@ -93,11 +93,11 @@ const taskLogModelPlugin = createCmsModel( enabled: true, values: [ { - value: ITaskLogItemType.INFO, + value: TaskLogItemType.INFO, label: "Info" }, { - value: ITaskLogItemType.ERROR, + value: TaskLogItemType.ERROR, label: "Error" } ] diff --git a/packages/tasks/src/crud/trigger.tasks.ts b/packages/tasks/src/crud/service.tasks.ts similarity index 78% rename from packages/tasks/src/crud/trigger.tasks.ts rename to packages/tasks/src/crud/service.tasks.ts index 5688f9c6d80..1a393999eb6 100644 --- a/packages/tasks/src/crud/trigger.tasks.ts +++ b/packages/tasks/src/crud/service.tasks.ts @@ -1,22 +1,18 @@ import WebinyError from "@webiny/error"; -import { +import type { Context, ITask, ITaskAbortParams, - ITaskConfig, ITaskCreateData, ITaskDataInput, ITaskLog, - ITaskLogItemType, ITaskResponseDoneResultOutput, - ITasksContextTriggerObject, - ITaskTriggerParams, - PutEventsCommandOutput, - TaskDataStatus + ITasksContextServiceObject, + ITaskTriggerParams } from "~/types"; +import { TaskDataStatus, TaskLogItemType } from "~/types"; import { NotFoundError } from "@webiny/handler-graphql"; -import { createTransport } from "~/transport/createTransport"; -import { EventBridgeEventTransportPlugin } from "~/crud/transport/EventBridgeEventTransportPlugin"; +import { createService } from "~/service/createService"; const MAX_DELAY_DAYS = 355; const MAX_DELAY_SECONDS = MAX_DELAY_DAYS * 24 * 60 * 60; @@ -42,15 +38,9 @@ const validateDelay = ({ input, delay }: ValidateDelayParams ); }; -export const createTriggerTasksCrud = ( - context: Context, - config: ITaskConfig -): ITasksContextTriggerObject => { - context.plugins.register(new EventBridgeEventTransportPlugin()); - - const transport = createTransport({ - context, - config +export const createServiceCrud = (context: Context): ITasksContextServiceObject => { + const service = createService({ + context }); return { @@ -86,14 +76,14 @@ export const createTriggerTasksCrud = ( const task = await context.tasks.createTask(input); - let event: PutEventsCommandOutput | null = null; + let result: Awaited> | null = null; try { - event = await transport.send(task, delay); + result = await service.send(task, delay); - if (!event) { + if (!result) { throw new WebinyError( - `Could not create the Event Bridge Event!`, - "CREATE_EVENT_BRIDGE_EVENT_ERROR", + `Could not trigger the step function!`, + "TRIGGER_STEP_FUNCTION_ERROR", { task } @@ -107,10 +97,28 @@ export const createTriggerTasksCrud = ( await context.tasks.deleteTask(task.id); throw ex; } - return await context.tasks.updateTask(task.id, { - eventResponse: event + return await context.tasks.updateTask(task.id, { + eventResponse: result }); }, + fetchServiceInfo: async (input: ITask | string) => { + const task = typeof input === "object" ? input : await context.tasks.getTask(input); + if (!task && typeof input === "string") { + throw new NotFoundError(`Task "${input}" was not found!`); + } else if (!task) { + throw new WebinyError(`Task was not found!`, "TASK_FETCH_ERROR", { + input + }); + } + + try { + return await service.fetch(task); + } catch (ex) { + console.log("Service fetch error."); + console.error(ex); + return null; + } + }, abort: async < T = ITaskDataInput, O extends ITaskResponseDoneResultOutput = ITaskResponseDoneResultOutput @@ -161,7 +169,7 @@ export const createTriggerTasksCrud = ( items: taskLog.items.concat([ { message: params.message || "Task aborted.", - type: ITaskLogItemType.INFO, + type: TaskLogItemType.INFO, createdOn: new Date().toISOString() } ]) diff --git a/packages/tasks/src/crud/transport/EventBridgeEventTransportPlugin.ts b/packages/tasks/src/crud/transport/EventBridgeEventTransportPlugin.ts index 76defe367c5..aafb20ebd69 100644 --- a/packages/tasks/src/crud/transport/EventBridgeEventTransportPlugin.ts +++ b/packages/tasks/src/crud/transport/EventBridgeEventTransportPlugin.ts @@ -1,34 +1,31 @@ import { - ITaskTriggerTransport, - ITaskTriggerTransportPluginParams, - PutEventsCommandOutput, - TaskTriggerTransportPlugin + ITaskService, + ITaskServiceCreatePluginParams, + ITaskServiceTask, + TaskServicePlugin } from "~/plugins"; -import { Context, ITask, ITaskConfig, ITaskEventInput } from "~/types"; +import { Context, ITaskEventInput } from "~/types"; +import type { PutEventsCommandOutput } from "@webiny/aws-sdk/client-eventbridge"; import { EventBridgeClient, PutEventsCommand } from "@webiny/aws-sdk/client-eventbridge"; import { WebinyError } from "@webiny/error"; +import { GenericRecord } from "@webiny/api/types"; -class EventBridgeEventTransport implements ITaskTriggerTransport { +class EventBridgeService implements ITaskService { protected readonly context: Context; - protected readonly config: ITaskConfig; protected readonly getTenant: () => string; protected readonly getLocale: () => string; private readonly client: EventBridgeClient; - public constructor(params: ITaskTriggerTransportPluginParams) { + public constructor(params: ITaskServiceCreatePluginParams) { this.client = new EventBridgeClient({ region: process.env.AWS_REGION }); this.context = params.context; - this.config = params.config; this.getTenant = params.getTenant; this.getLocale = params.getLocale; } - public async send( - task: Pick, - delay: number - ): Promise { + public async send(task: ITaskServiceTask, delay: number): Promise { /** * The ITaskEvent is what our handler expect to get. * Endpoint and stateMachineId are added by the step function. @@ -45,7 +42,7 @@ class EventBridgeEventTransport implements ITaskTriggerTransport { Entries: [ { Source: "webiny-api-tasks", - EventBusName: this.config.eventBusName, + EventBusName: String(process.env.EVENT_BUS), DetailType: "WebinyBackgroundTask", Detail: JSON.stringify(event) } @@ -64,11 +61,15 @@ class EventBridgeEventTransport implements ITaskTriggerTransport { ); } } + + public async fetch(): Promise { + throw new WebinyError("Not implemented!", "NOT_IMPLEMENTED"); + } } -export class EventBridgeEventTransportPlugin extends TaskTriggerTransportPlugin { +export class EventBridgeEventTransportPlugin extends TaskServicePlugin { public override name = "task.eventBridgeEventTransport"; - public createTransport(params: ITaskTriggerTransportPluginParams): ITaskTriggerTransport { - return new EventBridgeEventTransport(params); + public createService(params: ITaskServiceCreatePluginParams) { + return new EventBridgeService(params); } } diff --git a/packages/tasks/src/crud/transport/StepFunctionServicePlugin.ts b/packages/tasks/src/crud/transport/StepFunctionServicePlugin.ts new file mode 100644 index 00000000000..db606a6c843 --- /dev/null +++ b/packages/tasks/src/crud/transport/StepFunctionServicePlugin.ts @@ -0,0 +1,99 @@ +import { + ITaskService, + ITaskServiceCreatePluginParams, + ITaskServiceTask, + TaskServicePlugin +} from "~/plugins"; +import { GenericRecord } from "@webiny/api/types"; +import { + createStepFunctionClient, + describeExecutionFactory, + triggerStepFunctionFactory +} from "@webiny/aws-sdk/client-sfn"; +import { ITaskEventInput } from "~/handler/types"; +import { generateAlphaNumericId } from "@webiny/utils"; +import { ServiceDiscovery } from "@webiny/api"; +import { ITask } from "~/types"; + +export interface IDetailWrapper { + detail: T; +} + +class StepFunctionService implements ITaskService { + private readonly getTenant: () => string; + private readonly getLocale: () => string; + private readonly trigger: ReturnType; + private readonly get: ReturnType; + + public constructor(params: ITaskServiceCreatePluginParams) { + this.getTenant = params.getTenant; + this.getLocale = params.getLocale; + const client = createStepFunctionClient(); + this.trigger = triggerStepFunctionFactory(client); + this.get = describeExecutionFactory(client); + } + public async send(task: ITaskServiceTask, delay: number) { + const manifest = await ServiceDiscovery.load(); + if (!manifest) { + console.error("Service manifest not found."); + return null; + } + const { bgTaskSfn } = manifest.api || {}; + if (!bgTaskSfn) { + console.error("Background task state machine not found."); + return null; + } + + const input: ITaskEventInput = { + webinyTaskId: task.id, + webinyTaskDefinitionId: task.definitionId, + tenant: this.getTenant(), + locale: this.getLocale(), + delay + }; + const name = `${task.definitionId}_${task.id}_${generateAlphaNumericId(10)}`; + try { + const result = await this.trigger>({ + input: { + detail: input + }, + stateMachineArn: bgTaskSfn, + name + }); + return { + ...result, + name + }; + } catch (ex) { + console.log("Could not trigger a step function."); + console.error(ex); + return null; + } + } + + public async fetch(task: ITask): Promise { + const executionArn = task.eventResponse?.executionArn; + if (!executionArn) { + console.error(`Execution ARN not found in task "${task.id}".`); + return null; + } + try { + const result = await this.get({ + executionArn + }); + return (result || null) as R; + } catch (ex) { + console.log("Could not get the execution details."); + console.error(ex); + return null; + } + } +} + +export class StepFunctionServicePlugin extends TaskServicePlugin { + public override name = "task.stepFunctionTriggerTransport"; + + public createService(params: ITaskServiceCreatePluginParams) { + return new StepFunctionService(params); + } +} diff --git a/packages/tasks/src/crud/transport/index.ts b/packages/tasks/src/crud/transport/index.ts new file mode 100644 index 00000000000..d6f4c691da0 --- /dev/null +++ b/packages/tasks/src/crud/transport/index.ts @@ -0,0 +1,9 @@ +import { EventBridgeEventTransportPlugin } from "./EventBridgeEventTransportPlugin"; +import { StepFunctionServicePlugin } from "./StepFunctionServicePlugin"; + +export const createTransportPlugins = () => { + return [ + new StepFunctionServicePlugin({ default: true }), + new EventBridgeEventTransportPlugin() + ]; +}; diff --git a/packages/tasks/src/plugins/TaskServicePlugin.ts b/packages/tasks/src/plugins/TaskServicePlugin.ts new file mode 100644 index 00000000000..4808f516192 --- /dev/null +++ b/packages/tasks/src/plugins/TaskServicePlugin.ts @@ -0,0 +1,32 @@ +import { Plugin } from "@webiny/plugins"; +import { Context, ITask } from "~/types"; +import { GenericRecord } from "@webiny/api/types"; + +export interface ITaskServiceCreatePluginParams { + context: Context; + getTenant(): string; + getLocale(): string; +} + +export type ITaskServiceTask = Pick; + +export interface ITaskService { + send(task: ITaskServiceTask, delay: number): Promise; + fetch(task: ITask): Promise; +} + +export interface ITaskServicePluginParams { + default?: boolean; +} + +export abstract class TaskServicePlugin extends Plugin { + public static override readonly type: string = "tasks.taskService"; + public readonly default: boolean; + + public constructor(params?: ITaskServicePluginParams) { + super(); + this.default = !!params?.default; + } + + public abstract createService(params: ITaskServiceCreatePluginParams): ITaskService; +} diff --git a/packages/tasks/src/plugins/TaskTriggerTransportPlugin.ts b/packages/tasks/src/plugins/TaskTriggerTransportPlugin.ts deleted file mode 100644 index 33ecbbc2923..00000000000 --- a/packages/tasks/src/plugins/TaskTriggerTransportPlugin.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { Plugin } from "@webiny/plugins"; -import { Context, ITask, ITaskConfig } from "~/types"; -import { PutEventsCommandOutput } from "@webiny/aws-sdk/client-eventbridge"; - -export { PutEventsCommandOutput }; - -export interface ITaskTriggerTransportPluginParams { - context: Context; - config: ITaskConfig; - getTenant(): string; - getLocale(): string; -} - -export interface ITaskTriggerTransport { - send(task: Pick, delay: number): Promise; -} - -export abstract class TaskTriggerTransportPlugin extends Plugin { - public static override readonly type: string = "tasks.taskTriggerTransport"; - - public abstract createTransport( - params: ITaskTriggerTransportPluginParams - ): ITaskTriggerTransport; -} diff --git a/packages/tasks/src/plugins/index.ts b/packages/tasks/src/plugins/index.ts index bcd15a83e5a..c06ff99d870 100644 --- a/packages/tasks/src/plugins/index.ts +++ b/packages/tasks/src/plugins/index.ts @@ -1 +1 @@ -export * from "./TaskTriggerTransportPlugin"; +export * from "./TaskServicePlugin"; diff --git a/packages/tasks/src/runner/TaskManagerStore.ts b/packages/tasks/src/runner/TaskManagerStore.ts index c7f3e21f92e..e924ffc0dbf 100644 --- a/packages/tasks/src/runner/TaskManagerStore.ts +++ b/packages/tasks/src/runner/TaskManagerStore.ts @@ -3,7 +3,7 @@ import { ITask, ITaskDataInput, ITaskLog, - ITaskLogItemType, + TaskLogItemType, ITaskManagerStoreInfoLog, ITaskManagerStorePrivate, ITaskManagerStoreSetOutputOptions, @@ -176,7 +176,7 @@ export class TaskManagerStore< { message: log.message, data: log.data, - type: ITaskLogItemType.INFO, + type: TaskLogItemType.INFO, createdOn: new Date().toISOString() } ] @@ -210,7 +210,7 @@ export class TaskManagerStore< { message: log.message, error: log.error instanceof Error ? getObjectProperties(log.error) : log.error, - type: ITaskLogItemType.ERROR, + type: TaskLogItemType.ERROR, createdOn: new Date().toISOString() } ] diff --git a/packages/tasks/src/service/createService.ts b/packages/tasks/src/service/createService.ts new file mode 100644 index 00000000000..e36620dc440 --- /dev/null +++ b/packages/tasks/src/service/createService.ts @@ -0,0 +1,33 @@ +import { Context } from "~/types"; +import { ITaskService, TaskServicePlugin } from "~/plugins"; +import { WebinyError } from "@webiny/error"; + +export interface ICreateTransport { + context: Context; +} + +export const createService = (params: ICreateTransport): ITaskService => { + const plugins = params.context.plugins + .byType(TaskServicePlugin.type) + .reverse(); + + const plugin = plugins.find(plugin => plugin.default) || plugins[0]; + if (!plugin) { + throw new WebinyError("Missing TaskServicePlugin.", "PLUGIN_ERROR", { + type: TaskServicePlugin.type + }); + } + + const getTenant = (): string => { + return params.context.tenancy.getCurrentTenant().id; + }; + const getLocale = (): string => { + return params.context.cms.getLocale().code; + }; + + return plugin.createService({ + context: params.context, + getTenant, + getLocale + }); +}; diff --git a/packages/tasks/src/transport/createTransport.ts b/packages/tasks/src/transport/createTransport.ts deleted file mode 100644 index f7be33941d3..00000000000 --- a/packages/tasks/src/transport/createTransport.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { Context, ITaskConfig } from "~/types"; -import { ITaskTriggerTransport, TaskTriggerTransportPlugin } from "~/plugins"; -import { WebinyError } from "@webiny/error"; - -export interface ICreateTransport { - context: Context; - config: ITaskConfig; -} - -export const createTransport = (params: ICreateTransport): ITaskTriggerTransport => { - const plugins = params.context.plugins.byType( - TaskTriggerTransportPlugin.type - ); - const [plugin] = plugins; - if (!plugin) { - throw new WebinyError("Missing TaskTriggerTransportPlugin.", "PLUGIN_ERROR", { - type: TaskTriggerTransportPlugin.type - }); - } - - const getTenant = (): string => { - return params.context.tenancy.getCurrentTenant().id; - }; - const getLocale = (): string => { - return params.context.cms.getLocale().code; - }; - - return plugin.createTransport({ - context: params.context, - config: params.config, - getTenant, - getLocale - }); -}; diff --git a/packages/tasks/src/types.ts b/packages/tasks/src/types.ts index ae3e2314a31..04a38c8d2fc 100644 --- a/packages/tasks/src/types.ts +++ b/packages/tasks/src/types.ts @@ -14,23 +14,16 @@ import { ITaskResponseResult } from "~/response/abstractions"; import { IIsCloseToTimeoutCallable, ITaskManagerStore } from "./runner/abstractions"; -import { PutEventsCommandOutput } from "@webiny/aws-sdk/client-eventbridge"; import { SecurityPermission } from "@webiny/api-security/types"; import { GenericRecord } from "@webiny/api/types"; -export { PutEventsCommandOutput }; - export * from "./handler/types"; export * from "./response/abstractions"; export * from "./runner/abstractions"; -export interface ITaskConfig { - readonly eventBusName: string; -} - export type ITaskDataInput = GenericRecord; -export enum ITaskLogItemType { +export enum TaskLogItemType { INFO = "info", ERROR = "error" } @@ -42,16 +35,16 @@ export interface ITaskLogItemData { export interface ITaskLogItemBase { message: string; createdOn: string; - type: ITaskLogItemType; + type: TaskLogItemType; data?: ITaskLogItemData; } export interface ITaskLogItemInfo extends ITaskLogItemBase { - type: ITaskLogItemType.INFO; + type: TaskLogItemType.INFO; } export interface ITaskLogItemError extends ITaskLogItemBase { - type: ITaskLogItemType.ERROR; + type: TaskLogItemType.ERROR; error?: IResponseError; } @@ -103,7 +96,7 @@ export interface ITask< createdBy: ITaskIdentity; startedOn?: string; finishedOn?: string; - eventResponse: PutEventsCommandOutput | undefined; + eventResponse: GenericRecord | undefined; iterations: number; parentId?: string; } @@ -189,7 +182,7 @@ export interface ITaskUpdateData< executionName?: string; startedOn?: string; finishedOn?: string; - eventResponse?: PutEventsCommandOutput; + eventResponse?: GenericRecord; iterations?: number; } @@ -273,10 +266,6 @@ export interface ITasksContextCrudObject { onTaskAfterDelete: Topic; } -export interface ITasksContextConfigObject { - config: ITaskConfig; -} - export interface ITasksContextDefinitionObject { getDefinition: < C extends Context = Context, @@ -301,7 +290,7 @@ export interface ITaskAbortParams { message?: string; } -export interface ITasksContextTriggerObject { +export interface ITasksContextServiceObject { trigger: < T = ITaskDataInput, O extends ITaskResponseDoneResultOutput = ITaskResponseDoneResultOutput @@ -314,13 +303,13 @@ export interface ITasksContextTriggerObject { >( params: ITaskAbortParams ) => Promise>; + fetchServiceInfo: (input: ITask | string) => Promise; } export interface ITasksContextObject extends ITasksContextCrudObject, ITasksContextDefinitionObject, - ITasksContextTriggerObject, - ITasksContextConfigObject {} + ITasksContextServiceObject {} export interface Context extends BaseContext { tasks: ITasksContextObject; From 8f43ff3a411a92de39ac45090d7d34c4550908ca Mon Sep 17 00:00:00 2001 From: Leonardo Giacone Date: Wed, 25 Sep 2024 11:11:56 +0200 Subject: [PATCH 60/70] fix(api-headless-cms): improve moveEntryToBin operation (#4292) --- .../src/crud/contentEntry/afterDelete.ts | 6 +++++- .../useCases/DeleteEntry/TransformEntryMoveToBin.ts | 9 --------- .../RestoreEntryFromBin/TransformEntryRestoreFromBin.ts | 9 --------- 3 files changed, 5 insertions(+), 19 deletions(-) diff --git a/packages/api-headless-cms/src/crud/contentEntry/afterDelete.ts b/packages/api-headless-cms/src/crud/contentEntry/afterDelete.ts index 26aaced2915..3d9b80f99ea 100644 --- a/packages/api-headless-cms/src/crud/contentEntry/afterDelete.ts +++ b/packages/api-headless-cms/src/crud/contentEntry/afterDelete.ts @@ -10,8 +10,12 @@ export const assignAfterEntryDelete = (params: AssignAfterEntryDeleteParams) => const { context, onEntryAfterDelete } = params; onEntryAfterDelete.subscribe(async params => { - const { entry, model } = params; + const { entry, model, permanent } = params; + // If the entry is being moved to the trash, we keep the model fields locked because the entry can be restored. + if (!permanent) { + return; + } const { items } = await context.cms.storageOperations.entries.list(model, { where: { entryId_not: entry.entryId, diff --git a/packages/api-headless-cms/src/crud/contentEntry/useCases/DeleteEntry/TransformEntryMoveToBin.ts b/packages/api-headless-cms/src/crud/contentEntry/useCases/DeleteEntry/TransformEntryMoveToBin.ts index d4449d1a66d..99f820ad2cd 100644 --- a/packages/api-headless-cms/src/crud/contentEntry/useCases/DeleteEntry/TransformEntryMoveToBin.ts +++ b/packages/api-headless-cms/src/crud/contentEntry/useCases/DeleteEntry/TransformEntryMoveToBin.ts @@ -2,7 +2,6 @@ import { SecurityIdentity } from "@webiny/api-security/types"; import { entryFromStorageTransform, entryToStorageTransform } from "~/utils/entryStorage"; import { getDate } from "~/utils/date"; import { getIdentity } from "~/utils/identity"; -import { validateModelEntryDataOrThrow } from "~/crud/contentEntry/entryDataValidation"; import { CmsContext, CmsEntry, CmsEntryStorageOperationsMoveToBinParams, CmsModel } from "~/types"; import { ROOT_FOLDER } from "~/constants"; @@ -29,14 +28,6 @@ export class TransformEntryMoveToBin { } private async createDeleteEntryData(model: CmsModel, originalEntry: CmsEntry) { - await validateModelEntryDataOrThrow({ - context: this.context, - model, - data: originalEntry.values, - entry: originalEntry, - skipValidators: ["required"] - }); - const currentDateTime = new Date().toISOString(); const currentIdentity = this.getIdentity(); diff --git a/packages/api-headless-cms/src/crud/contentEntry/useCases/RestoreEntryFromBin/TransformEntryRestoreFromBin.ts b/packages/api-headless-cms/src/crud/contentEntry/useCases/RestoreEntryFromBin/TransformEntryRestoreFromBin.ts index 1f84fa445a5..3a15561d39c 100644 --- a/packages/api-headless-cms/src/crud/contentEntry/useCases/RestoreEntryFromBin/TransformEntryRestoreFromBin.ts +++ b/packages/api-headless-cms/src/crud/contentEntry/useCases/RestoreEntryFromBin/TransformEntryRestoreFromBin.ts @@ -2,7 +2,6 @@ import { SecurityIdentity } from "@webiny/api-security/types"; import { entryFromStorageTransform, entryToStorageTransform } from "~/utils/entryStorage"; import { getDate } from "~/utils/date"; import { getIdentity } from "~/utils/identity"; -import { validateModelEntryDataOrThrow } from "~/crud/contentEntry/entryDataValidation"; import { CmsContext, CmsEntry, CmsEntryStorageOperationsMoveToBinParams, CmsModel } from "~/types"; export class TransformEntryRestoreFromBin { @@ -28,14 +27,6 @@ export class TransformEntryRestoreFromBin { } private async createRestoreFromBinEntryData(model: CmsModel, originalEntry: CmsEntry) { - await validateModelEntryDataOrThrow({ - context: this.context, - model, - data: originalEntry.values, - entry: originalEntry, - skipValidators: ["required"] - }); - const currentDateTime = new Date().toISOString(); const currentIdentity = this.getIdentity(); From 98bc68682f57646319bab783a997090fb71ae985 Mon Sep 17 00:00:00 2001 From: Pavel Denisjuk Date: Wed, 25 Sep 2024 13:13:54 +0200 Subject: [PATCH 61/70] fix(app-headless-cms): ensure time value contains seconds (#4293) --- .../dateTime/DateTimeWithTimezone.tsx | 44 ++++++++----------- .../dateTime/DateTimeWithoutTimezone.tsx | 31 ++++++------- .../plugins/fieldRenderers/dateTime/utils.tsx | 16 +++++++ 3 files changed, 48 insertions(+), 43 deletions(-) diff --git a/packages/app-headless-cms/src/admin/plugins/fieldRenderers/dateTime/DateTimeWithTimezone.tsx b/packages/app-headless-cms/src/admin/plugins/fieldRenderers/dateTime/DateTimeWithTimezone.tsx index 3c1ab4350f4..20a91a5a6c1 100644 --- a/packages/app-headless-cms/src/admin/plugins/fieldRenderers/dateTime/DateTimeWithTimezone.tsx +++ b/packages/app-headless-cms/src/admin/plugins/fieldRenderers/dateTime/DateTimeWithTimezone.tsx @@ -1,4 +1,4 @@ -import React, { useEffect } from "react"; +import React from "react"; import { Input } from "./Input"; import { Select } from "./Select"; import { Grid, Cell } from "@webiny/ui/Grid"; @@ -8,7 +8,9 @@ import { DEFAULT_TIMEZONE, getCurrentDate, getCurrentLocalTime, - getCurrentTimeZone + getCurrentTimeZone, + getHHmmss, + getHHmm } from "./utils"; import { CmsModelField } from "~/types"; import { BindComponentRenderProp } from "@webiny/form"; @@ -37,11 +39,11 @@ const parseDateTime = (value?: string): Pick & { rest: string } = }; }; -const parseTime = (value?: string): Pick => { +const parseTime = (value?: string, defaultTimeZone?: string): Pick => { if (!value) { return { time: "", - timezone: "" + timezone: defaultTimeZone || "" }; } const sign = value.includes("+") ? "+" : "-"; @@ -62,20 +64,15 @@ export const DateTimeWithTimezone = ({ bind, trailingIcon, field }: DateTimeWith const defaultTimeZone = getCurrentTimeZone() || DEFAULT_TIMEZONE; // "2020-05-18T09:00+10:00" - const initialValue = getDefaultFieldValue(field, bind, () => { - const date = new Date(); - return `${getCurrentDate(date)}T${getCurrentLocalTime(date)}${defaultTimeZone}`; - }); - const { date, rest } = parseDateTime(initialValue); - const { time, timezone = defaultTimeZone } = parseTime(rest); + const value = + bind.value || + getDefaultFieldValue(field, bind, () => { + const date = new Date(); + return `${getCurrentDate(date)}T${getCurrentLocalTime(date)}${defaultTimeZone}`; + }); - const bindValue = bind.value || ""; - useEffect(() => { - if (!date || !time || !timezone || bindValue === initialValue) { - return; - } - bind.onChange(initialValue); - }, [bindValue]); + const { date, rest } = parseDateTime(value); + const { time, timezone } = parseTime(rest, defaultTimeZone); const cellSize = trailingIcon ? 3 : 4; @@ -93,11 +90,8 @@ export const DateTimeWithTimezone = ({ bind, trailingIcon, field }: DateTimeWith } return bind.onChange(""); } - return bind.onChange( - `${value}T${time || getCurrentLocalTime()}${ - timezone || defaultTimeZone - }` - ); + + return bind.onChange(`${value}T${getHHmmss(time)}${timezone}`); } }} field={{ @@ -111,7 +105,7 @@ export const DateTimeWithTimezone = ({ bind, trailingIcon, field }: DateTimeWith { if (!value) { if (!bind.value) { @@ -120,7 +114,7 @@ export const DateTimeWithTimezone = ({ bind, trailingIcon, field }: DateTimeWith return bind.onChange(""); } return bind.onChange( - `${date || getCurrentDate()}T${value}${timezone || defaultTimeZone}` + `${date || getCurrentDate()}T${getHHmmss(value)}${timezone}` ); } }} @@ -135,7 +129,7 @@ export const DateTimeWithTimezone = ({ bind, trailingIcon, field }: DateTimeWith { if (!value) { if (!bind.value) { @@ -109,7 +104,7 @@ export const DateTimeWithoutTimezone = ({ } return bind.onChange(""); } - return bind.onChange(`${date || getCurrentDate()} ${value}`); + return bind.onChange(`${date || getCurrentDate()} ${getHHmmss(value)}`); } }} field={{ diff --git a/packages/app-headless-cms/src/admin/plugins/fieldRenderers/dateTime/utils.tsx b/packages/app-headless-cms/src/admin/plugins/fieldRenderers/dateTime/utils.tsx index f145ec76a1a..28d1bb6722c 100644 --- a/packages/app-headless-cms/src/admin/plugins/fieldRenderers/dateTime/utils.tsx +++ b/packages/app-headless-cms/src/admin/plugins/fieldRenderers/dateTime/utils.tsx @@ -58,6 +58,22 @@ export const getCurrentDate = (date?: Date): string => { return `${year}-${month}-${day}`; }; +export const getHHmm = (time?: string) => { + if (!time) { + return ""; + } + const parsableTime = time || getCurrentLocalTime(); + return parsableTime.split(":").slice(0, 2).join(":"); +}; + +// Ensure a valid HH:mm:ss string, ending with :00 for seconds. +export const getHHmmss = (time?: string) => { + const parsableTime = time || getCurrentLocalTime(); + const parts = [...parsableTime.split(":").slice(0, 2), "00"]; + + return parts.join(":"); +}; + const deleteIconStyles = css({ width: "100% !important", height: "100% !important", From af4195f09c6ae779066c1810f721f843ff81ca81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20Zori=C4=87?= Date: Thu, 26 Sep 2024 10:56:17 +0200 Subject: [PATCH 62/70] fix(api-headless-cms-import-export): check for real status of child tasks [skip ci] (#4294) --- .../crud/utils/makeSureModelsAreIdentical.ts | 4 +- .../tasks/domain/ImportFromUrlController.ts | 11 +++-- .../abstractions/ImportFromUrlController.ts | 40 +++++----------- .../getChildTasks.ts | 47 ++++++++++++++++++- packages/tasks/src/context.ts | 4 +- packages/tasks/src/crud/service.tasks.ts | 9 ++-- .../tasks/src/plugins/TaskServicePlugin.ts | 11 ++--- .../EventBridgeEventTransportPlugin.ts | 2 +- .../StepFunctionServicePlugin.ts | 12 +++-- .../src/{crud/transport => service}/index.ts | 4 +- packages/tasks/src/types.ts | 5 +- 11 files changed, 96 insertions(+), 53 deletions(-) rename packages/tasks/src/{crud/transport => service}/EventBridgeEventTransportPlugin.ts (97%) rename packages/tasks/src/{crud/transport => service}/StepFunctionServicePlugin.ts (91%) rename packages/tasks/src/{crud/transport => service}/index.ts (78%) diff --git a/packages/api-headless-cms-import-export/src/crud/utils/makeSureModelsAreIdentical.ts b/packages/api-headless-cms-import-export/src/crud/utils/makeSureModelsAreIdentical.ts index f91cc8f8741..51cbbb5dbc8 100644 --- a/packages/api-headless-cms-import-export/src/crud/utils/makeSureModelsAreIdentical.ts +++ b/packages/api-headless-cms-import-export/src/crud/utils/makeSureModelsAreIdentical.ts @@ -66,7 +66,9 @@ export const makeSureModelsAreIdentical = (params: IMakeSureModelsAreIdenticalPa message: `Field "${value.field.fieldId}" not found in the model provided via the JSON data.`, code: "MODEL_FIELD_NOT_FOUND", data: { - ...value + field: value, + targetValues, + modelValues } }); } diff --git a/packages/api-headless-cms-import-export/src/tasks/domain/ImportFromUrlController.ts b/packages/api-headless-cms-import-export/src/tasks/domain/ImportFromUrlController.ts index 213ac2f1665..c6cc700a30f 100644 --- a/packages/api-headless-cms-import-export/src/tasks/domain/ImportFromUrlController.ts +++ b/packages/api-headless-cms-import-export/src/tasks/domain/ImportFromUrlController.ts @@ -1,7 +1,8 @@ import type { ITaskResponseResult, ITaskRunParams } from "@webiny/tasks"; -import type { +import { IImportFromUrlController, IImportFromUrlControllerInput, + IImportFromUrlControllerInputStepsStep, IImportFromUrlControllerOutput } from "~/tasks/domain/abstractions/ImportFromUrlController"; import { IImportFromUrlControllerInputStep } from "~/tasks/domain/abstractions/ImportFromUrlController"; @@ -10,7 +11,7 @@ import { ImportFromUrlControllerDownloadStep } from "~/tasks/domain/importFromUr import { ImportFromUrlControllerProcessEntriesStep } from "./importFromUrlControllerSteps/ImportFromUrlControllerProcessEntriesStep"; import { ImportFromUrlControllerProcessAssetsStep } from "./importFromUrlControllerSteps/ImportFromUrlControllerProcessAssetsStep"; -const getDefaultStepValues = () => { +const getDefaultStepValues = (): IImportFromUrlControllerInputStepsStep => { return { files: [], triggered: false, @@ -56,7 +57,7 @@ export class ImportFromUrlController< const downloadStep = steps[IImportFromUrlControllerInputStep.DOWNLOAD] || getDefaultStepValues(); - if (!downloadStep.done) { + if (!downloadStep.finished) { const step = new ImportFromUrlControllerDownloadStep(); return await step.execute(params); } else if (downloadStep.failed.length) { @@ -69,7 +70,7 @@ export class ImportFromUrlController< const processEntriesStep = steps[IImportFromUrlControllerInputStep.PROCESS_ENTRIES] || getDefaultStepValues(); - if (!processEntriesStep.done) { + if (!processEntriesStep.finished) { const step = new ImportFromUrlControllerProcessEntriesStep(); return await step.execute(params); } else if (processEntriesStep.failed.length) { @@ -82,7 +83,7 @@ export class ImportFromUrlController< const processAssetsStep = steps[IImportFromUrlControllerInputStep.PROCESS_ASSETS] || getDefaultStepValues(); - if (!processAssetsStep.done) { + if (!processAssetsStep.finished) { const step = new ImportFromUrlControllerProcessAssetsStep(); return await step.execute(params); } else if (processAssetsStep.failed.length) { diff --git a/packages/api-headless-cms-import-export/src/tasks/domain/abstractions/ImportFromUrlController.ts b/packages/api-headless-cms-import-export/src/tasks/domain/abstractions/ImportFromUrlController.ts index ea93fa7f0f3..21a0a88d46e 100644 --- a/packages/api-headless-cms-import-export/src/tasks/domain/abstractions/ImportFromUrlController.ts +++ b/packages/api-headless-cms-import-export/src/tasks/domain/abstractions/ImportFromUrlController.ts @@ -16,34 +16,20 @@ export enum IImportFromUrlControllerInputStep { PROCESS_ASSETS = "processAssets" } +export interface IImportFromUrlControllerInputStepsStep { + files: string[]; + triggered: boolean; + finished: boolean; + done: string[]; + failed: string[]; + invalid: string[]; + aborted: string[]; +} + export interface IImportFromUrlControllerInputSteps { - [IImportFromUrlControllerInputStep.DOWNLOAD]?: { - files: string[]; - triggered: boolean; - finished: boolean; - done: string[]; - failed: string[]; - invalid: string[]; - aborted: string[]; - }; - [IImportFromUrlControllerInputStep.PROCESS_ENTRIES]?: { - files: string[]; - triggered: boolean; - finished: boolean; - done: string[]; - failed: string[]; - invalid: string[]; - aborted: string[]; - }; - [IImportFromUrlControllerInputStep.PROCESS_ASSETS]?: { - files: string[]; - triggered: boolean; - finished: boolean; - done: string[]; - failed: string[]; - invalid: string[]; - aborted: string[]; - }; + [IImportFromUrlControllerInputStep.DOWNLOAD]?: IImportFromUrlControllerInputStepsStep; + [IImportFromUrlControllerInputStep.PROCESS_ENTRIES]?: IImportFromUrlControllerInputStepsStep; + [IImportFromUrlControllerInputStep.PROCESS_ASSETS]?: IImportFromUrlControllerInputStepsStep; } export interface IImportFromUrlControllerInput { diff --git a/packages/api-headless-cms-import-export/src/tasks/domain/importFromUrlControllerSteps/getChildTasks.ts b/packages/api-headless-cms-import-export/src/tasks/domain/importFromUrlControllerSteps/getChildTasks.ts index e37a7187778..45a2a2143ea 100644 --- a/packages/api-headless-cms-import-export/src/tasks/domain/importFromUrlControllerSteps/getChildTasks.ts +++ b/packages/api-headless-cms-import-export/src/tasks/domain/importFromUrlControllerSteps/getChildTasks.ts @@ -1,6 +1,7 @@ import type { ITask, ITaskResponseDoneResultOutput } from "@webiny/tasks"; import { TaskDataStatus } from "@webiny/tasks"; import type { Context } from "~/types"; +import { IStepFunctionServiceFetchResult } from "@webiny/tasks/service/StepFunctionServicePlugin"; export interface IGetChildTasksParams { context: Context; @@ -8,6 +9,31 @@ export interface IGetChildTasksParams { definition: string; } +const mapServiceStatusToTaskStatus = ( + task: ITask, + serviceInfo: IStepFunctionServiceFetchResult | null +) => { + if (!serviceInfo) { + console.log(`Service info is missing for task ${task.id} (${task.definitionId}).`); + return null; + } + if (serviceInfo.status === "RUNNING") { + return TaskDataStatus.RUNNING; + } else if (serviceInfo.status === "SUCCEEDED") { + return TaskDataStatus.SUCCESS; + } else if (serviceInfo.status === "FAILED") { + return TaskDataStatus.FAILED; + } else if (serviceInfo.status === "ABORTED") { + return TaskDataStatus.ABORTED; + } else if (serviceInfo.status === "TIMED_OUT" || serviceInfo.status === "PENDING_REDRIVE") { + console.log( + `Service status is ${serviceInfo.status} for task ${task.id} (${task.definitionId}).` + ); + return null; + } + return TaskDataStatus.PENDING; +}; + export const getChildTasks = async ({ context, task, @@ -24,14 +50,33 @@ export const getChildTasks = async ( where: { parentId: task.id, definitionId: definition - } + }, + limit: 100000 }); for (const task of items) { collection.push(task); + if ( task.taskStatus === TaskDataStatus.RUNNING || task.taskStatus === TaskDataStatus.PENDING ) { + /** + * We also need to check the actual status of the service. + * It can happen that the task is marked as running, but the service is not running. + */ + const serviceInfo = await context.tasks.fetchServiceInfo(task); + const status = mapServiceStatusToTaskStatus(task, serviceInfo); + + if (status === null || !serviceInfo) { + invalid.push(task.id); + continue; + } else if (status !== task.taskStatus) { + console.error( + `Status of the task is not same as the status of the service (task: ${task.taskStatus}, service: ${status} / ${serviceInfo.status}).` + ); + invalid.push(task.id); + continue; + } running.push(task.id); continue; } else if (task.taskStatus === TaskDataStatus.SUCCESS) { diff --git a/packages/tasks/src/context.ts b/packages/tasks/src/context.ts index 90cc88f38f4..bdb12e5b065 100644 --- a/packages/tasks/src/context.ts +++ b/packages/tasks/src/context.ts @@ -6,7 +6,7 @@ import { createDefinitionCrud } from "./crud/definition.tasks"; import { createServiceCrud } from "~/crud/service.tasks"; import { createTaskCrud } from "./crud/crud.tasks"; import { createTestingRunTask } from "~/tasks/testingRunTask"; -import { createTransportPlugins } from "~/crud/transport"; +import { createServicePlugins } from "~/service"; const createTasksCrud = () => { const plugin = new ContextPlugin(async context => { @@ -23,7 +23,7 @@ const createTasksCrud = () => { }; const createTasksContext = (): Plugin[] => { - return [...createTransportPlugins(), ...createTaskModel(), createTasksCrud()]; + return [...createServicePlugins(), ...createTaskModel(), createTasksCrud()]; }; export const createBackgroundTaskContext = (): Plugin[] => { diff --git a/packages/tasks/src/crud/service.tasks.ts b/packages/tasks/src/crud/service.tasks.ts index 1a393999eb6..5856cff9455 100644 --- a/packages/tasks/src/crud/service.tasks.ts +++ b/packages/tasks/src/crud/service.tasks.ts @@ -12,7 +12,8 @@ import type { } from "~/types"; import { TaskDataStatus, TaskLogItemType } from "~/types"; import { NotFoundError } from "@webiny/handler-graphql"; -import { createService } from "~/service/createService"; +import { createService } from "~/service"; +import { IStepFunctionServiceFetchResult } from "~/service/StepFunctionServicePlugin"; const MAX_DELAY_DAYS = 355; const MAX_DELAY_SECONDS = MAX_DELAY_DAYS * 24 * 60 * 60; @@ -101,7 +102,9 @@ export const createServiceCrud = (context: Context): ITasksContextServiceObject eventResponse: result }); }, - fetchServiceInfo: async (input: ITask | string) => { + fetchServiceInfo: async ( + input: ITask | string + ): Promise => { const task = typeof input === "object" ? input : await context.tasks.getTask(input); if (!task && typeof input === "string") { throw new NotFoundError(`Task "${input}" was not found!`); @@ -112,7 +115,7 @@ export const createServiceCrud = (context: Context): ITasksContextServiceObject } try { - return await service.fetch(task); + return (await service.fetch(task)) as IStepFunctionServiceFetchResult | null; } catch (ex) { console.log("Service fetch error."); console.error(ex); diff --git a/packages/tasks/src/plugins/TaskServicePlugin.ts b/packages/tasks/src/plugins/TaskServicePlugin.ts index 4808f516192..be0beaa223d 100644 --- a/packages/tasks/src/plugins/TaskServicePlugin.ts +++ b/packages/tasks/src/plugins/TaskServicePlugin.ts @@ -1,6 +1,5 @@ import { Plugin } from "@webiny/plugins"; import { Context, ITask } from "~/types"; -import { GenericRecord } from "@webiny/api/types"; export interface ITaskServiceCreatePluginParams { context: Context; @@ -10,16 +9,16 @@ export interface ITaskServiceCreatePluginParams { export type ITaskServiceTask = Pick; -export interface ITaskService { - send(task: ITaskServiceTask, delay: number): Promise; - fetch(task: ITask): Promise; +export interface ITaskService { + send(task: ITaskServiceTask, delay: number): Promise; + fetch(task: ITask): Promise; } export interface ITaskServicePluginParams { default?: boolean; } -export abstract class TaskServicePlugin extends Plugin { +export abstract class TaskServicePlugin extends Plugin { public static override readonly type: string = "tasks.taskService"; public readonly default: boolean; @@ -28,5 +27,5 @@ export abstract class TaskServicePlugin extends Plugin { this.default = !!params?.default; } - public abstract createService(params: ITaskServiceCreatePluginParams): ITaskService; + public abstract createService(params: ITaskServiceCreatePluginParams): ITaskService; } diff --git a/packages/tasks/src/crud/transport/EventBridgeEventTransportPlugin.ts b/packages/tasks/src/service/EventBridgeEventTransportPlugin.ts similarity index 97% rename from packages/tasks/src/crud/transport/EventBridgeEventTransportPlugin.ts rename to packages/tasks/src/service/EventBridgeEventTransportPlugin.ts index aafb20ebd69..76cdcd1be84 100644 --- a/packages/tasks/src/crud/transport/EventBridgeEventTransportPlugin.ts +++ b/packages/tasks/src/service/EventBridgeEventTransportPlugin.ts @@ -10,7 +10,7 @@ import { EventBridgeClient, PutEventsCommand } from "@webiny/aws-sdk/client-even import { WebinyError } from "@webiny/error"; import { GenericRecord } from "@webiny/api/types"; -class EventBridgeService implements ITaskService { +class EventBridgeService implements ITaskService { protected readonly context: Context; protected readonly getTenant: () => string; protected readonly getLocale: () => string; diff --git a/packages/tasks/src/crud/transport/StepFunctionServicePlugin.ts b/packages/tasks/src/service/StepFunctionServicePlugin.ts similarity index 91% rename from packages/tasks/src/crud/transport/StepFunctionServicePlugin.ts rename to packages/tasks/src/service/StepFunctionServicePlugin.ts index db606a6c843..de538b86948 100644 --- a/packages/tasks/src/crud/transport/StepFunctionServicePlugin.ts +++ b/packages/tasks/src/service/StepFunctionServicePlugin.ts @@ -4,9 +4,9 @@ import { ITaskServiceTask, TaskServicePlugin } from "~/plugins"; -import { GenericRecord } from "@webiny/api/types"; import { createStepFunctionClient, + DescribeExecutionCommandOutput, describeExecutionFactory, triggerStepFunctionFactory } from "@webiny/aws-sdk/client-sfn"; @@ -15,11 +15,13 @@ import { generateAlphaNumericId } from "@webiny/utils"; import { ServiceDiscovery } from "@webiny/api"; import { ITask } from "~/types"; +export type IStepFunctionServiceFetchResult = DescribeExecutionCommandOutput; + export interface IDetailWrapper { detail: T; } -class StepFunctionService implements ITaskService { +class StepFunctionService implements ITaskService { private readonly getTenant: () => string; private readonly getLocale: () => string; private readonly trigger: ReturnType; @@ -71,7 +73,7 @@ class StepFunctionService implements ITaskService { } } - public async fetch(task: ITask): Promise { + public async fetch(task: ITask): Promise { const executionArn = task.eventResponse?.executionArn; if (!executionArn) { console.error(`Execution ARN not found in task "${task.id}".`); @@ -81,7 +83,7 @@ class StepFunctionService implements ITaskService { const result = await this.get({ executionArn }); - return (result || null) as R; + return result || null; } catch (ex) { console.log("Could not get the execution details."); console.error(ex); @@ -90,7 +92,7 @@ class StepFunctionService implements ITaskService { } } -export class StepFunctionServicePlugin extends TaskServicePlugin { +export class StepFunctionServicePlugin extends TaskServicePlugin { public override name = "task.stepFunctionTriggerTransport"; public createService(params: ITaskServiceCreatePluginParams) { diff --git a/packages/tasks/src/crud/transport/index.ts b/packages/tasks/src/service/index.ts similarity index 78% rename from packages/tasks/src/crud/transport/index.ts rename to packages/tasks/src/service/index.ts index d6f4c691da0..1d6f503219b 100644 --- a/packages/tasks/src/crud/transport/index.ts +++ b/packages/tasks/src/service/index.ts @@ -1,9 +1,11 @@ import { EventBridgeEventTransportPlugin } from "./EventBridgeEventTransportPlugin"; import { StepFunctionServicePlugin } from "./StepFunctionServicePlugin"; -export const createTransportPlugins = () => { +export const createServicePlugins = () => { return [ new StepFunctionServicePlugin({ default: true }), new EventBridgeEventTransportPlugin() ]; }; + +export * from "./createService"; diff --git a/packages/tasks/src/types.ts b/packages/tasks/src/types.ts index 04a38c8d2fc..cad417447a0 100644 --- a/packages/tasks/src/types.ts +++ b/packages/tasks/src/types.ts @@ -16,6 +16,7 @@ import { import { IIsCloseToTimeoutCallable, ITaskManagerStore } from "./runner/abstractions"; import { SecurityPermission } from "@webiny/api-security/types"; import { GenericRecord } from "@webiny/api/types"; +import { IStepFunctionServiceFetchResult } from "~/service/StepFunctionServicePlugin"; export * from "./handler/types"; export * from "./response/abstractions"; @@ -303,7 +304,9 @@ export interface ITasksContextServiceObject { >( params: ITaskAbortParams ) => Promise>; - fetchServiceInfo: (input: ITask | string) => Promise; + fetchServiceInfo: ( + input: ITask | string + ) => Promise; } export interface ITasksContextObject From a9a1ef1ba725f0735b39bd354b1be9f73313b9ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20Zori=C4=87?= Date: Thu, 26 Sep 2024 13:19:45 +0200 Subject: [PATCH 63/70] fix(api-headless-cms): singleton model reference field (#4296) --- .../contentAPI/mocks/contentModels.ts | 103 ++++++++++++++++++ .../contentAPI/singletonContentEntry.test.ts | 2 +- .../useSingletonCategoryHandler.ts | 2 +- .../graphql/schema/createSingularResolvers.ts | 2 +- .../src/graphql/schema/createSingularSDL.ts | 2 +- .../src/utils/getSchemaFromFieldPlugins.ts | 6 +- 6 files changed, 111 insertions(+), 6 deletions(-) diff --git a/packages/api-headless-cms/__tests__/contentAPI/mocks/contentModels.ts b/packages/api-headless-cms/__tests__/contentAPI/mocks/contentModels.ts index 50510526ec1..1643a2658f3 100644 --- a/packages/api-headless-cms/__tests__/contentAPI/mocks/contentModels.ts +++ b/packages/api-headless-cms/__tests__/contentAPI/mocks/contentModels.ts @@ -253,6 +253,109 @@ const models: CmsModel[] = [ tenant: "root", webinyVersion }, + // category + { + createdOn: new Date().toISOString(), + savedOn: new Date().toISOString(), + locale: "en-US", + titleFieldId: "title", + lockedFields: [], + name: "Category Singleton", + description: "Product category Singleton", + modelId: "categorySingleton", + singularApiName: "CategoryApiNameWhichIsABitDifferentThanModelIdSingleton", + pluralApiName: "CategoriesApiModelSingleton", + group: { + id: contentModelGroup.id, + name: contentModelGroup.name + }, + layout: [[ids.field11], [ids.field12]], + fields: [ + { + id: ids.field11, + multipleValues: false, + helpText: "", + label: "Title", + type: "text", + storageId: "text@titleStorageId", + fieldId: "title", + validation: [ + { + name: "required", + message: "This field is required" + }, + { + name: "minLength", + message: "Enter at least 3 characters", + settings: { + min: 3.0 + } + } + ], + listValidation: [], + placeholderText: "placeholder text", + predefinedValues: { + enabled: false, + values: [] + }, + renderer: { + name: "renderer" + } + }, + { + id: ids.field12, + multipleValues: false, + helpText: "", + label: "Slug", + type: "text", + storageId: "text@slugStorageId", + fieldId: "slug", + validation: [ + { + name: "required", + message: "This field is required" + } + ], + listValidation: [], + placeholderText: "placeholder text", + predefinedValues: { + enabled: false, + values: [] + }, + renderer: { + name: "renderer" + } + }, + { + id: ids.field34, + multipleValues: false, + helpText: "", + label: "Category", + type: "ref", + storageId: "ref@categoryRef", + fieldId: "categoryRef", + validation: [], + listValidation: [], + placeholderText: "placeholder text", + settings: { + models: [ + { + modelId: "categorySingleton" + } + ] + }, + predefinedValues: { + enabled: false, + values: [] + }, + renderer: { + name: "renderer" + } + } + ], + tenant: "root", + webinyVersion + }, // product { createdOn: new Date().toISOString(), diff --git a/packages/api-headless-cms/__tests__/contentAPI/singletonContentEntry.test.ts b/packages/api-headless-cms/__tests__/contentAPI/singletonContentEntry.test.ts index 893e9515939..6659bd37add 100644 --- a/packages/api-headless-cms/__tests__/contentAPI/singletonContentEntry.test.ts +++ b/packages/api-headless-cms/__tests__/contentAPI/singletonContentEntry.test.ts @@ -5,7 +5,7 @@ import { CMS_MODEL_SINGLETON_TAG } from "~/constants"; describe("singleton model content entries", () => { const plugins = createPluginFromCmsModel({ - ...getCmsModel("category"), + ...getCmsModel("categorySingleton"), tags: [CMS_MODEL_SINGLETON_TAG] }); diff --git a/packages/api-headless-cms/__tests__/testHelpers/useSingletonCategoryHandler.ts b/packages/api-headless-cms/__tests__/testHelpers/useSingletonCategoryHandler.ts index b0a1465d905..af5b5dc5cb2 100644 --- a/packages/api-headless-cms/__tests__/testHelpers/useSingletonCategoryHandler.ts +++ b/packages/api-headless-cms/__tests__/testHelpers/useSingletonCategoryHandler.ts @@ -66,7 +66,7 @@ const updateCategoryMutation = (model: CmsModel) => { }; export const useSingletonCategoryHandler = (params: GraphQLHandlerParams) => { - const model = getCmsModel("category"); + const model = getCmsModel("categorySingleton"); const contentHandler = useGraphQLHandler(params); return { diff --git a/packages/api-headless-cms/src/graphql/schema/createSingularResolvers.ts b/packages/api-headless-cms/src/graphql/schema/createSingularResolvers.ts index 4985f2d790d..c1327e8d742 100644 --- a/packages/api-headless-cms/src/graphql/schema/createSingularResolvers.ts +++ b/packages/api-headless-cms/src/graphql/schema/createSingularResolvers.ts @@ -31,7 +31,7 @@ export const createSingularResolvers: CreateSingularResolvers = ({ } const createFieldResolvers = createFieldResolversFactory({ - endpointType: "manage", + endpointType: type, models, model, fieldTypePlugins diff --git a/packages/api-headless-cms/src/graphql/schema/createSingularSDL.ts b/packages/api-headless-cms/src/graphql/schema/createSingularSDL.ts index f0728b0a5b0..55f1e183c86 100644 --- a/packages/api-headless-cms/src/graphql/schema/createSingularSDL.ts +++ b/packages/api-headless-cms/src/graphql/schema/createSingularSDL.ts @@ -34,7 +34,7 @@ export const createSingularSDL: CreateSingularSDL = ({ models, model, fields: model.fields, - type: "manage", + type, fieldTypePlugins }); diff --git a/packages/api-headless-cms/src/utils/getSchemaFromFieldPlugins.ts b/packages/api-headless-cms/src/utils/getSchemaFromFieldPlugins.ts index 14cb1f7aaea..09d2d224d68 100644 --- a/packages/api-headless-cms/src/utils/getSchemaFromFieldPlugins.ts +++ b/packages/api-headless-cms/src/utils/getSchemaFromFieldPlugins.ts @@ -34,13 +34,15 @@ interface Params { export const createGraphQLSchemaPluginFromFieldPlugins = (params: Params) => { const { models, fieldTypePlugins, type, createPlugin = defaultCreatePlugin } = params; + const apiType = TYPE_MAP[type]; + const plugins: ICmsGraphQLSchemaPlugin[] = []; for (const key in fieldTypePlugins) { const fieldTypePlugin = fieldTypePlugins[key]; - if (!TYPE_MAP[type] || !fieldTypePlugin[TYPE_MAP[type]]) { + if (!apiType || !fieldTypePlugin[apiType]) { continue; } - const createSchema = fieldTypePlugin[TYPE_MAP[type]].createSchema; + const createSchema = fieldTypePlugin[apiType].createSchema; // Render gql types generated by field type plugins if (!createSchema) { continue; From c35c61aa7379631e2e45c21a11c985b6de65a414 Mon Sep 17 00:00:00 2001 From: Pavel Denisjuk Date: Thu, 26 Sep 2024 13:34:31 +0200 Subject: [PATCH 64/70] fix(app-headless-cms): add missing contexts to singleton entry form (#4298) --- packages/app-aco/src/contexts/app.tsx | 4 +- .../components/NewReferencedEntryDialog.tsx | 51 +++++++++++++++---- 2 files changed, 43 insertions(+), 12 deletions(-) diff --git a/packages/app-aco/src/contexts/app.tsx b/packages/app-aco/src/contexts/app.tsx index 43843ab4cb2..e3afcbbb39f 100644 --- a/packages/app-aco/src/contexts/app.tsx +++ b/packages/app-aco/src/contexts/app.tsx @@ -60,7 +60,7 @@ interface CreateAppParams { getFields?: () => AcoModelField[]; } -const createApp = (data: CreateAppParams): AcoApp => { +export const createAppFromModel = (data: CreateAppParams): AcoApp => { return { ...data, getFields: @@ -138,7 +138,7 @@ export const AcoAppProvider = ({ ...prev, loading: false, model: inputModel, - app: createApp({ + app: createAppFromModel({ id, model: inputModel, getFields diff --git a/packages/app-headless-cms/src/admin/plugins/fieldRenderers/ref/components/NewReferencedEntryDialog.tsx b/packages/app-headless-cms/src/admin/plugins/fieldRenderers/ref/components/NewReferencedEntryDialog.tsx index 8440f8bd78e..113d99f4911 100644 --- a/packages/app-headless-cms/src/admin/plugins/fieldRenderers/ref/components/NewReferencedEntryDialog.tsx +++ b/packages/app-headless-cms/src/admin/plugins/fieldRenderers/ref/components/NewReferencedEntryDialog.tsx @@ -17,12 +17,19 @@ import { import { useCms } from "~/admin/hooks"; import { FullWidthDialog } from "./dialog/Dialog"; import { NavigateFolderProvider as AbstractNavigateFolderProvider } from "@webiny/app-aco/contexts/navigateFolder"; +import { SearchRecordsProvider } from "@webiny/app-aco/contexts/records"; import { FolderTree, useNavigateFolder } from "@webiny/app-aco"; import styled from "@emotion/styled"; import { Elevation } from "@webiny/ui/Elevation"; import { SplitView, LeftPanel, RightPanel } from "@webiny/app-admin/components/SplitView"; import { CircularProgress } from "@webiny/ui/Progress"; import { usePersistEntry } from "~/admin/hooks/usePersistEntry"; +import { + AcoAppContext, + AcoAppProviderContext, + createAppFromModel +} from "@webiny/app-aco/contexts/app"; +import { DialogsProvider } from "@webiny/app-admin"; const t = i18n.ns("app-headless-cms/admin/fields/ref"); @@ -159,22 +166,46 @@ export const NewReferencedEntryDialog = ({ }, [onChange, model] ); + if (!model) { return null; } + const acoAppContext: AcoAppProviderContext = { + app: createAppFromModel({ + model, + id: `cms:${model.modelId}` + }), + mode: "cms", + client: apolloClient, + model, + folderIdPath: "wbyAco_location.folderId", + folderIdInPath: "wbyAco_location.folderId_in", + loading: false + }; + return ( - - - - - + + + + + + + + + + + - + ); }; From 1d2077734249f6c12c76b5b1ae3611eec1505f13 Mon Sep 17 00:00:00 2001 From: Pavel Denisjuk Date: Thu, 26 Sep 2024 13:34:53 +0200 Subject: [PATCH 65/70] fix(ui): make dropdown fill the available space (#4297) --- packages/ui/src/Select/styled.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ui/src/Select/styled.tsx b/packages/ui/src/Select/styled.tsx index d24690790b8..787574e6a31 100644 --- a/packages/ui/src/Select/styled.tsx +++ b/packages/ui/src/Select/styled.tsx @@ -8,7 +8,7 @@ export const webinySelect = css` background-color: transparent; border-color: transparent; color: var(--webiny-theme-color-primary); - min-width: 200px; + width: 100%; .rmwc-select__native-control { opacity: 0; From 7a1603edf3ce733e19c234a78c702afdff90e84e Mon Sep 17 00:00:00 2001 From: adrians5j Date: Thu, 26 Sep 2024 10:36:49 +0200 Subject: [PATCH 66/70] fix: remove `data-on-enter` part of the condition (not used anywhere) Searched the whole codebase for usages of `data-on-enter` and didn't find any. This was also causing the cmd+enter not to submit the form correctly in case it was pressed before the value was stored in the form state (delayed on change). --- packages/ui/src/DelayedOnChange/DelayedOnChange.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/ui/src/DelayedOnChange/DelayedOnChange.ts b/packages/ui/src/DelayedOnChange/DelayedOnChange.ts index 03656b476f1..636ebbf20d5 100644 --- a/packages/ui/src/DelayedOnChange/DelayedOnChange.ts +++ b/packages/ui/src/DelayedOnChange/DelayedOnChange.ts @@ -8,6 +8,7 @@ const emptyFunction = (): undefined => { export interface ApplyValueCb { (value: TValue): void; } + /** * This component is used to wrap Input and Textarea components to optimize form re-render. * These 2 are the only components that trigger form model change on each character input. @@ -145,7 +146,7 @@ export const DelayedOnChange = ({ if (ev.key === "Tab") { applyValue((ev.target as HTMLInputElement).value as any as TValue); realOnKeyDown(ev); - } else if (ev.key === "Enter" && props["data-on-enter"]) { + } else if (ev.key === "Enter") { applyValue((ev.target as HTMLInputElement).value as any as TValue); realOnKeyDown(ev); } else { From d2c074f24c3ca00c1084e8113a8edeb0165bb936 Mon Sep 17 00:00:00 2001 From: adrians5j Date: Thu, 26 Sep 2024 10:39:35 +0200 Subject: [PATCH 67/70] fix: remove `console.log` call --- .../admin/views/contentModels/NewContentModelDialog.tsx | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/packages/app-headless-cms/src/admin/views/contentModels/NewContentModelDialog.tsx b/packages/app-headless-cms/src/admin/views/contentModels/NewContentModelDialog.tsx index 66ad6fecb33..83ded9896be 100644 --- a/packages/app-headless-cms/src/admin/views/contentModels/NewContentModelDialog.tsx +++ b/packages/app-headless-cms/src/admin/views/contentModels/NewContentModelDialog.tsx @@ -288,12 +288,7 @@ const NewContentModelDialog = ({ open, onClose }: NewContentModelDialogProps) => - { - console.log("submitting click", data); - submit(ev); - }} - > + + {t`Create Model`} From 5f712dd44899cb3632ea0f671812ee7378bca205 Mon Sep 17 00:00:00 2001 From: adrians5j Date: Thu, 26 Sep 2024 13:27:43 +0200 Subject: [PATCH 68/70] fix: remove `console.log` call --- .../admin/views/contentModels/NewContentModelDialog.tsx | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/packages/app-headless-cms/src/admin/views/contentModels/NewContentModelDialog.tsx b/packages/app-headless-cms/src/admin/views/contentModels/NewContentModelDialog.tsx index 83ded9896be..f9a780911c5 100644 --- a/packages/app-headless-cms/src/admin/views/contentModels/NewContentModelDialog.tsx +++ b/packages/app-headless-cms/src/admin/views/contentModels/NewContentModelDialog.tsx @@ -164,13 +164,7 @@ const NewContentModelDialog = ({ open, onClose }: NewContentModelDialogProps) => return ( {open && ( - - data={{ group, singleton: false }} - onSubmit={data => { - console.log("submitting", data); - onSubmit(data); - }} - > + data={{ group, singleton: false }} onSubmit={onSubmit}> {({ Bind, submit, data }) => { return ( <> From 4e1127ec35aefa2e3c00d13f31a48b1259918426 Mon Sep 17 00:00:00 2001 From: Leonardo Giacone Date: Thu, 26 Sep 2024 14:46:56 +0200 Subject: [PATCH 69/70] fix(api-headless-cms-bulk-actions): fix task execution for large entries list (#4283) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bruno Zorić --- .../__tests__/tasks/createBulkAction.test.ts | 6 +- .../src/index.ts | 9 +- .../src/plugins/createBulkAction.ts | 4 +- .../src/plugins/createBulkActionTasks.ts | 110 ++++++++++++------ .../src/tasks/createBulkActionEntriesTasks.ts | 3 +- .../src/tasks/index.ts | 8 +- .../src/types.ts | 13 ++- .../src/useCases/ListNotPublishedEntries.ts | 36 ++++++ .../src/useCases/index.ts | 1 + .../ChildTaskCleanup/ChildTasksCleanup.ts | 19 ++- .../CreateTasksByModel/CreateTasksByModel.ts | 70 +++++------ .../internals/CreateTasksByModel/TaskCache.ts | 8 ++ .../internals/ProcessTask/ProcessTask.ts | 24 ++-- .../ProcessTasksByModel.ts | 23 ++-- .../api-headless-cms-tasks-ddb-es/.babelrc.js | 1 + .../api-headless-cms-tasks-ddb-es/LICENSE | 21 ++++ .../api-headless-cms-tasks-ddb-es/README.md | 15 +++ .../jest.setup.js | 2 + .../package.json | 35 ++++++ .../src/index.ts | 64 ++++++++++ .../src/types.ts | 3 + .../tsconfig.build.json | 15 +++ .../tsconfig.json | 22 ++++ .../webiny.config.js | 8 ++ packages/api-headless-cms-tasks/src/index.ts | 13 ++- packages/api-headless-cms/src/types/types.ts | 7 ++ .../BulkActions/ActionDelete.tsx | 2 +- .../ContentEntries/BulkActions/ActionMove.tsx | 12 +- .../BulkActions/ActionPublish.tsx | 4 +- .../BulkActions/ActionUnpublish.tsx | 4 +- .../list/Browser/BulkAction.tsx | 12 +- .../ddb-es/apps/api/graphql/package.json | 2 +- .../ddb-es/apps/api/graphql/src/index.ts | 2 +- .../ddb-es/apps/api/graphql/src/types.ts | 2 +- .../ddb-os/apps/api/graphql/package.json | 2 +- .../ddb-os/apps/api/graphql/src/index.ts | 2 +- .../ddb-os/apps/api/graphql/src/types.ts | 2 +- yarn.lock | 15 +++ 38 files changed, 463 insertions(+), 138 deletions(-) create mode 100644 packages/api-headless-cms-bulk-actions/src/useCases/ListNotPublishedEntries.ts create mode 100644 packages/api-headless-cms-tasks-ddb-es/.babelrc.js create mode 100644 packages/api-headless-cms-tasks-ddb-es/LICENSE create mode 100644 packages/api-headless-cms-tasks-ddb-es/README.md create mode 100644 packages/api-headless-cms-tasks-ddb-es/jest.setup.js create mode 100644 packages/api-headless-cms-tasks-ddb-es/package.json create mode 100644 packages/api-headless-cms-tasks-ddb-es/src/index.ts create mode 100644 packages/api-headless-cms-tasks-ddb-es/src/types.ts create mode 100644 packages/api-headless-cms-tasks-ddb-es/tsconfig.build.json create mode 100644 packages/api-headless-cms-tasks-ddb-es/tsconfig.json create mode 100644 packages/api-headless-cms-tasks-ddb-es/webiny.config.js diff --git a/packages/api-headless-cms-bulk-actions/__tests__/tasks/createBulkAction.test.ts b/packages/api-headless-cms-bulk-actions/__tests__/tasks/createBulkAction.test.ts index 69ef24f03bf..cffdbb6c139 100644 --- a/packages/api-headless-cms-bulk-actions/__tests__/tasks/createBulkAction.test.ts +++ b/packages/api-headless-cms-bulk-actions/__tests__/tasks/createBulkAction.test.ts @@ -2,6 +2,7 @@ import { IntrospectionField, IntrospectionInterfaceType } from "graphql"; import { useGraphQlHandler } from "~tests/context/useGraphQLHandler"; import { createBulkAction } from "~/plugins"; import { createMockModels, createPrivateMockModels } from "~tests/mocks"; +import { createBulkActionEntriesTasks } from "~/tasks"; interface GraphQlType { kind: Uppercase; @@ -36,7 +37,7 @@ const defaultBulkActionsEnumNames = [ describe("createBulkAction", () => { it("should create GraphQL schema with default bulk actions ENUMS", async () => { const { introspect } = useGraphQlHandler({ - plugins: [...createMockModels()] + plugins: [...createMockModels(), createBulkActionEntriesTasks()] }); const [result] = await introspect(); @@ -59,7 +60,7 @@ describe("createBulkAction", () => { it("should NOT create bulk actions ENUMS in case of a private model", async () => { const { introspect } = useGraphQlHandler({ - plugins: [...createPrivateMockModels()] + plugins: [...createPrivateMockModels(), createBulkActionEntriesTasks()] }); const [result] = await introspect(); @@ -79,6 +80,7 @@ describe("createBulkAction", () => { const { introspect } = useGraphQlHandler({ plugins: [ ...createMockModels(), + createBulkActionEntriesTasks(), createBulkAction({ name: "print", dataLoader, diff --git a/packages/api-headless-cms-bulk-actions/src/index.ts b/packages/api-headless-cms-bulk-actions/src/index.ts index 9ea9e9a5b5f..4d3f57c7ac1 100644 --- a/packages/api-headless-cms-bulk-actions/src/index.ts +++ b/packages/api-headless-cms-bulk-actions/src/index.ts @@ -1,13 +1,10 @@ -import { createTasks } from "~/tasks"; import { createHandlers } from "~/handlers"; import { createDefaultGraphQL } from "~/plugins"; export * from "./abstractions"; +export * from "./handlers"; export * from "./useCases"; export * from "./plugins"; +export * from "./tasks"; -export const createHcmsBulkActions = () => [ - createTasks(), - createHandlers(), - createDefaultGraphQL() -]; +export const createHcmsBulkActions = () => [createHandlers(), createDefaultGraphQL()]; diff --git a/packages/api-headless-cms-bulk-actions/src/plugins/createBulkAction.ts b/packages/api-headless-cms-bulk-actions/src/plugins/createBulkAction.ts index 155ef12b5e2..8fbf50c7287 100644 --- a/packages/api-headless-cms-bulk-actions/src/plugins/createBulkAction.ts +++ b/packages/api-headless-cms-bulk-actions/src/plugins/createBulkAction.ts @@ -8,6 +8,7 @@ export interface CreateBulkActionConfig { dataLoader: (context: HcmsBulkActionsContext) => IListEntries; dataProcessor: (context: HcmsBulkActionsContext) => IProcessEntry; modelIds?: string[]; + batchSize?: number; } function toPascalCase(str: string) { @@ -31,7 +32,8 @@ export const createBulkAction = (config: CreateBulkActionConfig) => { createBulkActionTasks({ name, dataLoader: config.dataLoader, - dataProcessor: config.dataProcessor + dataProcessor: config.dataProcessor, + batchSize: config.batchSize }), createBulkActionGraphQL({ name, diff --git a/packages/api-headless-cms-bulk-actions/src/plugins/createBulkActionTasks.ts b/packages/api-headless-cms-bulk-actions/src/plugins/createBulkActionTasks.ts index ce148e88331..70c5f8e9c3a 100644 --- a/packages/api-headless-cms-bulk-actions/src/plugins/createBulkActionTasks.ts +++ b/packages/api-headless-cms-bulk-actions/src/plugins/createBulkActionTasks.ts @@ -1,4 +1,4 @@ -import { createPrivateTaskDefinition } from "@webiny/tasks"; +import { createPrivateTaskDefinition, ITask } from "@webiny/tasks"; import { IListEntries, IProcessEntry } from "~/abstractions"; import { ChildTasksCleanup, @@ -7,6 +7,7 @@ import { ProcessTasksByModel } from "~/useCases/internals"; import { + BulkActionOperationByModelAction, HcmsBulkActionsContext, IBulkActionOperationByModelInput, IBulkActionOperationByModelOutput, @@ -18,17 +19,22 @@ export interface CreateBackgroundTasksConfig { name: string; dataLoader: (context: HcmsBulkActionsContext) => IListEntries; dataProcessor: (context: HcmsBulkActionsContext) => IProcessEntry; + batchSize?: number; } +const BATCH_SIZE = 100; // Default number of entries to fetch in each batch + class BulkActionTasks { private readonly name: string; private readonly dataLoader: (context: HcmsBulkActionsContext) => IListEntries; private readonly dataProcessor: (context: HcmsBulkActionsContext) => IProcessEntry; + private readonly batchSize: number; constructor(config: CreateBackgroundTasksConfig) { this.name = config.name; this.dataLoader = config.dataLoader; this.dataProcessor = config.dataProcessor; + this.batchSize = config.batchSize || BATCH_SIZE; } public createListTaskDefinition() { @@ -40,6 +46,7 @@ class BulkActionTasks { id: this.createListTaskDefinitionName(), title: `Headless CMS: list "${this.name}" entries by model`, maxIterations: 500, + disableDatabaseLogs: true, run: async params => { const { response, input, context } = params; @@ -48,35 +55,49 @@ class BulkActionTasks { return response.error(`Missing "modelId" in the input.`); } - if (input.processing) { - const processTasks = new ProcessTasksByModel( - this.createProcessTaskDefinitionName() - ); - return await processTasks.execute(params); - } + const action = this.getCurrentAction(input); - const createTasks = new CreateTasksByModel( - this.createProcessTaskDefinitionName(), - this.dataLoader(context) - ); - return await createTasks.execute(params); + switch (action) { + case BulkActionOperationByModelAction.PROCESS_SUBTASKS: { + const processTasks = new ProcessTasksByModel( + this.createProcessTaskDefinitionName() + ); + return await processTasks.execute(params); + } + case BulkActionOperationByModelAction.CREATE_SUBTASKS: + case BulkActionOperationByModelAction.CHECK_MORE_SUBTASKS: { + const createTasks = new CreateTasksByModel( + this.createProcessTaskDefinitionName(), + this.dataLoader(context), + this.batchSize + ); + return await createTasks.execute(params); + } + case BulkActionOperationByModelAction.END_TASK: { + return response.done( + `Task done: task "${this.createProcessTaskDefinitionName()}" has been successfully processed for entries from "${ + input.modelId + }" model.` + ); + } + default: + return response.error(`Unknown action: ${action}`); + } } catch (ex) { return response.error(ex.message ?? "Error while executing list task"); } }, onDone: async ({ context, task }) => { - /** - * We want to clean all child tasks and logs, which have no errors. - */ - const childTasksCleanup = new ChildTasksCleanup(); - try { - await childTasksCleanup.execute({ - context, - task - }); - } catch (ex) { - console.error("Error while cleaning list child tasks.", ex); - } + await this.onCreateListTaskDefinitionFinish(context, task, "done"); + }, + onError: async ({ context, task }) => { + await this.onCreateListTaskDefinitionFinish(context, task, "error"); + }, + onAbort: async ({ context, task }) => { + await this.onCreateListTaskDefinitionFinish(context, task, "abort"); + }, + onMaxIterations: async ({ context, task }) => { + await this.onCreateListTaskDefinitionFinish(context, task, "maxIterations"); } }); } @@ -90,6 +111,7 @@ class BulkActionTasks { id: this.createProcessTaskDefinitionName(), title: `Headless CMS: process "${this.name}" entries`, maxIterations: 2, + disableDatabaseLogs: true, run: async params => { const { response, context } = params; @@ -99,20 +121,6 @@ class BulkActionTasks { } catch (ex) { return response.error(ex.message ?? "Error while executing process task"); } - }, - onDone: async ({ context, task }) => { - /** - * We want to clean all child tasks and logs, which have no errors. - */ - const childTasksCleanup = new ChildTasksCleanup(); - try { - await childTasksCleanup.execute({ - context, - task - }); - } catch (ex) { - console.error("Error while cleaning process child tasks.", ex); - } } }); } @@ -124,6 +132,32 @@ class BulkActionTasks { private createProcessTaskDefinitionName() { return `hcmsBulkProcess${this.name}Entries`; } + + private getCurrentAction(input: IBulkActionOperationByModelInput) { + return input.action ?? BulkActionOperationByModelAction.CREATE_SUBTASKS; + } + + private async onCreateListTaskDefinitionFinish( + context: HcmsBulkActionsContext, + task: ITask, + cause: string + ) { + /** + * We want to clean all child tasks and logs, which have no errors. + */ + const childTasksCleanup = new ChildTasksCleanup(); + try { + await childTasksCleanup.execute({ + context, + task + }); + } catch (ex) { + console.error( + `Error while cleaning "${this.createListTaskDefinitionName()} child tasks - ${cause}."`, + ex + ); + } + } } export const createBulkActionTasks = (config: CreateBackgroundTasksConfig) => { diff --git a/packages/api-headless-cms-bulk-actions/src/tasks/createBulkActionEntriesTasks.ts b/packages/api-headless-cms-bulk-actions/src/tasks/createBulkActionEntriesTasks.ts index b42e8c17aad..5629d6b59ff 100644 --- a/packages/api-headless-cms-bulk-actions/src/tasks/createBulkActionEntriesTasks.ts +++ b/packages/api-headless-cms-bulk-actions/src/tasks/createBulkActionEntriesTasks.ts @@ -3,6 +3,7 @@ import { createDeleteEntry, createListDeletedEntries, createListLatestEntries, + createListNotPublishedEntries, createListPublishedEntries, createMoveEntryToFolder, createMoveEntryToTrash, @@ -30,7 +31,7 @@ export const createBulkActionEntriesTasks = () => { }), createBulkAction({ name: "publish", - dataLoader: createListLatestEntries, + dataLoader: createListNotPublishedEntries, dataProcessor: createPublishEntry }), createBulkAction({ diff --git a/packages/api-headless-cms-bulk-actions/src/tasks/index.ts b/packages/api-headless-cms-bulk-actions/src/tasks/index.ts index d42ebc472b7..5db351d7818 100644 --- a/packages/api-headless-cms-bulk-actions/src/tasks/index.ts +++ b/packages/api-headless-cms-bulk-actions/src/tasks/index.ts @@ -1,6 +1,2 @@ -import { createBulkActionEntriesTasks } from "./createBulkActionEntriesTasks"; -import { createEmptyTrashBinsTask } from "./createEmptyTrashBinsTask"; - -export const createTasks = () => { - return [createBulkActionEntriesTasks(), createEmptyTrashBinsTask()]; -}; +export * from "./createBulkActionEntriesTasks"; +export * from "./createEmptyTrashBinsTask"; diff --git a/packages/api-headless-cms-bulk-actions/src/types.ts b/packages/api-headless-cms-bulk-actions/src/types.ts index c02a9e553d2..e31a64d11ea 100644 --- a/packages/api-headless-cms-bulk-actions/src/types.ts +++ b/packages/api-headless-cms-bulk-actions/src/types.ts @@ -37,16 +37,21 @@ export type IBulkActionOperationTaskParams = ITaskRunParams< * Bulk Action Operation By Model */ +export enum BulkActionOperationByModelAction { + CREATE_SUBTASKS = "CREATE_SUBTASKS", + CHECK_MORE_SUBTASKS = "CHECK_MORE_SUBTASKS", + PROCESS_SUBTASKS = "PROCESS_SUBTASKS", + END_TASK = "END_TASK" +} + export interface IBulkActionOperationByModelInput { modelId: string; identity?: SecurityIdentity; where?: Record; search?: string; - data?: Record; after?: string | null; - currentBatch?: number; - processing?: boolean; - totalCount?: number; + data?: Record; + action?: BulkActionOperationByModelAction; } export interface IBulkActionOperationByModelOutput extends ITaskResponseDoneResultOutput { diff --git a/packages/api-headless-cms-bulk-actions/src/useCases/ListNotPublishedEntries.ts b/packages/api-headless-cms-bulk-actions/src/useCases/ListNotPublishedEntries.ts new file mode 100644 index 00000000000..c214d5d03c4 --- /dev/null +++ b/packages/api-headless-cms-bulk-actions/src/useCases/ListNotPublishedEntries.ts @@ -0,0 +1,36 @@ +import { CmsEntryListParams } from "@webiny/api-headless-cms/types"; +import { IListEntries } from "~/abstractions"; +import { HcmsBulkActionsContext } from "~/types"; + +class ListNotPublishedEntries implements IListEntries { + private readonly context: HcmsBulkActionsContext; + + constructor(context: HcmsBulkActionsContext) { + this.context = context; + } + + async execute(modelId: string, params: CmsEntryListParams) { + const model = await this.context.cms.getModel(modelId); + + if (!model) { + throw new Error(`Model with ${modelId} not found!`); + } + + const [entries, meta] = await this.context.cms.listLatestEntries(model, { + ...params, + where: { + ...params.where, + status_not: "published" + } + }); + + return { + entries, + meta + }; + } +} + +export const createListNotPublishedEntries = (context: HcmsBulkActionsContext) => { + return new ListNotPublishedEntries(context); +}; diff --git a/packages/api-headless-cms-bulk-actions/src/useCases/index.ts b/packages/api-headless-cms-bulk-actions/src/useCases/index.ts index f499e2c8cc2..eea90ee47bc 100644 --- a/packages/api-headless-cms-bulk-actions/src/useCases/index.ts +++ b/packages/api-headless-cms-bulk-actions/src/useCases/index.ts @@ -1,5 +1,6 @@ export * from "./DeleteEntry"; export * from "./ListDeletedEntries"; +export * from "./ListNotPublishedEntries"; export * from "./ListLatestEntries"; export * from "./ListPublishedEntries"; export * from "./MoveEntryToFolder"; diff --git a/packages/api-headless-cms-bulk-actions/src/useCases/internals/ChildTaskCleanup/ChildTasksCleanup.ts b/packages/api-headless-cms-bulk-actions/src/useCases/internals/ChildTaskCleanup/ChildTasksCleanup.ts index 3537d7cef64..b833d5ba5b6 100644 --- a/packages/api-headless-cms-bulk-actions/src/useCases/internals/ChildTaskCleanup/ChildTasksCleanup.ts +++ b/packages/api-headless-cms-bulk-actions/src/useCases/internals/ChildTaskCleanup/ChildTasksCleanup.ts @@ -1,5 +1,6 @@ import { ITask, Context, TaskLogItemType } from "@webiny/tasks"; import { IUseCase } from "~/abstractions"; +import { HcmsBulkActionsContext } from "~/types"; export interface IChildTasksCleanupExecuteParams { context: Context; @@ -35,6 +36,13 @@ export class ChildTasksCleanup implements IUseCase { + for (const taskId of taskIds) { + await context.tasks.deleteTask(taskId); } } } diff --git a/packages/api-headless-cms-bulk-actions/src/useCases/internals/CreateTasksByModel/CreateTasksByModel.ts b/packages/api-headless-cms-bulk-actions/src/useCases/internals/CreateTasksByModel/CreateTasksByModel.ts index 199ee98fd0c..2a0243eaa13 100644 --- a/packages/api-headless-cms-bulk-actions/src/useCases/internals/CreateTasksByModel/CreateTasksByModel.ts +++ b/packages/api-headless-cms-bulk-actions/src/useCases/internals/CreateTasksByModel/CreateTasksByModel.ts @@ -2,10 +2,9 @@ import { ITaskResponseResult } from "@webiny/tasks"; import { TaskCache } from "./TaskCache"; import { CmsEntryListParams } from "@webiny/api-headless-cms/types"; import { IListEntries } from "~/abstractions"; -import { IBulkActionOperationByModelTaskParams } from "~/types"; +import { BulkActionOperationByModelAction, IBulkActionOperationByModelTaskParams } from "~/types"; -const BATCH_SIZE = 50; // Number of entries to fetch in each batch -const WAITING_TIME = 5; // Time to wait in seconds before retrying +const MAX_TASK_LIST_LENGTH = 10; /** * The `CreateTasksByModel` class handles the execution of a task to process entries in batches. @@ -13,10 +12,12 @@ const WAITING_TIME = 5; // Time to wait in seconds before retrying export class CreateTasksByModel { private readonly taskCache: TaskCache; private listEntriesGateway: IListEntries; + private readonly batchSize: number; - constructor(taskDefinition: string, listEntriesGateway: IListEntries) { + constructor(taskDefinition: string, listEntriesGateway: IListEntries, batchSize: number) { this.taskCache = new TaskCache(taskDefinition); this.listEntriesGateway = listEntriesGateway; + this.batchSize = batchSize; } async execute(params: IBulkActionOperationByModelTaskParams): Promise { @@ -27,11 +28,9 @@ export class CreateTasksByModel { where: input.where, search: input.search, after: input.after, - limit: BATCH_SIZE + limit: this.batchSize }; - let currentBatch = input.currentBatch || 1; - while (true) { if (isAborted()) { return response.aborted(); @@ -39,8 +38,7 @@ export class CreateTasksByModel { await this.taskCache.triggerTask(context, store.getTask()); return response.continue({ ...input, - ...listEntriesParams, - currentBatch + action: BulkActionOperationByModelAction.PROCESS_SUBTASKS }); } @@ -52,27 +50,35 @@ export class CreateTasksByModel { // End the task if no entries match the query if (meta.totalCount === 0) { - return response.done( - `Task done: no entries found for model "${input.modelId}", skipping task creation.` - ); + return response.continue({ + ...input, + action: BulkActionOperationByModelAction.END_TASK + }); + } + + // Continue processing if we are reached the task list length limit + if (this.taskCache.getTasksLength() === MAX_TASK_LIST_LENGTH) { + await this.taskCache.triggerTask(context, store.getTask()); + return response.continue({ + ...input, + action: BulkActionOperationByModelAction.PROCESS_SUBTASKS + }); } // Continue processing if no entries are returned in the current batch if (entries.length === 0) { await this.taskCache.triggerTask(context, store.getTask()); - return response.continue( - { - ...input, - ...listEntriesParams, - currentBatch, - totalCount: meta.totalCount, - processing: true - }, - { seconds: WAITING_TIME } - ); + return response.continue({ + ...input, + action: BulkActionOperationByModelAction.PROCESS_SUBTASKS + }); } - const ids = entries.map(entry => entry.id); // Extract entry IDs + // Extract entry IDs + const ids: string[] = []; + for (let i = 0; i < entries.length; i++) { + ids.push(entries[i].id); + } if (ids.length > 0) { // Cache the task with the entry IDs @@ -87,23 +93,17 @@ export class CreateTasksByModel { // Continue processing if there are no more entries or pagination is complete if (!meta.hasMoreItems || !meta.cursor) { await this.taskCache.triggerTask(context, store.getTask()); - return response.continue( - { - ...input, - ...listEntriesParams, - currentBatch, - totalCount: meta.totalCount, - processing: true - }, - { seconds: WAITING_TIME } - ); + + return response.continue({ + ...input, + action: BulkActionOperationByModelAction.PROCESS_SUBTASKS + }); } listEntriesParams.after = meta.cursor; - currentBatch++; } } catch (ex) { - throw new Error(ex.message ?? `Error while creating task.`); + return response.error(ex.message ?? `Error while creating task.`); } } } diff --git a/packages/api-headless-cms-bulk-actions/src/useCases/internals/CreateTasksByModel/TaskCache.ts b/packages/api-headless-cms-bulk-actions/src/useCases/internals/CreateTasksByModel/TaskCache.ts index 0325625aad9..4d693656317 100644 --- a/packages/api-headless-cms-bulk-actions/src/useCases/internals/CreateTasksByModel/TaskCache.ts +++ b/packages/api-headless-cms-bulk-actions/src/useCases/internals/CreateTasksByModel/TaskCache.ts @@ -49,6 +49,14 @@ export class TaskCache { this.clearTasks(); } + /** + * Retrieves the cached tasks length. + * @returns number + */ + getTasksLength() { + return this.getTasks().length; + } + /** * Retrieves the cached tasks. * @returns {TTask[]} The list of cached tasks. diff --git a/packages/api-headless-cms-bulk-actions/src/useCases/internals/ProcessTask/ProcessTask.ts b/packages/api-headless-cms-bulk-actions/src/useCases/internals/ProcessTask/ProcessTask.ts index f15dce4e63c..93c3058b1fb 100644 --- a/packages/api-headless-cms-bulk-actions/src/useCases/internals/ProcessTask/ProcessTask.ts +++ b/packages/api-headless-cms-bulk-actions/src/useCases/internals/ProcessTask/ProcessTask.ts @@ -18,7 +18,7 @@ export class ProcessTask { } async execute(params: IBulkActionOperationTaskParams) { - const { input, response, isAborted, isCloseToTimeout, context, store } = params; + const { input, response, isAborted, isCloseToTimeout, context } = params; try { if (isAborted()) { @@ -48,30 +48,20 @@ export class ProcessTask { ); } + // Set the security identity in the context. + context.security.setIdentity(input.identity); + // Process each ID in the input. for (const id of input.ids) { try { - // Set the security identity in the context. - context.security.setIdentity(input.identity); // Execute the gateway operation for the current ID. await this.gateway.execute(model, id, input.data); // Add the ID to the list of successfully processed entries. this.result.addDone(id); } catch (ex) { - // Handle any errors that occur during processing of the current ID. - const message = ex.message || `Failed to process entry with id "${id}".`; - try { - console.error(message); - await store.addErrorLog({ - message, - error: ex - }); - } catch { - console.error(`Failed to add error log: "${message}"`); - } finally { - // Add the ID to the list of failed entries. - this.result.addFailed(id); - } + console.error(ex.message || `Failed to process entry with id "${id}".`); + // Add the ID to the list of failed entries. + this.result.addFailed(id); } } diff --git a/packages/api-headless-cms-bulk-actions/src/useCases/internals/ProcessTasksByModel/ProcessTasksByModel.ts b/packages/api-headless-cms-bulk-actions/src/useCases/internals/ProcessTasksByModel/ProcessTasksByModel.ts index 6be66d57042..e35cc6867a2 100644 --- a/packages/api-headless-cms-bulk-actions/src/useCases/internals/ProcessTasksByModel/ProcessTasksByModel.ts +++ b/packages/api-headless-cms-bulk-actions/src/useCases/internals/ProcessTasksByModel/ProcessTasksByModel.ts @@ -1,7 +1,5 @@ import { TaskDataStatus } from "@webiny/tasks"; -import { IBulkActionOperationByModelTaskParams } from "~/types"; - -const WAITING_TIME = 10; +import { BulkActionOperationByModelAction, IBulkActionOperationByModelTaskParams } from "~/types"; /** * The `ProcessTasksByModel` class is responsible for processing tasks for a specific model. @@ -23,11 +21,12 @@ export class ProcessTasksByModel { return response.aborted(); } else if (isCloseToTimeout()) { return response.continue({ - ...input + ...input, + action: BulkActionOperationByModelAction.PROCESS_SUBTASKS }); } - const result = await context.tasks.listTasks({ + const { items } = await context.tasks.listTasks({ where: { parentId: store.getTask().id, definitionId: this.taskDefinition, @@ -37,20 +36,22 @@ export class ProcessTasksByModel { }); // If there are running or pending tasks, continue with a wait. - if (result.items.length > 0) { + if (items.length > 0) { return response.continue( { - ...input + ...input, + action: BulkActionOperationByModelAction.PROCESS_SUBTASKS }, { - seconds: WAITING_TIME + seconds: 120 } ); } - return response.done( - `Task done: task "${this.taskDefinition}" has been successfully processed for entries from "${input.modelId}" model.` - ); + return response.continue({ + ...input, + action: BulkActionOperationByModelAction.CHECK_MORE_SUBTASKS + }); } catch (ex) { return response.error( ex.message ?? `Error while processing task "${this.taskDefinition}"` diff --git a/packages/api-headless-cms-tasks-ddb-es/.babelrc.js b/packages/api-headless-cms-tasks-ddb-es/.babelrc.js new file mode 100644 index 00000000000..9da7674cb52 --- /dev/null +++ b/packages/api-headless-cms-tasks-ddb-es/.babelrc.js @@ -0,0 +1 @@ +module.exports = require("@webiny/project-utils").createBabelConfigForNode({ path: __dirname }); diff --git a/packages/api-headless-cms-tasks-ddb-es/LICENSE b/packages/api-headless-cms-tasks-ddb-es/LICENSE new file mode 100644 index 00000000000..f772d04d4db --- /dev/null +++ b/packages/api-headless-cms-tasks-ddb-es/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) Webiny + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/packages/api-headless-cms-tasks-ddb-es/README.md b/packages/api-headless-cms-tasks-ddb-es/README.md new file mode 100644 index 00000000000..2a2cc567c98 --- /dev/null +++ b/packages/api-headless-cms-tasks-ddb-es/README.md @@ -0,0 +1,15 @@ +# @webiny/api-headless-cms-tasks-ddb-es +[![](https://img.shields.io/npm/dw/@webiny/api-headless-cms-tasks-ddb-es.svg)](https://www.npmjs.com/package/@webiny/api-headless-cms-tasks-ddb-es) +[![](https://img.shields.io/npm/v/@webiny/api-headless-cms-tasks-ddb-es.svg)](https://www.npmjs.com/package/@webiny/api-headless-cms-tasks-ddb-es) +[![code style: prettier](https://img.shields.io/badge/code_style-prettier-ff69b4.svg?style=flat-square)](https://github.com/prettier/prettier) +[![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg?style=flat-square)](http://makeapullrequest.com) + +## Install +``` +npm install --save @webiny/api-headless-cms-tasks-ddb-es +``` + +Or if you prefer yarn: +``` +yarn add @webiny/api-headless-cms-tasks-ddb-es +``` diff --git a/packages/api-headless-cms-tasks-ddb-es/jest.setup.js b/packages/api-headless-cms-tasks-ddb-es/jest.setup.js new file mode 100644 index 00000000000..190b62b5397 --- /dev/null +++ b/packages/api-headless-cms-tasks-ddb-es/jest.setup.js @@ -0,0 +1,2 @@ +const base = require("../../jest.config.base"); +module.exports = base({ name: "api-headless-cms-tasks", path: __dirname }, []); diff --git a/packages/api-headless-cms-tasks-ddb-es/package.json b/packages/api-headless-cms-tasks-ddb-es/package.json new file mode 100644 index 00000000000..9b88067b8d1 --- /dev/null +++ b/packages/api-headless-cms-tasks-ddb-es/package.json @@ -0,0 +1,35 @@ +{ + "name": "@webiny/api-headless-cms-tasks-ddb-es", + "version": "0.0.0", + "main": "index.js", + "description": "Background tasks for Webiny Headless CMS - DDB+ES", + "keywords": [ + "api-headless-cms-tasks:base" + ], + "repository": { + "type": "git", + "url": "https://github.com/webiny/webiny-js.git", + "directory": "packages/api-headless-cms-tasks-ddb-es" + }, + "license": "MIT", + "dependencies": { + "@webiny/api-headless-cms-bulk-actions": "0.0.0", + "@webiny/api-headless-cms-import-export": "0.0.0" + }, + "devDependencies": { + "@babel/cli": "^7.23.9", + "@babel/core": "^7.24.0", + "@webiny/cli": "0.0.0", + "@webiny/project-utils": "0.0.0", + "ttypescript": "^1.5.12", + "typescript": "4.9.5" + }, + "scripts": { + "build": "yarn webiny run build", + "watch": "yarn webiny run watch" + }, + "publishConfig": { + "access": "public", + "directory": "dist" + } +} diff --git a/packages/api-headless-cms-tasks-ddb-es/src/index.ts b/packages/api-headless-cms-tasks-ddb-es/src/index.ts new file mode 100644 index 00000000000..46bae73f419 --- /dev/null +++ b/packages/api-headless-cms-tasks-ddb-es/src/index.ts @@ -0,0 +1,64 @@ +import { + createBulkAction, + createDeleteEntry, + createEmptyTrashBinsTask, + createHcmsBulkActions, + createListDeletedEntries, + createListLatestEntries, + createListNotPublishedEntries, + createListPublishedEntries, + createMoveEntryToFolder, + createMoveEntryToTrash, + createPublishEntry, + createRestoreEntryFromTrash, + createUnpublishEntry +} from "@webiny/api-headless-cms-bulk-actions"; +import { createHeadlessCmsImportExport } from "@webiny/api-headless-cms-import-export"; + +const createEsBulkActionEntriesTasks = () => { + return [ + createBulkAction({ + name: "delete", + dataLoader: createListDeletedEntries, + dataProcessor: createDeleteEntry, + batchSize: 1000 + }), + createBulkAction({ + name: "moveToFolder", + dataLoader: createListLatestEntries, + dataProcessor: createMoveEntryToFolder, + batchSize: 1000 + }), + createBulkAction({ + name: "moveToTrash", + dataLoader: createListLatestEntries, + dataProcessor: createMoveEntryToTrash, + batchSize: 1000 + }), + createBulkAction({ + name: "publish", + dataLoader: createListNotPublishedEntries, + dataProcessor: createPublishEntry, + batchSize: 1000 + }), + createBulkAction({ + name: "unpublish", + dataLoader: createListPublishedEntries, + dataProcessor: createUnpublishEntry, + batchSize: 1000 + }), + createBulkAction({ + name: "restore", + dataLoader: createListDeletedEntries, + dataProcessor: createRestoreEntryFromTrash, + batchSize: 1000 + }) + ]; +}; + +export const createHcmsTasks = () => [ + createHcmsBulkActions(), + createEsBulkActionEntriesTasks(), + createEmptyTrashBinsTask(), + createHeadlessCmsImportExport() +]; diff --git a/packages/api-headless-cms-tasks-ddb-es/src/types.ts b/packages/api-headless-cms-tasks-ddb-es/src/types.ts new file mode 100644 index 00000000000..97a4b06e49f --- /dev/null +++ b/packages/api-headless-cms-tasks-ddb-es/src/types.ts @@ -0,0 +1,3 @@ +import { HcmsBulkActionsContext } from "@webiny/api-headless-cms-bulk-actions/types"; + +export type HcmsTasksContext = HcmsBulkActionsContext; diff --git a/packages/api-headless-cms-tasks-ddb-es/tsconfig.build.json b/packages/api-headless-cms-tasks-ddb-es/tsconfig.build.json new file mode 100644 index 00000000000..4ecdb594134 --- /dev/null +++ b/packages/api-headless-cms-tasks-ddb-es/tsconfig.build.json @@ -0,0 +1,15 @@ +{ + "extends": "../../tsconfig.build.json", + "include": ["src"], + "references": [ + { "path": "../api-headless-cms-bulk-actions/tsconfig.build.json" }, + { "path": "../api-headless-cms-import-export/tsconfig.build.json" } + ], + "compilerOptions": { + "rootDir": "./src", + "outDir": "./dist", + "declarationDir": "./dist", + "paths": { "~/*": ["./src/*"], "~tests/*": ["./__tests__/*"] }, + "baseUrl": "." + } +} diff --git a/packages/api-headless-cms-tasks-ddb-es/tsconfig.json b/packages/api-headless-cms-tasks-ddb-es/tsconfig.json new file mode 100644 index 00000000000..bc71a530f8b --- /dev/null +++ b/packages/api-headless-cms-tasks-ddb-es/tsconfig.json @@ -0,0 +1,22 @@ +{ + "extends": "../../tsconfig.json", + "include": ["src", "__tests__"], + "references": [ + { "path": "../api-headless-cms-bulk-actions" }, + { "path": "../api-headless-cms-import-export" } + ], + "compilerOptions": { + "rootDirs": ["./src", "./__tests__"], + "outDir": "./dist", + "declarationDir": "./dist", + "paths": { + "~/*": ["./src/*"], + "~tests/*": ["./__tests__/*"], + "@webiny/api-headless-cms-bulk-actions/*": ["../api-headless-cms-bulk-actions/src/*"], + "@webiny/api-headless-cms-bulk-actions": ["../api-headless-cms-bulk-actions/src"], + "@webiny/api-headless-cms-import-export/*": ["../api-headless-cms-import-export/src/*"], + "@webiny/api-headless-cms-import-export": ["../api-headless-cms-import-export/src"] + }, + "baseUrl": "." + } +} diff --git a/packages/api-headless-cms-tasks-ddb-es/webiny.config.js b/packages/api-headless-cms-tasks-ddb-es/webiny.config.js new file mode 100644 index 00000000000..6dff86766c9 --- /dev/null +++ b/packages/api-headless-cms-tasks-ddb-es/webiny.config.js @@ -0,0 +1,8 @@ +const { createWatchPackage, createBuildPackage } = require("@webiny/project-utils"); + +module.exports = { + commands: { + build: createBuildPackage({ cwd: __dirname }), + watch: createWatchPackage({ cwd: __dirname }) + } +}; diff --git a/packages/api-headless-cms-tasks/src/index.ts b/packages/api-headless-cms-tasks/src/index.ts index 13dc12ab4c1..d7ed17a4a03 100644 --- a/packages/api-headless-cms-tasks/src/index.ts +++ b/packages/api-headless-cms-tasks/src/index.ts @@ -1,4 +1,13 @@ -import { createHcmsBulkActions } from "@webiny/api-headless-cms-bulk-actions"; +import { + createBulkActionEntriesTasks, + createEmptyTrashBinsTask, + createHcmsBulkActions +} from "@webiny/api-headless-cms-bulk-actions"; import { createHeadlessCmsImportExport } from "@webiny/api-headless-cms-import-export"; -export const createHcmsTasks = () => [createHcmsBulkActions(), createHeadlessCmsImportExport()]; +export const createHcmsTasks = () => [ + createHcmsBulkActions(), + createBulkActionEntriesTasks(), + createEmptyTrashBinsTask(), + createHeadlessCmsImportExport() +]; diff --git a/packages/api-headless-cms/src/types/types.ts b/packages/api-headless-cms/src/types/types.ts index 9911dfe4d5a..5b15e6b46c1 100644 --- a/packages/api-headless-cms/src/types/types.ts +++ b/packages/api-headless-cms/src/types/types.ts @@ -947,6 +947,13 @@ export interface CmsEntryListWhere { entryId_not?: string; entryId_in?: string[]; entryId_not_in?: string[]; + /** + * Status of the entry. + */ + status?: CmsEntryStatus; + status_not?: CmsEntryStatus; + status_in?: CmsEntryStatus[]; + status_not_in?: CmsEntryStatus[]; /** * Revision-level meta fields. 👇 diff --git a/packages/app-headless-cms/src/admin/components/ContentEntries/BulkActions/ActionDelete.tsx b/packages/app-headless-cms/src/admin/components/ContentEntries/BulkActions/ActionDelete.tsx index 7571609cb5a..8187c673b9d 100644 --- a/packages/app-headless-cms/src/admin/components/ContentEntries/BulkActions/ActionDelete.tsx +++ b/packages/app-headless-cms/src/admin/components/ContentEntries/BulkActions/ActionDelete.tsx @@ -28,7 +28,7 @@ export const ActionDelete = observer(() => { loadingLabel: `Processing ${entriesLabel}`, execute: async () => { if (worker.isSelectedAll) { - await worker.processInBulk("MoveToTrash"); + await worker.processInBulk({ action: "MoveToTrash" }); worker.resetItems(); showSnackbar( "All entries will be moved to trash. This process will be carried out in the background and may take some time. You can safely navigate away from this page while the process is running.", diff --git a/packages/app-headless-cms/src/admin/components/ContentEntries/BulkActions/ActionMove.tsx b/packages/app-headless-cms/src/admin/components/ContentEntries/BulkActions/ActionMove.tsx index f864fd10ac1..2ef05a1d7c7 100644 --- a/packages/app-headless-cms/src/admin/components/ContentEntries/BulkActions/ActionMove.tsx +++ b/packages/app-headless-cms/src/admin/components/ContentEntries/BulkActions/ActionMove.tsx @@ -29,8 +29,16 @@ export const ActionMove = observer(() => { loadingLabel: `Processing ${entriesLabel}`, execute: async () => { if (worker.isSelectedAll) { - await worker.processInBulk("MoveToFolder", { - folderId: folder.id + await worker.processInBulk({ + action: "MoveToFolder", + where: { + wbyAco_location: { + folderId_not: folder.id + } + }, + data: { + folderId: folder.id + } }); worker.resetItems(); showSnackbar( diff --git a/packages/app-headless-cms/src/admin/components/ContentEntries/BulkActions/ActionPublish.tsx b/packages/app-headless-cms/src/admin/components/ContentEntries/BulkActions/ActionPublish.tsx index 566d06b3cbf..529e063507e 100644 --- a/packages/app-headless-cms/src/admin/components/ContentEntries/BulkActions/ActionPublish.tsx +++ b/packages/app-headless-cms/src/admin/components/ContentEntries/BulkActions/ActionPublish.tsx @@ -28,7 +28,9 @@ export const ActionPublish = observer(() => { loadingLabel: `Processing ${entriesLabel}`, execute: async () => { if (worker.isSelectedAll) { - await worker.processInBulk("Publish"); + await worker.processInBulk({ + action: "Publish" + }); worker.resetItems(); showSnackbar( "All entries will be published. This process will be carried out in the background and may take some time. You can safely navigate away from this page while the process is running.", diff --git a/packages/app-headless-cms/src/admin/components/ContentEntries/BulkActions/ActionUnpublish.tsx b/packages/app-headless-cms/src/admin/components/ContentEntries/BulkActions/ActionUnpublish.tsx index 8def71f4105..d6d03887e35 100644 --- a/packages/app-headless-cms/src/admin/components/ContentEntries/BulkActions/ActionUnpublish.tsx +++ b/packages/app-headless-cms/src/admin/components/ContentEntries/BulkActions/ActionUnpublish.tsx @@ -28,7 +28,9 @@ export const ActionUnpublish = observer(() => { loadingLabel: `Processing ${entriesLabel}`, execute: async () => { if (worker.isSelectedAll) { - await worker.processInBulk("Unpublish"); + await worker.processInBulk({ + action: "Unpublish" + }); worker.resetItems(); showSnackbar( "All entries will be unpublished. This process will be carried out in the background and may take some time. You can safely navigate away from this page while the process is running.", diff --git a/packages/app-headless-cms/src/admin/config/contentEntries/list/Browser/BulkAction.tsx b/packages/app-headless-cms/src/admin/config/contentEntries/list/Browser/BulkAction.tsx index 6301866769a..cc81c310321 100644 --- a/packages/app-headless-cms/src/admin/config/contentEntries/list/Browser/BulkAction.tsx +++ b/packages/app-headless-cms/src/admin/config/contentEntries/list/Browser/BulkAction.tsx @@ -9,6 +9,7 @@ import { import { Property, useIdGenerator } from "@webiny/react-properties"; import { useCms, useContentEntriesList, useModel } from "~/admin/hooks"; import { CmsContentEntry } from "@webiny/app-headless-cms-common/types"; +import merge from "lodash/merge"; export interface BulkActionConfig { name: string; @@ -24,6 +25,12 @@ export interface BulkActionProps { element?: React.ReactElement; } +export interface ProcessInBulkParams { + action: string; + where?: Record; + data?: Record; +} + export const BaseBulkAction = makeDecoratable( "BulkAction", ({ @@ -91,8 +98,9 @@ const useWorker = () => { }: CallbackParams) => Promise, chunkSize?: number ) => worker.processInSeries(callback, chunkSize), - processInBulk: async (action: string, data?: Record) => { - await bulkAction({ model, action, where: getWhere(), data }); + processInBulk: async ({ action, where: initialWhere, data }: ProcessInBulkParams) => { + const where = merge(getWhere(), initialWhere); + await bulkAction({ model, action, where, data }); }, resetItems: resetItems, results: worker.results, diff --git a/packages/cwp-template-aws/template/ddb-es/apps/api/graphql/package.json b/packages/cwp-template-aws/template/ddb-es/apps/api/graphql/package.json index 2618162f417..9293936e261 100644 --- a/packages/cwp-template-aws/template/ddb-es/apps/api/graphql/package.json +++ b/packages/cwp-template-aws/template/ddb-es/apps/api/graphql/package.json @@ -22,7 +22,7 @@ "@webiny/api-headless-cms": "latest", "@webiny/api-headless-cms-aco": "latest", "@webiny/api-headless-cms-ddb-es": "latest", - "@webiny/api-headless-cms-tasks": "latest", + "@webiny/api-headless-cms-tasks-ddb-es": "latest", "@webiny/api-record-locking": "latest", "@webiny/api-page-builder": "latest", "@webiny/api-page-builder-aco": "latest", diff --git a/packages/cwp-template-aws/template/ddb-es/apps/api/graphql/src/index.ts b/packages/cwp-template-aws/template/ddb-es/apps/api/graphql/src/index.ts index 2056a49a986..93f67110ae8 100644 --- a/packages/cwp-template-aws/template/ddb-es/apps/api/graphql/src/index.ts +++ b/packages/cwp-template-aws/template/ddb-es/apps/api/graphql/src/index.ts @@ -28,7 +28,7 @@ import { createFormBuilder } from "@webiny/api-form-builder"; import { createFormBuilderStorageOperations } from "@webiny/api-form-builder-so-ddb-es"; import { createHeadlessCmsContext, createHeadlessCmsGraphQL } from "@webiny/api-headless-cms"; import { createStorageOperations as createHeadlessCmsStorageOperations } from "@webiny/api-headless-cms-ddb-es"; -import { createHcmsTasks } from "@webiny/api-headless-cms-tasks"; +import { createHcmsTasks } from "@webiny/api-headless-cms-tasks-ddb-es"; import { createAco } from "@webiny/api-aco"; import { createAcoPageBuilderContext } from "@webiny/api-page-builder-aco"; import { createAcoHcmsContext } from "@webiny/api-headless-cms-aco"; diff --git a/packages/cwp-template-aws/template/ddb-es/apps/api/graphql/src/types.ts b/packages/cwp-template-aws/template/ddb-es/apps/api/graphql/src/types.ts index 535a088f930..43ecee27cb5 100644 --- a/packages/cwp-template-aws/template/ddb-es/apps/api/graphql/src/types.ts +++ b/packages/cwp-template-aws/template/ddb-es/apps/api/graphql/src/types.ts @@ -12,7 +12,7 @@ import { CmsContext } from "@webiny/api-headless-cms/types"; import { AcoContext } from "@webiny/api-aco/types"; import { PbAcoContext } from "@webiny/api-page-builder-aco/types"; import { HcmsAcoContext } from "@webiny/api-headless-cms-aco/types"; -import { HcmsTasksContext } from "@webiny/api-headless-cms-tasks/types"; +import { HcmsTasksContext } from "@webiny/api-headless-cms-tasks-ddb-es/types"; // When working with the `context` object (for example while defining a new GraphQL resolver function), // you can import this interface and assign it to it. This will give you full autocomplete functionality diff --git a/packages/cwp-template-aws/template/ddb-os/apps/api/graphql/package.json b/packages/cwp-template-aws/template/ddb-os/apps/api/graphql/package.json index 4981a0191f7..3a8dc68fb07 100644 --- a/packages/cwp-template-aws/template/ddb-os/apps/api/graphql/package.json +++ b/packages/cwp-template-aws/template/ddb-os/apps/api/graphql/package.json @@ -22,7 +22,7 @@ "@webiny/api-headless-cms": "latest", "@webiny/api-headless-cms-aco": "latest", "@webiny/api-headless-cms-ddb-es": "latest", - "@webiny/api-headless-cms-tasks": "latest", + "@webiny/api-headless-cms-tasks-ddb-es": "latest", "@webiny/api-record-locking": "latest", "@webiny/api-page-builder": "latest", "@webiny/api-page-builder-aco": "latest", diff --git a/packages/cwp-template-aws/template/ddb-os/apps/api/graphql/src/index.ts b/packages/cwp-template-aws/template/ddb-os/apps/api/graphql/src/index.ts index d1199759a1e..d01c107bdac 100644 --- a/packages/cwp-template-aws/template/ddb-os/apps/api/graphql/src/index.ts +++ b/packages/cwp-template-aws/template/ddb-os/apps/api/graphql/src/index.ts @@ -28,7 +28,7 @@ import { createFormBuilder } from "@webiny/api-form-builder"; import { createFormBuilderStorageOperations } from "@webiny/api-form-builder-so-ddb-es"; import { createHeadlessCmsContext, createHeadlessCmsGraphQL } from "@webiny/api-headless-cms"; import { createStorageOperations as createHeadlessCmsStorageOperations } from "@webiny/api-headless-cms-ddb-es"; -import { createHcmsTasks } from "@webiny/api-headless-cms-tasks"; +import { createHcmsTasks } from "@webiny/api-headless-cms-tasks-ddb-es"; import { createAco } from "@webiny/api-aco"; import { createAcoPageBuilderContext } from "@webiny/api-page-builder-aco"; import { createAcoHcmsContext } from "@webiny/api-headless-cms-aco"; diff --git a/packages/cwp-template-aws/template/ddb-os/apps/api/graphql/src/types.ts b/packages/cwp-template-aws/template/ddb-os/apps/api/graphql/src/types.ts index 535a088f930..43ecee27cb5 100644 --- a/packages/cwp-template-aws/template/ddb-os/apps/api/graphql/src/types.ts +++ b/packages/cwp-template-aws/template/ddb-os/apps/api/graphql/src/types.ts @@ -12,7 +12,7 @@ import { CmsContext } from "@webiny/api-headless-cms/types"; import { AcoContext } from "@webiny/api-aco/types"; import { PbAcoContext } from "@webiny/api-page-builder-aco/types"; import { HcmsAcoContext } from "@webiny/api-headless-cms-aco/types"; -import { HcmsTasksContext } from "@webiny/api-headless-cms-tasks/types"; +import { HcmsTasksContext } from "@webiny/api-headless-cms-tasks-ddb-es/types"; // When working with the `context` object (for example while defining a new GraphQL resolver function), // you can import this interface and assign it to it. This will give you full autocomplete functionality diff --git a/yarn.lock b/yarn.lock index 5bfff7f2472..c713212371a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -14433,6 +14433,21 @@ __metadata: languageName: unknown linkType: soft +"@webiny/api-headless-cms-tasks-ddb-es@workspace:packages/api-headless-cms-tasks-ddb-es": + version: 0.0.0-use.local + resolution: "@webiny/api-headless-cms-tasks-ddb-es@workspace:packages/api-headless-cms-tasks-ddb-es" + dependencies: + "@babel/cli": ^7.23.9 + "@babel/core": ^7.24.0 + "@webiny/api-headless-cms-bulk-actions": 0.0.0 + "@webiny/api-headless-cms-import-export": 0.0.0 + "@webiny/cli": 0.0.0 + "@webiny/project-utils": 0.0.0 + ttypescript: ^1.5.12 + typescript: 4.9.5 + languageName: unknown + linkType: soft + "@webiny/api-headless-cms-tasks@0.0.0, @webiny/api-headless-cms-tasks@workspace:packages/api-headless-cms-tasks": version: 0.0.0-use.local resolution: "@webiny/api-headless-cms-tasks@workspace:packages/api-headless-cms-tasks" From a010ea47427f079d8a3b3b8d25ce70dc3b4a99df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20Zori=C4=87?= Date: Mon, 30 Sep 2024 14:26:08 +0200 Subject: [PATCH 70/70] fix: wrong types --- packages/api-headless-cms/src/types/types.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/api-headless-cms/src/types/types.ts b/packages/api-headless-cms/src/types/types.ts index 219ff3029ff..1ee778ca5b4 100644 --- a/packages/api-headless-cms/src/types/types.ts +++ b/packages/api-headless-cms/src/types/types.ts @@ -893,7 +893,9 @@ export interface CmsModelContext { /** * A model manager for a model which has a single entry. */ - getSingletonEntryManager(model: CmsModel | string): Promise>; + getSingletonEntryManager( + model: CmsModel | string + ): Promise>; /** * Get all content model managers mapped by modelId. * @see CmsModelManager