From be9a56de8c3e2a7938929d754a415c6784b5b74f Mon Sep 17 00:00:00 2001 From: Jeremy Cloarec Date: Thu, 4 Apr 2024 14:16:08 +0200 Subject: [PATCH] [backend] WIP migration file --- .../opencti-graphql/src/database/engine.js | 3 +- ...5768-convert_linked-to_to_related-toOld.js | 50 ----------- ...225747207-convert_linked-to_related-to.js} | 82 +++++++++++++------ 3 files changed, 59 insertions(+), 76 deletions(-) delete mode 100644 opencti-platform/opencti-graphql/src/migrations/1711029985768-convert_linked-to_to_related-toOld.js rename opencti-platform/opencti-graphql/src/migrations/{1711029986768-convert_linked-to_related-to.js => 1712225747207-convert_linked-to_related-to.js} (51%) diff --git a/opencti-platform/opencti-graphql/src/database/engine.js b/opencti-platform/opencti-graphql/src/database/engine.js index c1064b1f353df..754b427ef81fa 100644 --- a/opencti-platform/opencti-graphql/src/database/engine.js +++ b/opencti-platform/opencti-graphql/src/database/engine.js @@ -10,7 +10,6 @@ import { buildPagination, cursorToOffset, ES_INDEX_PREFIX, - INDEX_FILES, INDEX_INTERNAL_OBJECTS, inferIndexFromConceptType, isEmptyField, @@ -324,7 +323,7 @@ const elOperationForMigration = (operation) => { logApp.info(`${message} > started`); // Execute the update by query in async mode const queryAsync = await operation({ - index, + ...(index ?? {}), refresh: true, wait_for_completion: false, body diff --git a/opencti-platform/opencti-graphql/src/migrations/1711029985768-convert_linked-to_to_related-toOld.js b/opencti-platform/opencti-graphql/src/migrations/1711029985768-convert_linked-to_to_related-toOld.js deleted file mode 100644 index 96ea5ca6d3809..0000000000000 --- a/opencti-platform/opencti-graphql/src/migrations/1711029985768-convert_linked-to_to_related-toOld.js +++ /dev/null @@ -1,50 +0,0 @@ -import { logApp } from '../config/conf'; -import { executionContext, SYSTEM_USER } from '../utils/access'; -import { createRelationRaw, deleteElementById } from '../database/middleware'; -import { elList } from '../database/engine'; -import { READ_RELATIONSHIPS_INDICES } from '../database/utils'; -import { RELATION_RELATED_TO } from '../schema/stixCoreRelationship'; -import { stixObjectOrRelationshipAddRefRelations } from '../domain/stixObjectOrStixRelationship'; -import { ABSTRACT_STIX_DOMAIN_OBJECT, ENTITY_TYPE_CONTAINER } from '../schema/general'; -import { listEntitiesThroughRelationsPaginated } from '../database/middleware-loader'; -import { RELATION_OBJECT } from '../schema/stixRefRelationship'; - -export const up = async (next) => { - const context = executionContext('migration'); - logApp.info('[MIGRATION] Update all linked-to refs to related-to rels'); - const callback = async (linkedToRefs) => { - // For each linked-to ref, we create a related-to rel, add the containers from the linked-to ref to the related-to rel, and we delete the linked-to ref - for (let linkedRefIndex = 0; linkedRefIndex < linkedToRefs.length; linkedRefIndex += 1) { - const linkedToRef = linkedToRefs[linkedRefIndex]; - // Create related-to - const createdRelatedToRel = await createRelationRaw(context, SYSTEM_USER, { - fromId: linkedToRef.fromId, - relationship_type: RELATION_RELATED_TO, - toId: linkedToRef.toId - }); - // Get all linked-to ref containers, add new related-to rel to those containers - // We first check if linkedToRef has any rel_object.internal_id before making an unecessary containers list query - if (linkedToRef['rel_object.internal_id'] !== undefined) { - const linkedToContainers = await listEntitiesThroughRelationsPaginated(context, SYSTEM_USER, linkedToRef.id, RELATION_OBJECT, [ENTITY_TYPE_CONTAINER], true, opts); - - for (let containerIndex = 0; containerIndex < linkedToContainers.edges?.length; containerIndex += 1) { - const container = linkedToContainers.edges[containerIndex].node; - const addToContainerInput = { relationship_type: 'object', toIds: [createdRelatedToRel.element.id], }; - await stixObjectOrRelationshipAddRefRelations(context, SYSTEM_USER, container.id, addToContainerInput, ABSTRACT_STIX_DOMAIN_OBJECT); - } - } - // Delete linked-to ref - await deleteElementById(context, SYSTEM_USER, linkedToRef.id, 'x_opencti_linked-to'); - } - }; - const opts = { types: ['x_opencti_linked-to'], callback }; - await elList(context, SYSTEM_USER, READ_RELATIONSHIPS_INDICES, opts); - - logApp.info('[MIGRATION] Update all linked-to refs to related-to rels finished'); - - next(); -}; - -export const down = async (next) => { - next(); -}; diff --git a/opencti-platform/opencti-graphql/src/migrations/1711029986768-convert_linked-to_related-to.js b/opencti-platform/opencti-graphql/src/migrations/1712225747207-convert_linked-to_related-to.js similarity index 51% rename from opencti-platform/opencti-graphql/src/migrations/1711029986768-convert_linked-to_related-to.js rename to opencti-platform/opencti-graphql/src/migrations/1712225747207-convert_linked-to_related-to.js index c84c5d3beb2d5..cdbf13e86d3f4 100644 --- a/opencti-platform/opencti-graphql/src/migrations/1711029986768-convert_linked-to_related-to.js +++ b/opencti-platform/opencti-graphql/src/migrations/1712225747207-convert_linked-to_related-to.js @@ -1,6 +1,6 @@ import { logApp } from '../config/conf'; -import { elReindexByQueryForMigration, elUpdateByQueryForMigration } from '../database/engine'; -import { READ_INDEX_STIX_CORE_RELATIONSHIPS, READ_INDEX_STIX_META_RELATIONSHIPS, READ_RELATIONSHIPS_INDICES } from '../database/utils'; +import { elDeleteByQueryForMigration, elReindexByQueryForMigration, elUpdateByQueryForMigration } from '../database/engine'; +import { READ_DATA_INDICES, READ_INDEX_STIX_CORE_RELATIONSHIPS, READ_INDEX_STIX_META_RELATIONSHIPS, READ_RELATIONSHIPS_INDICES } from '../database/utils'; import { RELATION_RELATED_TO } from '../schema/stixCoreRelationship'; export const up = async (next) => { @@ -8,7 +8,7 @@ export const up = async (next) => { const linkedToType = 'x_opencti_linked-to'; // First, we reindex all linked-to meta refs to core rel index, updating all linked-to refs to related-to rel in the process - const linkedToToRelatedToSript = ` + const reindexLinkedToToRelatedToSource = ` ctx._source.entity_type = params.relToType; ctx._source.relationship_type = params.relToType; ctx._source.parentTypes = params.relToParentTypes; @@ -19,24 +19,22 @@ export const up = async (next) => { const relToParentTypes = ['basic-relationship', 'stix-relationship', 'stix-core-relationship']; const reindexLinkedToToRelatedToQuery = { - body: { - source: { - index: READ_INDEX_STIX_META_RELATIONSHIPS, - query: { - bool: { - must: [ - { term: { 'entity_type.keyword': { value: linkedToType } } } - ] - } + source: { + index: READ_INDEX_STIX_META_RELATIONSHIPS, + query: { + bool: { + must: [ + { term: { 'entity_type.keyword': { value: linkedToType } } } + ] } - }, - dest: { - index: READ_INDEX_STIX_CORE_RELATIONSHIPS - }, - script: { - source: linkedToToRelatedToSript, - params: { linkedToType, relToType: RELATION_RELATED_TO, relToParentTypes } } + }, + dest: { + index: READ_INDEX_STIX_CORE_RELATIONSHIPS + }, + script: { + source: reindexLinkedToToRelatedToSource, + params: { linkedToType, relToType: RELATION_RELATED_TO, relToParentTypes } } }; @@ -44,15 +42,15 @@ export const up = async (next) => { // Then, We need to update all rel that had a linked-to ref as from or to const relToTypes = [RELATION_RELATED_TO, 'basic-relationship', 'stix-relationship', 'stix-core-relationship']; - const relWithLinkedToToRelatedToSource = ` - if(ctx._source.fromType == params.linkedToType) {ctx._source.fromType=params.relToType} ; - if(ctx._source.toType == params.linkedToType) {ctx._source.toType=params.relToType} ; + const updateSCOWithLinkedToFromOrToSource = ` + if(ctx._source.fromType == params.linkedToType) {ctx._source.fromType=params.relToType}; + if(ctx._source.toType == params.linkedToType) {ctx._source.toType=params.relToType}; for(connection in ctx._source.connections) { if(connection.types.contains(params.linkedToType) { connection.types = params.relToTypes) }; }`; const updateSCOWithLinkedToFromOrToQuery = { script: { - source: relWithLinkedToToRelatedToSource, + source: updateSCOWithLinkedToFromOrToSource, params: { linkedToType, relToType: RELATION_RELATED_TO, relToTypes } }, query: { @@ -71,7 +69,43 @@ export const up = async (next) => { await elUpdateByQueryForMigration('[MIGRATION] Updating relation with a linked-to from or to', READ_RELATIONSHIPS_INDICES, updateSCOWithLinkedToFromOrToQuery); - // Then we need to move all denormalized linked-to rel in objects to related-to + // Then we need to move all denormalized linked-to rel in objects to related-to (if id is not already in related-to) + const updateDenormalizedLinkedToSource = ` + ArrayList linkedTos = ctx._source[params.relLinkedTo]; + if(!ctx._source.containsKey(params.relRelatedTo)) {ctx._source[params.relRelatedTo]=[] }; + for(linkedTo in linkedTos) { + if(!ctx._source[relRelatedTo].contains(linkedTo)) { ctx._source[params.relRelatedTo].add(linkedTo) } + }; + ctx._source.remove(params.relLinkedTo)`; + const updateDenormalizedLinkedToQuery = { + script: { + source: updateDenormalizedLinkedToSource, + params: { relLinkedTo: `rel_${linkedToType}.internal_id`, relRelatedTo: `rel_${RELATION_RELATED_TO}.internal_id` } + }, + query: { + exists: { + field: `rel_${linkedToType}` + } + }, + }; + + await elUpdateByQueryForMigration('[MIGRATION] Updating entities with rel_linked-to to rel_related-to', READ_DATA_INDICES, updateDenormalizedLinkedToQuery); + + // Finally, we delete all original linked-to refs in meta rel index + + await elDeleteByQueryForMigration( + '[MIGRATION] Deleting all linked-to refs', + [READ_INDEX_STIX_META_RELATIONSHIPS], + { + query: { + bool: { + must: [ + { term: { 'entity_type.keyword': { value: linkedToType } } } + ] + } + } + } + ); logApp.info('[MIGRATION] Update all linked-to refs to related-to rels finished');