[backend] WIP migration file

OpenCTI-Platform · Apr 4, 2024 · be9a56d · be9a56d
1 parent 1be8bf1
commit be9a56d
Show file tree

Hide file tree

Showing 3 changed files with 59 additions and 76 deletions.
diff --git a/opencti-platform/opencti-graphql/src/database/engine.js b/opencti-platform/opencti-graphql/src/database/engine.js
@@ -10,7 +10,6 @@ import {
   buildPagination,
   cursorToOffset,
   ES_INDEX_PREFIX,
-  INDEX_FILES,
   INDEX_INTERNAL_OBJECTS,
   inferIndexFromConceptType,
   isEmptyField,
@@ -324,7 +323,7 @@ const elOperationForMigration = (operation) => {
     logApp.info(`${message} > started`);
     // Execute the update by query in async mode
     const queryAsync = await operation({
-      index,
+      ...(index ?? {}),
       refresh: true,
       wait_for_completion: false,
       body

diff --git a/...atform/opencti-graphql/src/migrations/1711029985768-convert_linked-to_to_related-toOld.js b/...atform/opencti-graphql/src/migrations/1711029985768-convert_linked-to_to_related-toOld.js
diff --git a/...029986768-convert_linked-to_related-to.js → ...225747207-convert_linked-to_related-to.js b/...029986768-convert_linked-to_related-to.js → ...225747207-convert_linked-to_related-to.js
@@ -1,14 +1,14 @@
 import { logApp } from '../config/conf';
-import { elReindexByQueryForMigration, elUpdateByQueryForMigration } from '../database/engine';
-import { READ_INDEX_STIX_CORE_RELATIONSHIPS, READ_INDEX_STIX_META_RELATIONSHIPS, READ_RELATIONSHIPS_INDICES } from '../database/utils';
+import { elDeleteByQueryForMigration, elReindexByQueryForMigration, elUpdateByQueryForMigration } from '../database/engine';
+import { READ_DATA_INDICES, READ_INDEX_STIX_CORE_RELATIONSHIPS, READ_INDEX_STIX_META_RELATIONSHIPS, READ_RELATIONSHIPS_INDICES } from '../database/utils';
 import { RELATION_RELATED_TO } from '../schema/stixCoreRelationship';
 
 export const up = async (next) => {
   logApp.info('[MIGRATION] Update all linked-to refs to related-to rels');
 
   const linkedToType = 'x_opencti_linked-to';
   // First, we reindex all linked-to meta refs to core rel index, updating all linked-to refs to related-to rel in the process
-  const linkedToToRelatedToSript = `
+  const reindexLinkedToToRelatedToSource = `
     ctx._source.entity_type = params.relToType;
     ctx._source.relationship_type = params.relToType;
     ctx._source.parentTypes = params.relToParentTypes;
@@ -19,40 +19,38 @@ export const up = async (next) => {
   const relToParentTypes = ['basic-relationship', 'stix-relationship', 'stix-core-relationship'];
 
   const reindexLinkedToToRelatedToQuery = {
-    body: {
-      source: {
-        index: READ_INDEX_STIX_META_RELATIONSHIPS,
-        query: {
-          bool: {
-            must: [
-              { term: { 'entity_type.keyword': { value: linkedToType } } }
-            ]
-          }
+    source: {
+      index: READ_INDEX_STIX_META_RELATIONSHIPS,
+      query: {
+        bool: {
+          must: [
+            { term: { 'entity_type.keyword': { value: linkedToType } } }
+          ]
         }
-      },
-      dest: {
-        index: READ_INDEX_STIX_CORE_RELATIONSHIPS
-      },
-      script: {
-        source: linkedToToRelatedToSript,
-        params: { linkedToType, relToType: RELATION_RELATED_TO, relToParentTypes }
       }
+    },
+    dest: {
+      index: READ_INDEX_STIX_CORE_RELATIONSHIPS
+    },
+    script: {
+      source: reindexLinkedToToRelatedToSource,
+      params: { linkedToType, relToType: RELATION_RELATED_TO, relToParentTypes }
     }
   };
 
   await elReindexByQueryForMigration('[MIGRATION] Reindexing and updating linked-to refs', null, reindexLinkedToToRelatedToQuery);
 
   // Then, We need to update all rel that had a linked-to ref as from or to
   const relToTypes = [RELATION_RELATED_TO, 'basic-relationship', 'stix-relationship', 'stix-core-relationship'];
-  const relWithLinkedToToRelatedToSource = `
-    if(ctx._source.fromType == params.linkedToType) {ctx._source.fromType=params.relToType} ;
-    if(ctx._source.toType == params.linkedToType) {ctx._source.toType=params.relToType} ;
+  const updateSCOWithLinkedToFromOrToSource = `
+    if(ctx._source.fromType == params.linkedToType) {ctx._source.fromType=params.relToType};
+    if(ctx._source.toType == params.linkedToType) {ctx._source.toType=params.relToType};
     for(connection in ctx._source.connections) {
       if(connection.types.contains(params.linkedToType) { connection.types = params.relToTypes) };
     }`;
   const updateSCOWithLinkedToFromOrToQuery = {
     script: {
-      source: relWithLinkedToToRelatedToSource,
+      source: updateSCOWithLinkedToFromOrToSource,
       params: { linkedToType, relToType: RELATION_RELATED_TO, relToTypes }
     },
     query: {
@@ -71,7 +69,43 @@ export const up = async (next) => {
 
   await elUpdateByQueryForMigration('[MIGRATION] Updating relation with a linked-to from or to', READ_RELATIONSHIPS_INDICES, updateSCOWithLinkedToFromOrToQuery);
 
-  // Then we need to move all denormalized linked-to rel in objects to related-to
+  // Then we need to move all denormalized linked-to rel in objects to related-to (if id is not already in related-to)
+  const updateDenormalizedLinkedToSource = `
+    ArrayList linkedTos = ctx._source[params.relLinkedTo];
+    if(!ctx._source.containsKey(params.relRelatedTo)) {ctx._source[params.relRelatedTo]=[] };
+    for(linkedTo in linkedTos) {
+      if(!ctx._source[relRelatedTo].contains(linkedTo)) { ctx._source[params.relRelatedTo].add(linkedTo) }
+    };
+    ctx._source.remove(params.relLinkedTo)`;
+  const updateDenormalizedLinkedToQuery = {
+    script: {
+      source: updateDenormalizedLinkedToSource,
+      params: { relLinkedTo: `rel_${linkedToType}.internal_id`, relRelatedTo: `rel_${RELATION_RELATED_TO}.internal_id` }
+    },
+    query: {
+      exists: {
+        field: `rel_${linkedToType}`
+      }
+    },
+  };
+
+  await elUpdateByQueryForMigration('[MIGRATION] Updating entities with rel_linked-to to rel_related-to', READ_DATA_INDICES, updateDenormalizedLinkedToQuery);
+
+  // Finally, we delete all original linked-to refs in meta rel index
+
+  await elDeleteByQueryForMigration(
+    '[MIGRATION] Deleting all linked-to refs',
+    [READ_INDEX_STIX_META_RELATIONSHIPS],
+    {
+      query: {
+        bool: {
+          must: [
+            { term: { 'entity_type.keyword': { value: linkedToType } } }
+          ]
+        }
+      }
+    }
+  );
 
   logApp.info('[MIGRATION] Update all linked-to refs to related-to rels finished');