[backend] WIP migration file

opencti-platform/opencti-graphql/src/database/engine.js

@@ -10,6 +10,7 @@ import {
   buildPagination,
   cursorToOffset,
   ES_INDEX_PREFIX,
+  INDEX_FILES,
   INDEX_INTERNAL_OBJECTS,
   inferIndexFromConceptType,
   isEmptyField,
@@ -314,6 +315,7 @@ export const elRawSearch = (context, user, types, query) => {
 export const elRawDeleteByQuery = (query) => engine.deleteByQuery(query).then((r) => oebp(r));
 export const elRawBulk = (args) => engine.bulk(args).then((r) => oebp(r));
 export const elRawUpdateByQuery = (query) => engine.updateByQuery(query).then((r) => oebp(r));
+export const elRawReindexByQuery = (query) => engine.reindex(query).then((r) => oebp(r));
 
 const elOperationForMigration = (operation) => {
   const elGetTask = (taskId) => engine.tasks.get({ task_id: taskId }).then((r) => oebp(r));
@@ -347,6 +349,7 @@ const elOperationForMigration = (operation) => {
 
 export const elUpdateByQueryForMigration = elOperationForMigration(elRawUpdateByQuery);
 export const elDeleteByQueryForMigration = elOperationForMigration(elRawDeleteByQuery);
+export const elReindexByQueryForMigration = elOperationForMigration(elRawReindexByQuery);
 
 export const buildDataRestrictions = async (context, user, opts = {}) => {
   const must = [];
@@ -3171,6 +3174,12 @@ const computeDeleteElementsImpacts = async (cleanupRelations, toBeRemovedIds, re
   return elementsImpact;
 };
 
+export const elRawReindex = async (body) => {
+  return engine.reindex(body).catch((err) => {
+    throw DatabaseError(`Reindexing fail with ${body} to`, { cause: err, body });
+  });
+};
+
 export const elDeleteElements = async (context, user, elements) => {
   if (elements.length === 0) return;
   const { relations, relationsToRemoveMap } = await getRelationsToRemove(context, user, elements);

opencti-platform/opencti-graphql/src/migrations/1711029985768-convert_linked-to_to_related-toOld.js

@@ -0,0 +1,50 @@
+import { logApp } from '../config/conf';
+import { executionContext, SYSTEM_USER } from '../utils/access';
+import { createRelationRaw, deleteElementById } from '../database/middleware';
+import { elList } from '../database/engine';
+import { READ_RELATIONSHIPS_INDICES } from '../database/utils';
+import { RELATION_RELATED_TO } from '../schema/stixCoreRelationship';
+import { stixObjectOrRelationshipAddRefRelations } from '../domain/stixObjectOrStixRelationship';
+import { ABSTRACT_STIX_DOMAIN_OBJECT, ENTITY_TYPE_CONTAINER } from '../schema/general';
+import { listEntitiesThroughRelationsPaginated } from '../database/middleware-loader';
+import { RELATION_OBJECT } from '../schema/stixRefRelationship';
+
+export const up = async (next) => {
+  const context = executionContext('migration');
+  logApp.info('[MIGRATION] Update all linked-to refs to related-to rels');
+  const callback = async (linkedToRefs) => {
+    // For each linked-to ref, we create a related-to rel, add the containers from the linked-to ref to the related-to rel, and we delete the linked-to ref
+    for (let linkedRefIndex = 0; linkedRefIndex < linkedToRefs.length; linkedRefIndex += 1) {
+      const linkedToRef = linkedToRefs[linkedRefIndex];
+      // Create related-to
+      const createdRelatedToRel = await createRelationRaw(context, SYSTEM_USER, {
+        fromId: linkedToRef.fromId,
+        relationship_type: RELATION_RELATED_TO,
+        toId: linkedToRef.toId
+      });
+      // Get all linked-to ref containers, add new related-to rel to those containers
+      // We first check if linkedToRef has any rel_object.internal_id before making an unecessary containers list query
+      if (linkedToRef['rel_object.internal_id'] !== undefined) {
+        const linkedToContainers = await listEntitiesThroughRelationsPaginated(context, SYSTEM_USER, linkedToRef.id, RELATION_OBJECT, [ENTITY_TYPE_CONTAINER], true, opts);
+
+        for (let containerIndex = 0; containerIndex < linkedToContainers.edges?.length; containerIndex += 1) {
+          const container = linkedToContainers.edges[containerIndex].node;
+          const addToContainerInput = { relationship_type: 'object', toIds: [createdRelatedToRel.element.id], };
+          await stixObjectOrRelationshipAddRefRelations(context, SYSTEM_USER, container.id, addToContainerInput, ABSTRACT_STIX_DOMAIN_OBJECT);
+        }
+      }
+      // Delete linked-to ref
+      await deleteElementById(context, SYSTEM_USER, linkedToRef.id, 'x_opencti_linked-to');
+    }
+  };
+  const opts = { types: ['x_opencti_linked-to'], callback };
+  await elList(context, SYSTEM_USER, READ_RELATIONSHIPS_INDICES, opts);
+
+  logApp.info('[MIGRATION] Update all linked-to refs to related-to rels finished');
+
+  next();
+};
+
+export const down = async (next) => {
+  next();
+};

opencti-platform/opencti-graphql/src/migrations/1711029986768-convert_linked-to_related-to.js

@@ -0,0 +1,83 @@
+import { logApp } from '../config/conf';
+import { elReindexByQueryForMigration, elUpdateByQueryForMigration } from '../database/engine';
+import { READ_INDEX_STIX_CORE_RELATIONSHIPS, READ_INDEX_STIX_META_RELATIONSHIPS, READ_RELATIONSHIPS_INDICES } from '../database/utils';
+import { RELATION_RELATED_TO } from '../schema/stixCoreRelationship';
+
+export const up = async (next) => {
+  logApp.info('[MIGRATION] Update all linked-to refs to related-to rels');
+
+  const linkedToType = 'x_opencti_linked-to';
+  // First, we reindex all linked-to meta refs to core rel index, updating all linked-to refs to related-to rel in the process
+  const linkedToToRelatedToSript = `
+    ctx._source.entity_type = params.relToType;
+    ctx._source.relationship_type = params.relToType;
+    ctx._source.parentTypes = params.relToParentTypes;
+    ctx._source.standard_id = ctx._source.standard_id.replace("relationship-meta--", "relationship--");
+    for(connection in ctx._source.connections) {
+      connection.role = connection.role.replace(params.linkedToType, params.relToType);
+    }`;
+  const relToParentTypes = ['basic-relationship', 'stix-relationship', 'stix-core-relationship'];
+
+  const reindexLinkedToToRelatedToQuery = {
+    body: {
+      source: {
+        index: READ_INDEX_STIX_META_RELATIONSHIPS,
+        query: {
+          bool: {
+            must: [
+              { term: { 'entity_type.keyword': { value: linkedToType } } }
+            ]
+          }
+        }
+      },
+      dest: {
+        index: READ_INDEX_STIX_CORE_RELATIONSHIPS
+      },
+      script: {
+        source: linkedToToRelatedToSript,
+        params: { linkedToType, relToType: RELATION_RELATED_TO, relToParentTypes }
+      }
+    }
+  };
+
+  await elReindexByQueryForMigration('[MIGRATION] Reindexing and updating linked-to refs', null, reindexLinkedToToRelatedToQuery);
+
+  // Then, We need to update all rel that had a linked-to ref as from or to
+  const relToTypes = [RELATION_RELATED_TO, 'basic-relationship', 'stix-relationship', 'stix-core-relationship'];
+  const relWithLinkedToToRelatedToSource = `
+    if(ctx._source.fromType == params.linkedToType) {ctx._source.fromType=params.relToType} ;
+    if(ctx._source.toType == params.linkedToType) {ctx._source.toType=params.relToType} ;
+    for(connection in ctx._source.connections) {
+      if(connection.types.contains(params.linkedToType) { connection.types = params.relToTypes) };
+    }`;
+  const updateSCOWithLinkedToFromOrToQuery = {
+    script: {
+      source: relWithLinkedToToRelatedToSource,
+      params: { linkedToType, relToType: RELATION_RELATED_TO, relToTypes }
+    },
+    query: {
+      nested: {
+        path: 'connections',
+        query: {
+          bool: {
+            must: [
+              { term: { 'connections.types.keyword': { value: linkedToType } } }
+            ]
+          }
+        },
+      }
+    },
+  };
+
+  await elUpdateByQueryForMigration('[MIGRATION] Updating relation with a linked-to from or to', READ_RELATIONSHIPS_INDICES, updateSCOWithLinkedToFromOrToQuery);
+
+  // Then we need to move all denormalized linked-to rel in objects to related-to
+
+  logApp.info('[MIGRATION] Update all linked-to refs to related-to rels finished');
+
+  next();
+};
+
+export const down = async (next) => {
+  next();
+};