[backend] mark files as removed when deleting entity

OpenCTI-Platform · Apr 4, 2024 · 626d6b7 · 626d6b7
1 parent d0be277
commit 626d6b7
Show file tree

Hide file tree

Showing 4 changed files with 48 additions and 3 deletions.
diff --git a/opencti-platform/opencti-graphql/src/database/file-search.js b/opencti-platform/opencti-graphql/src/database/file-search.js
@@ -103,6 +103,29 @@ export const elUpdateFilesWithEntityRestrictions = async (entity) => {
   });
 };
 
+export const elUpdateRemovedFiles = async (entity, removed = true) => {
+  if (!entity) {
+    return null;
+  }
+  const params = { removed };
+  const source = 'ctx._source["removed"] = params.removed;';
+  return elRawUpdateByQuery({
+    index: READ_INDEX_FILES,
+    refresh: true,
+    conflicts: 'proceed',
+    body: {
+      script: { source, params },
+      query: {
+        term: {
+          'entity_id.keyword': entity.internal_id
+        }
+      },
+    },
+  }).catch((err) => {
+    throw DatabaseError('Files entity removed update fail', { cause: err, entityId: entity.internal_id });
+  });
+};
+
 const buildFilesSearchResult = (data, first, searchAfter, connectionFormat = true, includeContent = false) => {
   const convertedHits = data.hits.hits.map((hit) => {
     const elementData = hit._source;
@@ -142,7 +165,7 @@ const decodeSearch = (search) => {
 };
 const elBuildSearchFilesQueryBody = async (context, user, options = {}) => {
   const { search = null, fileIds = [], entityIds = [] } = options; // search options
-  const { includeAuthorities = false } = options;
+  const { includeAuthorities = false, excludeRemoved = true } = options;
   const dataRestrictions = await buildDataRestrictions(context, user, { includeAuthorities });
   const must = [...dataRestrictions.must];
   const mustNot = [...dataRestrictions.must_not];
@@ -162,6 +185,18 @@ const elBuildSearchFilesQueryBody = async (context, user, options = {}) => {
   if (entityIds?.length > 0) {
     must.push({ terms: { 'entity_id.keyword': entityIds } });
   }
+  // exclude removed files (logical deletion)
+  if (excludeRemoved) {
+    const excludeRemovedQuery = {
+      bool: {
+        should: [
+          { term: { removed: { value: false } } },
+          { bool: { must_not: [{ exists: { field: 'removed' } }] } }
+        ]
+      }
+    };
+    must.push(excludeRemovedQuery);
+  }
   return {
     query: {
       bool: {

diff --git a/opencti-platform/opencti-graphql/src/database/middleware.js b/opencti-platform/opencti-graphql/src/database/middleware.js
@@ -150,9 +150,10 @@ import {
 import { ENTITY_TYPE_EXTERNAL_REFERENCE, ENTITY_TYPE_LABEL, isStixMetaObject } from '../schema/stixMetaObject';
 import { isStixSightingRelationship } from '../schema/stixSightingRelationship';
 import { ENTITY_HASHED_OBSERVABLE_ARTIFACT, ENTITY_HASHED_OBSERVABLE_STIX_FILE, isStixCyberObservable, isStixCyberObservableHashedObservable } from '../schema/stixCyberObservable';
-import conf, { BUS_TOPICS, logApp } from '../config/conf';
+import conf, { BUS_TOPICS, isFeatureEnabled, logApp } from '../config/conf';
 import { FROM_START, FROM_START_STR, mergeDeepRightAll, now, prepareDate, UNTIL_END, UNTIL_END_STR, utcDate } from '../utils/format';
 import { checkObservableSyntax } from '../utils/syntax';
+import { elUpdateRemovedFiles } from './file-search';
 import { deleteAllObjectFiles, storeFileConverter, upload } from './file-storage';
 import {
   BYPASS_REFERENCE,
@@ -3379,7 +3380,14 @@ export const internalDeleteElementById = async (context, user, id, opts = {}) =>
       element.from = instance; // dynamically update the from to have an up to date relation
     } else {
       // Start by deleting external files
-      await deleteAllObjectFiles(context, user, element);
+      if (isFeatureEnabled('LOGICAL_DELETION')) {
+        // do not delete files if logical deletion enabled
+        // mark indexed files as removed to exclude them from search
+        await elUpdateRemovedFiles(element, true);
+      } else {
+        // if logical deletion is disabled, delete files as usual
+        await deleteAllObjectFiles(context, user, element);
+      }
       // Delete all linked elements
       await elDeleteElements(context, user, [element]);
       // Publish event in the stream

diff --git a/opencti-platform/opencti-graphql/src/modules/internal/document/document-domain.ts b/opencti-platform/opencti-graphql/src/modules/internal/document/document-domain.ts
@@ -26,6 +26,7 @@ export const getIndexFromDate = async (context: AuthContext) => {
     highlight: false,
     orderBy: 'uploaded_at',
     orderMode: 'desc',
+    excludeRemoved: false,
   };
   const lastIndexedFiles = await elSearchFiles(context, SYSTEM_USER, searchOptions);
   const lastIndexedFile = lastIndexedFiles?.length > 0 ? lastIndexedFiles[0] : null;

diff --git a/opencti-platform/opencti-graphql/src/modules/internal/document/document.ts b/opencti-platform/opencti-graphql/src/modules/internal/document/document.ts
@@ -79,6 +79,7 @@ const attributes: Array<AttributeDefinition> = [
   { name: 'uploaded_at', label: 'Upload date', type: 'date', mandatoryType: 'internal', editDefault: false, multiple: false, upsert: false, isFilterable: true },
   { name: 'file_id', label: 'File identifier', type: 'string', format: 'short', mandatoryType: 'internal', editDefault: false, multiple: false, upsert: false, isFilterable: true },
   { name: 'entity_id', label: 'Related entity', type: 'string', format: 'short', mandatoryType: 'internal', editDefault: false, multiple: false, upsert: false, isFilterable: true },
+  { name: 'removed', label: 'Removed', type: 'boolean', mandatoryType: 'no', editDefault: false, multiple: false, upsert: false, isFilterable: false },
 ];
 
 schemaAttributesDefinition.registerAttributes(ENTITY_TYPE_INTERNAL_FILE, attributes);