[backend] Remove organization cache (#8806)

opencti-platform/opencti-graphql/src/database/stix-converter.ts

@@ -225,6 +225,7 @@ export const buildOCTIExtensions = (instance: StoreObject): S.StixOpenctiExtensi
     granted_refs: (instance[INPUT_GRANTED_REFS] ?? []).map((m) => m.standard_id),
     // Internals
     creator_ids: builtCreatorIds,
+    granted_refs_ids: (instance[INPUT_GRANTED_REFS] ?? []).map((m) => m.internal_id),
     assignee_ids: (instance[INPUT_ASSIGNEE] ?? []).map((m) => m.internal_id),
     participant_ids: (instance[INPUT_PARTICIPANT] ?? []).map((m) => m.internal_id),
     authorized_members: instance.authorized_members ?? undefined,

opencti-platform/opencti-graphql/src/manager/cacheManager.ts

@@ -33,7 +33,6 @@ import type { StixObject } from '../types/stix-common';
 import { STIX_EXT_OCTI } from '../types/stix-extensions';
 import type { BasicStoreRelation, BasicStreamEntity, BasicTriggerEntity, BasicWorkflowStatusEntity, BasicWorkflowTemplateEntity, StoreEntity, StoreRelation } from '../types/store';
 import { executionContext, SYSTEM_USER } from '../utils/access';
-import { ENTITY_TYPE_IDENTITY_ORGANIZATION } from '../modules/organization/organization-types';
 import { ENTITY_TYPE_MANAGER_CONFIGURATION } from '../modules/managerConfiguration/managerConfiguration-types';
 import type { BasicStoreEntityPlaybook, ComponentDefinition } from '../modules/playbook/playbook-types';
 import { ENTITY_TYPE_PLAYBOOK } from '../modules/playbook/playbook-types';
@@ -102,12 +101,6 @@ const platformConnectors = (context: AuthContext) => {
   };
   return { values: null, fn: reloadConnectors };
 };
-const platformOrganizations = (context: AuthContext) => {
-  const reloadOrganizations = () => {
-    return listAllEntities(context, SYSTEM_USER, [ENTITY_TYPE_IDENTITY_ORGANIZATION], { connectionFormat: false });
-  };
-  return { values: null, fn: reloadOrganizations };
-};
 const platformRules = (context: AuthContext) => {
   const reloadRules = () => {
     return listAllEntities(context, SYSTEM_USER, [ENTITY_TYPE_RULE], { connectionFormat: false });
@@ -237,7 +230,6 @@ const initCacheManager = () => {
     writeCacheForEntity(ENTITY_TYPE_PLAYBOOK, platformRunningPlaybooks(context));
     writeCacheForEntity(ENTITY_TYPE_RULE, platformRules(context));
     writeCacheForEntity(ENTITY_TYPE_DECAY_RULE, platformDecayRules(context));
-    writeCacheForEntity(ENTITY_TYPE_IDENTITY_ORGANIZATION, platformOrganizations(context));
     writeCacheForEntity(ENTITY_TYPE_RESOLVED_FILTERS, platformResolvedFilters(context));
     writeCacheForEntity(ENTITY_TYPE_STREAM_COLLECTION, platformStreams(context));
     writeCacheForEntity(ENTITY_TYPE_NOTIFIER, platformNotifiers(context));

opencti-platform/opencti-graphql/src/manager/historyManager.ts

@@ -11,7 +11,7 @@ import type { SseEvent, StreamDataEvent, UpdateEvent } from '../types/event';
 import { utcDate } from '../utils/format';
 import { elIndexElements } from '../database/engine';
 import type { StixRelation, StixSighting } from '../types/stix-sro';
-import { listEntities } from '../database/middleware-loader';
+import { internalFindByIds, listEntities } from '../database/middleware-loader';
 import type { BasicRuleEntity, BasicStoreEntity } from '../types/store';
 import { BASE_TYPE_ENTITY, STIX_TYPE_RELATION, STIX_TYPE_SIGHTING } from '../schema/general';
 import { generateStandardId } from '../schema/identifier';
@@ -52,12 +52,42 @@ export interface HistoryData extends BasicStoreEntity {
   context_data: HistoryContext;
 }
 
-const eventsApplyHandler = async (context: AuthContext, events: Array<SseEvent<StreamDataEvent>>) => {
-  if (isEmptyField(events) || events.length === 0) {
-    return;
+/**
+ * Function to resolve granted_refs when granted_refs_ids are not present (have been added on nov 2024)
+ * This is needed to be able to process older events, and will be removed after a year
+ * @param context
+ * @param events
+ */
+export const resolveGrantedRefsIds = async (context: AuthContext, events: Array<SseEvent<StreamDataEvent>>) => {
+  const grantedRefsToResolve: StixId[] = [];
+  events.forEach((event) => {
+    const stix = event.data.data;
+    const eventGrantedRefsIds = (stix.extensions[STIX_EXT_OCTI].granted_refs_ids ?? []);
+    const eventGrantedRefsStandardIds = (stix.extensions[STIX_EXT_OCTI].granted_refs ?? []);
+    if (eventGrantedRefsIds.length === 0 && eventGrantedRefsStandardIds.length > 0) {
+      grantedRefsToResolve.push(...eventGrantedRefsStandardIds);
+    }
+  });
+  const organizationByIdsMap = new Map<string, string>();
+  if (grantedRefsToResolve.length === 0) {
+    return organizationByIdsMap; // nothing to resolve
   }
+  const organizationsByIds = await internalFindByIds(context, SYSTEM_USER, R.uniq(grantedRefsToResolve), {
+    type: ENTITY_TYPE_IDENTITY_ORGANIZATION,
+    baseData: true,
+    baseFields: ['standard_id', 'internal_id'],
+  });
+  organizationsByIds.forEach((o) => {
+    organizationByIdsMap.set(o.standard_id, o.internal_id);
+  });
+  return organizationByIdsMap;
+};
+
+export const buildHistoryElementsFromEvents = async (context:AuthContext, events: Array<SseEvent<StreamDataEvent>>) => {
+  // load all markings to resolve object_marking_refs
   const markingsById = await getEntitiesMapFromCache<BasicRuleEntity>(context, SYSTEM_USER, ENTITY_TYPE_MARKING_DEFINITION);
-  const organizationsById = await getEntitiesMapFromCache<BasicStoreEntity>(context, SYSTEM_USER, ENTITY_TYPE_IDENTITY_ORGANIZATION);
+  // resolve granted_refs
+  const grantedRefsResolved = await resolveGrantedRefsIds(context, events);
   // Build the history data
   const historyElements = events.map((event) => {
     const [time] = event.id.split('-');
@@ -66,9 +96,13 @@ const eventsApplyHandler = async (context: AuthContext, events: Array<SseEvent<S
     const eventMarkingRefs = (stix.object_marking_refs ?? [])
       .map((stixId) => markingsById.get(stixId)?.internal_id)
       .filter((o) => isNotEmptyField(o)) as string[];
-    const eventGrantedRefs = (stix.extensions[STIX_EXT_OCTI].granted_refs ?? [])
-      .map((stixId) => organizationsById.get(stixId)?.internal_id)
-      .filter((o) => isNotEmptyField(o));
+    let eventGrantedRefsIds = (stix.extensions[STIX_EXT_OCTI].granted_refs_ids ?? []);
+    const eventGrantedRefsStandardIds = (stix.extensions[STIX_EXT_OCTI].granted_refs ?? []);
+    if (eventGrantedRefsIds.length === 0 && eventGrantedRefsStandardIds.length > 0) {
+      eventGrantedRefsIds = eventGrantedRefsStandardIds
+        .map((stixId) => grantedRefsResolved.get(stixId))
+        .filter((o) => isNotEmptyField(o));
+    }
     const contextData: HistoryContext = {
       id: stix.extensions[STIX_EXT_OCTI].id,
       message: event.data.message,
@@ -128,9 +162,18 @@ const eventsApplyHandler = async (context: AuthContext, events: Array<SseEvent<S
       context_data: contextData,
       authorized_members: stix.extensions[STIX_EXT_OCTI].authorized_members,
       'rel_object-marking.internal_id': R.uniq(eventMarkingRefs),
-      'rel_granted.internal_id': R.uniq(eventGrantedRefs),
+      'rel_granted.internal_id': R.uniq(eventGrantedRefsIds),
     };
   });
+  return historyElements;
+};
+
+const eventsApplyHandler = async (context: AuthContext, events: Array<SseEvent<StreamDataEvent>>) => {
+  if (isEmptyField(events) || events.length === 0) {
+    return;
+  }
+  // Build the history data
+  const historyElements = await buildHistoryElementsFromEvents(context, events);
   // Bulk the history data insertions
   await elIndexElements(context, SYSTEM_USER, ENTITY_TYPE_HISTORY, historyElements);
 };

opencti-platform/opencti-graphql/src/types/stix-common.d.ts

@@ -41,6 +41,7 @@ interface StixOpenctiExtension {
   files: Array<StixFileExtension>
   aliases: Array<string>
   granted_refs: Array<StixId>
+  granted_refs_ids: string[]
   stix_ids: Array<StixId>
   type: string
   created_at: StixDate

opencti-platform/opencti-graphql/tests/02-integration/04-manager/historyManager-test.js

@@ -0,0 +1,161 @@
+import { describe, expect, it } from 'vitest';
+import { INDEX_HISTORY } from '../../../src/database/utils';
+import { buildHistoryElementsFromEvents, resolveGrantedRefsIds } from '../../../src/manager/historyManager';
+import { ENTITY_TYPE_HISTORY } from '../../../src/schema/internalObject';
+import { testContext } from '../../utils/testQuery';
+
+const eventWithGrantedRefIds = {
+  id: '1731595374948-0',
+  event: 'update',
+  data: {
+    version: '4',
+    type: 'update',
+    scope: 'external',
+    message: 'adds `Filigran` in `Shared with`',
+    origin: {
+      socket: 'query',
+      ip: '::1',
+      user_id: '88ec0c6a-13ce-5e39-b486-354fe4a7084f',
+      group_ids: ['9c746e48-28fd-432a-abd7-d7593eb310c4'],
+      organization_ids: [],
+      user_metadata: {},
+      referer: 'http://localhost:3000/dashboard/analyses/reports/58fbfcfa-01ce-4440-8edf-7ea38e7a6ae9'
+    },
+    data: {
+      id: 'report--d27398f3-8086-50e7-9c71-088b9bd69605',
+      spec_version: '2.1',
+      type: 'report',
+      extensions: {
+        'extension-definition--ea279b3e-5c71-4632-ac08-831c66a786ba': {
+          extension_type: 'property-extension',
+          id: '58fbfcfa-01ce-4440-8edf-7ea38e7a6ae9',
+          type: 'Report',
+          created_at: '2024-02-20T15:34:17.203Z',
+          updated_at: '2024-11-14T14:42:37.551Z',
+          is_inferred: false,
+          granted_refs: ['identity--67fabb23-c547-5c4a-b253-9d9a8548c466', 'identity--8cb00c79-ab20-5ed4-b37d-337241b96a29'],
+          creator_ids: ['88ec0c6a-13ce-5e39-b486-354fe4a7084f'],
+          granted_refs_ids: ['c080a677-f640-4643-9d2a-75929ac07b1c', '0c897410-3579-4770-b26e-1fce2e441204'],
+          workflow_id: '78973513-cebc-49f9-a316-12487acd7903',
+          labels_ids: ['7b705594-e2bc-48f8-bdc3-8c55ce1adb0e']
+        }
+      },
+      created: '2024-02-20T15:34:11.000Z',
+      modified: '2024-11-14T14:42:37.551Z',
+      revoked: false,
+      confidence: 100,
+      lang: 'en',
+      labels: ['label-debug-rename2'],
+      name: 'test',
+      published: '2024-02-20T15:34:11.000Z',
+    },
+    context: {
+      patch: [
+        { op: 'add', path: '/extensions/extension-definition--ea279b3e-5c71-4632-ac08-831c66a786ba/granted_refs_ids/1', value: '0c897410-3579-4770-b26e-1fce2e441204' },
+        { op: 'add', path: '/extensions/extension-definition--ea279b3e-5c71-4632-ac08-831c66a786ba/granted_refs/1', value: 'identity--8cb00c79-ab20-5ed4-b37d-337241b96a29' }
+      ],
+      reverse_patch: [
+        { op: 'remove', path: '/extensions/extension-definition--ea279b3e-5c71-4632-ac08-831c66a786ba/granted_refs_ids/1' },
+        { op: 'remove', path: '/extensions/extension-definition--ea279b3e-5c71-4632-ac08-831c66a786ba/granted_refs/1' }
+      ]
+    }
+  }
+};
+
+const eventWithGrantedRefsOnly = {
+  id: '1731597042395-0',
+  event: 'update',
+  data: {
+    version: '4',
+    type: 'update',
+    scope: 'external',
+    message: 'adds `TestOrganization` in `Shared with`',
+    origin: {
+      socket: 'query',
+      ip: '::1',
+      user_id: '88ec0c6a-13ce-5e39-b486-354fe4a7084f',
+      group_ids: ['9c746e48-28fd-432a-abd7-d7593eb310c4'],
+      organization_ids: [],
+      user_metadata: {},
+      referer: 'http://localhost:3000/dashboard/analyses/reports/58fbfcfa-01ce-4440-8edf-7ea38e7a6ae9'
+    },
+    data: {
+      id: 'report--609acc0c-c821-52e0-a6b2-25be0050bbc0',
+      spec_version: '2.1',
+      type: 'report',
+      extensions: {
+        'extension-definition--ea279b3e-5c71-4632-ac08-831c66a786ba': {
+          extension_type: 'property-extension',
+          id: 'a691be02-fb06-4358-8cf6-a08d97788340',
+          type: 'Report',
+          created_at: '2024-06-10T12:55:17.446Z',
+          updated_at: '2024-07-22T09:21:43.375Z',
+          is_inferred: false,
+          granted_refs: ['identity--a16d7ba8-5bea-5fe5-9d92-931e20e36727'], // TestOrganization
+          creator_ids: ['a93d949b-b56d-4426-b7fe-b79ec3718b0e'],
+          workflow_id: 'b28a370a-317b-4c50-8f0d-483b17d11abb'
+        }
+      },
+      created: '2024-06-10T12:55:08.000Z',
+      modified: '2024-06-10T12:55:40.833Z',
+      revoked: false,
+      confidence: 100,
+      lang: 'en',
+      name: 'test',
+      published: '2024-06-10T12:55:08.000Z',
+      object_refs: ['attack-pattern--033921be-85df-5f05-8bc0-d3d9fc945db9']
+    },
+    context: {
+      patch: [
+        { op: 'add', path: '/extensions/extension-definition--ea279b3e-5c71-4632-ac08-831c66a786ba/granted_refs', value: ['identity--a16d7ba8-5bea-5fe5-9d92-931e20e36727'] }
+      ],
+      reverse_patch: [
+        { op: 'remove', path: '/extensions/extension-definition--ea279b3e-5c71-4632-ac08-831c66a786ba/granted_refs' }
+      ]
+    }
+  }
+};
+
+describe('History manager test resolveGrantedRefsIds', () => {
+  it('should return empty map if granted refs ids are present', async () => {
+    const organizationByIdsMap = await resolveGrantedRefsIds(testContext, [eventWithGrantedRefIds]);
+    expect(organizationByIdsMap.size).toEqual(0);
+  });
+
+  it('should return organization if granted refs are present and not granted refs ids', async () => {
+    const organizationByIdsMap = await resolveGrantedRefsIds(testContext, [eventWithGrantedRefsOnly]);
+    expect(organizationByIdsMap.size).toEqual(1);
+    expect(organizationByIdsMap.has('identity--a16d7ba8-5bea-5fe5-9d92-931e20e36727')).toBeTruthy();
+  });
+});
+
+describe('history manager test buildHistoryElementsFromEvents', () => {
+  it('should build history with granted_refs_ids', async () => {
+    const historyElements = await buildHistoryElementsFromEvents(testContext, [eventWithGrantedRefIds]);
+    expect(historyElements.length).toEqual(1);
+    const historyElement = historyElements[0];
+    expect(historyElement.internal_id).toEqual(eventWithGrantedRefIds.id);
+    expect(historyElement._index).toEqual(INDEX_HISTORY);
+    expect(historyElement.entity_type).toEqual(ENTITY_TYPE_HISTORY);
+    expect(historyElement.event_type).toEqual('mutation');
+    expect(historyElement.event_scope).toEqual(eventWithGrantedRefIds.event);
+    expect(historyElement.user_id).toEqual(eventWithGrantedRefIds.data.origin.user_id);
+    expect(historyElement.group_ids).toEqual(eventWithGrantedRefIds.data.origin.group_ids);
+    expect(historyElement.organization_ids).toEqual(eventWithGrantedRefIds.data.origin.organization_ids);
+    expect(historyElement['rel_granted.internal_id']).toEqual(['c080a677-f640-4643-9d2a-75929ac07b1c', '0c897410-3579-4770-b26e-1fce2e441204']);
+  });
+  it('should build history with granted_refs ids resolved', async () => {
+    const historyElements = await buildHistoryElementsFromEvents(testContext, [eventWithGrantedRefsOnly]);
+    expect(historyElements.length).toEqual(1);
+    const historyElement = historyElements[0];
+    expect(historyElement.internal_id).toEqual(eventWithGrantedRefsOnly.id);
+    expect(historyElement._index).toEqual(INDEX_HISTORY);
+    expect(historyElement.entity_type).toEqual(ENTITY_TYPE_HISTORY);
+    expect(historyElement.event_type).toEqual('mutation');
+    expect(historyElement.event_scope).toEqual(eventWithGrantedRefsOnly.event);
+    expect(historyElement.user_id).toEqual(eventWithGrantedRefsOnly.data.origin.user_id);
+    expect(historyElement.group_ids).toEqual(eventWithGrantedRefsOnly.data.origin.group_ids);
+    expect(historyElement.organization_ids).toEqual(eventWithGrantedRefsOnly.data.origin.organization_ids);
+    expect(historyElement['rel_granted.internal_id'].length).toEqual(1);
+  });
+});