Merge pull request #224 from JupiterOne/jira-enhancement-dev

added complex custom field property

.env.example

@@ -15,4 +15,9 @@ LOCAL_SERVER_JIRA_HOST=http://localhost:8080
 LOCAL_SERVER_JIRA_USERNAME=jupiterone-dev
 LOCAL_SERVER_JIRA_PASSWORD=testing123
 LOCAL_SERVER_PROJECTS=["SP"]
-CUSTOM_FIELDS=["customField1", "custonField2"]
+
+# Simple Custom Fields
+CUSTOM_FIELDS=["customField1", "customField2"]
+
+# Complex Custom Fields
+COMPLEX_CUSTOM_FIELDS=["complexField1.path.to.value", "complexField2.path.to.value"]

src/config.ts

@@ -72,6 +72,11 @@ export const instanceConfigFields: IntegrationInstanceConfigFieldMap = {
     mask: false,
     optional: true,
   },
+  complexCustomFields: {
+    type: 'json',
+    mask: false,
+    optional: true,
+  },
 };
 
 /**
@@ -116,6 +121,13 @@ export interface JiraIntegrationInstanceConfig
    */
   customFields?: string[];
 
+  /**
+   * An optional array of complex custom field paths to transfer to `Issue` entity properties.
+   * The entity property names will be `camelCase(field.path)`.
+   *
+   * - `"complexField1.path.to.value"` - a fully qualified identifier
+   */
+  complexCustomFields?: string[];
   /**
    * Enable bulk ingestion of all Jira issues in the specified projects.
    */
@@ -130,6 +142,7 @@ export type IntegrationConfig = JiraIntegrationInstanceConfig &
   JiraClientConfig & {
     projects: string[];
     customFields: string[];
+    complexCustomFields: string[];
   };
 
 /**
@@ -212,6 +225,9 @@ export function buildNormalizedInstanceConfig(
     apiVersion: jiraApiVersion,
     projects: normalizeProjectKeys(config.projects),
     customFields: normalizeCustomFieldIdentifiers(config.customFields),
+    complexCustomFields: normalizeCustomFieldIdentifiers(
+      config.complexCustomFields,
+    ),
   };
 }
 

src/converters/IssueEntityConverter.ts

@@ -48,11 +48,38 @@ function getIssueDescription(issue: Issue, apiVersion: string): string {
     : parseContent((description as TextContent).content);
 }
 
+function getNestedValue(obj: any, path: string): any {
+  return path.split('.').reduce((acc, part, index) => {
+    if (!acc) {
+      return undefined;
+    }
+
+    const match = part.match(/^(\w+|\[\d+\])(?:\.(.+))?$/);
+    if (match) {
+      const [, key] = match;
+      if (key.startsWith('[') && key.endsWith(']')) {
+        // Accessing array element
+        const arrayIndex = Number(key.slice(1, -1));
+        return acc[arrayIndex];
+      } else {
+        // Accessing object property
+        return acc[key];
+      }
+    }
+    return acc[part];
+  }, obj);
+}
+
+function setFlatNestedValue(obj: any, path: string, value: any): void {
+  obj[path] = value;
+}
+
 export function createIssueEntity({
   issue,
   logger,
   fieldsById,
   customFieldsToInclude,
+  complexCustomFieldsToInclude,
   requestedClass,
   redactIssueDescriptions,
   apiVersion,
@@ -61,17 +88,20 @@ export function createIssueEntity({
   logger: IntegrationLogger;
   fieldsById?: { [id: string]: Field };
   customFieldsToInclude?: string[];
+  complexCustomFieldsToInclude?: string[];
   requestedClass?: unknown;
   redactIssueDescriptions: boolean;
   apiVersion: string;
 }): IssueEntity {
   fieldsById = fieldsById || {};
   customFieldsToInclude = customFieldsToInclude || [];
+  complexCustomFieldsToInclude = complexCustomFieldsToInclude || [];
 
   const status = issue.fields.status && issue.fields.status.name;
   const issueType = issue.fields.issuetype && issue.fields.issuetype.name;
   const customFields: { [key: string]: any } = {};
 
+  // Extract simple custom fields
   for (const [key, value] of Object.entries(issue.fields)) {
     if (
       key.startsWith('customfield_') &&
@@ -94,14 +124,41 @@ export function createIssueEntity({
     }
   }
 
+  // Handle complex custom fields
+  complexCustomFieldsToInclude.forEach((path) => {
+    const [baseFieldId, ...nestedPathParts] = path.split('.');
+    if (issue.fields[baseFieldId] !== undefined) {
+      const nestedPath = nestedPathParts.join('.');
+      const fieldValue = getNestedValue(issue.fields[baseFieldId], nestedPath);
+      if (fieldValue !== undefined) {
+        if (fieldsById && fieldsById[baseFieldId]) {
+          const baseFieldName = camelCase(fieldsById[baseFieldId].name);
+          const formattedPathParts = nestedPathParts.map((part) => {
+            const match = part.match(/^(\w+)(?:\[(\d+)\])?$/);
+            if (match) {
+              const [, key, index] = match;
+              if (index !== undefined) {
+                return `${camelCase(key)}${index}`;
+              }
+              return camelCase(key);
+            }
+            return camelCase(part);
+          });
+          const formattedPath = [baseFieldName, ...formattedPathParts].join('');
+          setFlatNestedValue(customFields, formattedPath, fieldValue);
+        }
+      }
+    }
+  });
+
   if (!['string', 'undefined'].includes(typeof requestedClass)) {
     logger.warn(
       { requestedClass },
       'Invalid entity class. Reverting to default.',
     );
     requestedClass = undefined;
   }
-
+  // Set issue class based on issue type or requested class
   let issueClass: string | string[];
 
   if (requestedClass) {
@@ -131,36 +188,26 @@ export function createIssueEntity({
     }
   }
 
+  // Redact issue descriptions if necessary
   let entityDescription: string;
   if (redactIssueDescriptions) {
     if (issue.fields.description) {
       issue.fields.description = {
         type: 'text',
         text: 'REDACTED',
       };
-    } // don't let description leak in rawData
+    }
     entityDescription = 'REDACTED';
   } else {
     entityDescription = getIssueDescription(issue, apiVersion);
   }
-  //TEMP INT-10020
-  if (Object.keys(customFields).length > 0) {
-    logger.info(
-      { customFieldsToAdd: Object.keys(customFields) },
-      'Adding custom fields to issue',
-    );
-  }
 
-  const entity = {
+  // Construct issue entity object
+  const entity: IssueEntity = {
     _key: generateEntityKey(ISSUE_ENTITY_TYPE, issue.id),
     _type: ISSUE_ENTITY_TYPE,
     _class: issueClass,
-    _rawData: [
-      {
-        name: 'default',
-        rawData: issue as any,
-      },
-    ],
+    _rawData: [{ name: 'default', rawData: issue as any }],
     ...customFields,
     id: issue.id,
     key: issue.key,
@@ -194,11 +241,10 @@ export function createIssueEntity({
       issue.fields.components && issue.fields.components.map((c) => c.name),
     priority: issue.fields.priority && issue.fields.priority.name,
   };
+
+  // Add event data if requested class is provided
   if (requestedClass) {
-    entity._rawData.push({
-      name: 'event',
-      rawData: { requestedClass },
-    });
+    entity._rawData?.push({ name: 'event', rawData: { requestedClass } });
   }
   return entity;
 }

src/steps/issues.ts

@@ -79,13 +79,24 @@ export async function fetchIssues({
     'Custom fields to ingest.',
   );
 
+  logger.info(
+    {
+      complexCustomFields: config.complexCustomFields,
+      allFieldIdsAndNames: Object.values(fieldsById).map(
+        (field) => `${field.id}: ${field.name ?? 'undefined field name'}`,
+      ),
+    },
+    'Complex custom fields to ingest.',
+  );
+
   const issueProcessor = async (projectKey: JiraProjectKey, issue: Issue) =>
     processIssue(
       {
         logger,
         jobState,
         fieldsById,
         customFieldsToInclude: config.customFields,
+        complexCustomFieldsToInclude: config.complexCustomFields,
         projectEntities,
         redactIssueDescriptions: redactIssueDescriptions ?? false,
         apiVersion,
@@ -134,6 +145,7 @@ type ProcessIssueContext = {
   logger: IntegrationLogger;
   fieldsById: IdFieldMap;
   customFieldsToInclude: string[];
+  complexCustomFieldsToInclude: string[];
   projectEntities: ProjectEntity[];
   redactIssueDescriptions: boolean;
   apiVersion: string;
@@ -153,6 +165,7 @@ async function processIssue(
     logger,
     jobState,
     customFieldsToInclude,
+    complexCustomFieldsToInclude,
     fieldsById,
     projectEntities,
     redactIssueDescriptions,
@@ -168,6 +181,7 @@ async function processIssue(
         logger,
         fieldsById,
         customFieldsToInclude,
+        complexCustomFieldsToInclude,
         redactIssueDescriptions,
         apiVersion,
       }),