From 00d5192fe7a09e3955455c593cb058a0d5b97289 Mon Sep 17 00:00:00 2001 From: poornima-metron Date: Thu, 27 Jun 2024 00:15:55 +0530 Subject: [PATCH 1/6] added complex custom field property --- .env.example | 7 ++++++- src/config.ts | 14 +++++++++++++ src/converters/IssueEntityConverter.ts | 27 ++++++++++++++++++++++++++ src/steps/issues.ts | 14 +++++++++++++ 4 files changed, 61 insertions(+), 1 deletion(-) diff --git a/.env.example b/.env.example index 87b7897..ab77b96 100644 --- a/.env.example +++ b/.env.example @@ -15,4 +15,9 @@ LOCAL_SERVER_JIRA_HOST=http://localhost:8080 LOCAL_SERVER_JIRA_USERNAME=jupiterone-dev LOCAL_SERVER_JIRA_PASSWORD=testing123 LOCAL_SERVER_PROJECTS=["SP"] -CUSTOM_FIELDS=["customField1", "custonField2"] + +# Simple Custom Fields +CUSTOM_FIELDS=["customField1", "customField2"] + +# Complex Custom Fields +COMPLEX_CUSTOM_FIELDS=["complexField1.path.to.value", "complexField2.path.to.value"] diff --git a/src/config.ts b/src/config.ts index 6ee232c..98b773f 100644 --- a/src/config.ts +++ b/src/config.ts @@ -72,6 +72,11 @@ export const instanceConfigFields: IntegrationInstanceConfigFieldMap = { mask: false, optional: true, }, + complexCustomFields: { + type: 'json', + mask: false, + optional: true, + } }; /** @@ -116,6 +121,13 @@ export interface JiraIntegrationInstanceConfig */ customFields?: string[]; + /** + * An optional array of complex custom field paths to transfer to `Issue` entity properties. + * The entity property names will be `camelCase(field.path)`. + * + * - `"complexField1.path.to.value"` - a fully qualified identifier + */ + complexCustomFields?: string[]; /** * Enable bulk ingestion of all Jira issues in the specified projects. */ @@ -130,6 +142,7 @@ export type IntegrationConfig = JiraIntegrationInstanceConfig & JiraClientConfig & { projects: string[]; customFields: string[]; + complexCustomFields: string[]; }; /** @@ -212,6 +225,7 @@ export function buildNormalizedInstanceConfig( apiVersion: jiraApiVersion, projects: normalizeProjectKeys(config.projects), customFields: normalizeCustomFieldIdentifiers(config.customFields), + complexCustomFields: normalizeCustomFieldIdentifiers(config.complexCustomFields) }; } diff --git a/src/converters/IssueEntityConverter.ts b/src/converters/IssueEntityConverter.ts index 8070617..0464064 100644 --- a/src/converters/IssueEntityConverter.ts +++ b/src/converters/IssueEntityConverter.ts @@ -48,11 +48,22 @@ function getIssueDescription(issue: Issue, apiVersion: string): string { : parseContent((description as TextContent).content); } +function getNestedValue(obj: any, path: string): any { + return path.split('.').reduce((acc, part) => acc && acc[part], obj); +} + +function setFlatNestedValue(obj: any, path: string, value: any): void { + const keys = path.split('.'); + const formattedPath = keys.map(camelCase).join('.'); + obj[formattedPath] = value; +} + export function createIssueEntity({ issue, logger, fieldsById, customFieldsToInclude, + complexCustomFieldsToInclude, requestedClass, redactIssueDescriptions, apiVersion, @@ -61,12 +72,14 @@ export function createIssueEntity({ logger: IntegrationLogger; fieldsById?: { [id: string]: Field }; customFieldsToInclude?: string[]; + complexCustomFieldsToInclude?: string[]; requestedClass?: unknown; redactIssueDescriptions: boolean; apiVersion: string; }): IssueEntity { fieldsById = fieldsById || {}; customFieldsToInclude = customFieldsToInclude || []; + complexCustomFieldsToInclude = complexCustomFieldsToInclude || []; const status = issue.fields.status && issue.fields.status.name; const issueType = issue.fields.issuetype && issue.fields.issuetype.name; @@ -94,6 +107,20 @@ export function createIssueEntity({ } } + // Handle complex custom fields + complexCustomFieldsToInclude.forEach(path => { + const [baseFieldId, ...nestedPathParts] = path.split('.'); + if (issue.fields[baseFieldId] !== undefined) { + const nestedPath = nestedPathParts.join('.'); + const fieldValue = getNestedValue(issue.fields[baseFieldId], nestedPath); + if (fieldValue !== undefined) { + const baseFieldName = camelCase(fieldsById[baseFieldId].name); + const formattedPath = [baseFieldName, ...nestedPathParts].map(camelCase).join('.'); + setFlatNestedValue(customFields, formattedPath, fieldValue); + } + } + }); + if (!['string', 'undefined'].includes(typeof requestedClass)) { logger.warn( { requestedClass }, diff --git a/src/steps/issues.ts b/src/steps/issues.ts index 4441ffe..9cd6c31 100644 --- a/src/steps/issues.ts +++ b/src/steps/issues.ts @@ -79,6 +79,16 @@ export async function fetchIssues({ 'Custom fields to ingest.', ); + logger.info( + { + complexCustomFields: config.complexCustomFields, + allFieldIdsAndNames: Object.values(fieldsById).map( + (field) => `${field.id}: ${field.name ?? 'undefined field name'}`, + ), + }, + 'Complex custom fields to ingest.', + ); + const issueProcessor = async (projectKey: JiraProjectKey, issue: Issue) => processIssue( { @@ -86,6 +96,7 @@ export async function fetchIssues({ jobState, fieldsById, customFieldsToInclude: config.customFields, + complexCustomFieldsToInclude: config.complexCustomFields, projectEntities, redactIssueDescriptions: redactIssueDescriptions ?? false, apiVersion, @@ -134,6 +145,7 @@ type ProcessIssueContext = { logger: IntegrationLogger; fieldsById: IdFieldMap; customFieldsToInclude: string[]; + complexCustomFieldsToInclude: string[]; projectEntities: ProjectEntity[]; redactIssueDescriptions: boolean; apiVersion: string; @@ -153,6 +165,7 @@ async function processIssue( logger, jobState, customFieldsToInclude, + complexCustomFieldsToInclude, fieldsById, projectEntities, redactIssueDescriptions, @@ -168,6 +181,7 @@ async function processIssue( logger, fieldsById, customFieldsToInclude, + complexCustomFieldsToInclude, redactIssueDescriptions, apiVersion, }), From a23392c09d4c4bd5a72283f31c90212a2d55562f Mon Sep 17 00:00:00 2001 From: poornima-metron Date: Thu, 27 Jun 2024 10:37:10 +0530 Subject: [PATCH 2/6] formatting fixed --- src/config.ts | 6 ++++-- src/converters/IssueEntityConverter.ts | 12 ++++++++---- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/config.ts b/src/config.ts index 98b773f..8d1266d 100644 --- a/src/config.ts +++ b/src/config.ts @@ -76,7 +76,7 @@ export const instanceConfigFields: IntegrationInstanceConfigFieldMap = { type: 'json', mask: false, optional: true, - } + }, }; /** @@ -225,7 +225,9 @@ export function buildNormalizedInstanceConfig( apiVersion: jiraApiVersion, projects: normalizeProjectKeys(config.projects), customFields: normalizeCustomFieldIdentifiers(config.customFields), - complexCustomFields: normalizeCustomFieldIdentifiers(config.complexCustomFields) + complexCustomFields: normalizeCustomFieldIdentifiers( + config.complexCustomFields, + ), }; } diff --git a/src/converters/IssueEntityConverter.ts b/src/converters/IssueEntityConverter.ts index 0464064..3c79e6a 100644 --- a/src/converters/IssueEntityConverter.ts +++ b/src/converters/IssueEntityConverter.ts @@ -108,15 +108,19 @@ export function createIssueEntity({ } // Handle complex custom fields - complexCustomFieldsToInclude.forEach(path => { + complexCustomFieldsToInclude.forEach((path) => { const [baseFieldId, ...nestedPathParts] = path.split('.'); if (issue.fields[baseFieldId] !== undefined) { const nestedPath = nestedPathParts.join('.'); const fieldValue = getNestedValue(issue.fields[baseFieldId], nestedPath); if (fieldValue !== undefined) { - const baseFieldName = camelCase(fieldsById[baseFieldId].name); - const formattedPath = [baseFieldName, ...nestedPathParts].map(camelCase).join('.'); - setFlatNestedValue(customFields, formattedPath, fieldValue); + if (fieldsById && fieldsById[baseFieldId]) { + const baseFieldName = camelCase(fieldsById[baseFieldId].name); + const formattedPath = [baseFieldName, ...nestedPathParts] + .map(camelCase) + .join('.'); + setFlatNestedValue(customFields, formattedPath, fieldValue); + } } } }); From 5653ea1fac6a4cd5cb6836ba0d155de3d0ef96fc Mon Sep 17 00:00:00 2001 From: poornima-metron Date: Thu, 27 Jun 2024 17:37:47 +0530 Subject: [PATCH 3/6] fixed bug for custom field property --- src/converters/IssueEntityConverter.ts | 70 ++++++++++++++++---------- 1 file changed, 43 insertions(+), 27 deletions(-) diff --git a/src/converters/IssueEntityConverter.ts b/src/converters/IssueEntityConverter.ts index 3c79e6a..abc4f82 100644 --- a/src/converters/IssueEntityConverter.ts +++ b/src/converters/IssueEntityConverter.ts @@ -49,13 +49,29 @@ function getIssueDescription(issue: Issue, apiVersion: string): string { } function getNestedValue(obj: any, path: string): any { - return path.split('.').reduce((acc, part) => acc && acc[part], obj); + return path.split('.').reduce((acc, part, index) => { + if (!acc) { + return undefined; + } + + const match = part.match(/^(\w+|\[\d+\])(?:\.(.+))?$/); // Updated regex pattern + if (match) { + const [, key, rest] = match; + if (key.startsWith('[') && key.endsWith(']')) { + // Accessing array element + const arrayIndex = Number(key.slice(1, -1)); + return acc[arrayIndex]; + } else { + // Accessing object property + return acc[key]; + } + } + return acc[part]; + }, obj); } function setFlatNestedValue(obj: any, path: string, value: any): void { - const keys = path.split('.'); - const formattedPath = keys.map(camelCase).join('.'); - obj[formattedPath] = value; + obj[path] = value; } export function createIssueEntity({ @@ -85,6 +101,7 @@ export function createIssueEntity({ const issueType = issue.fields.issuetype && issue.fields.issuetype.name; const customFields: { [key: string]: any } = {}; + // Extract simple custom fields for (const [key, value] of Object.entries(issue.fields)) { if ( key.startsWith('customfield_') && @@ -116,9 +133,18 @@ export function createIssueEntity({ if (fieldValue !== undefined) { if (fieldsById && fieldsById[baseFieldId]) { const baseFieldName = camelCase(fieldsById[baseFieldId].name); - const formattedPath = [baseFieldName, ...nestedPathParts] - .map(camelCase) - .join('.'); + const formattedPathParts = nestedPathParts.map((part) => { + const match = part.match(/^(\w+)(?:\[(\d+)\])?$/); + if (match) { + const [, key, index] = match; + if (index !== undefined) { + return `${camelCase(key)}${index}`; + } + return camelCase(key); + } + return camelCase(part); + }); + const formattedPath = [baseFieldName, ...formattedPathParts].join(''); setFlatNestedValue(customFields, formattedPath, fieldValue); } } @@ -132,7 +158,7 @@ export function createIssueEntity({ ); requestedClass = undefined; } - + // Set issue class based on issue type or requested class let issueClass: string | string[]; if (requestedClass) { @@ -162,6 +188,7 @@ export function createIssueEntity({ } } + // Redact issue descriptions if necessary let entityDescription: string; if (redactIssueDescriptions) { if (issue.fields.description) { @@ -169,29 +196,18 @@ export function createIssueEntity({ type: 'text', text: 'REDACTED', }; - } // don't let description leak in rawData + } entityDescription = 'REDACTED'; } else { entityDescription = getIssueDescription(issue, apiVersion); } - //TEMP INT-10020 - if (Object.keys(customFields).length > 0) { - logger.info( - { customFieldsToAdd: Object.keys(customFields) }, - 'Adding custom fields to issue', - ); - } - const entity = { + // Construct issue entity object + const entity: IssueEntity = { _key: generateEntityKey(ISSUE_ENTITY_TYPE, issue.id), _type: ISSUE_ENTITY_TYPE, _class: issueClass, - _rawData: [ - { - name: 'default', - rawData: issue as any, - }, - ], + _rawData: [{ name: 'default', rawData: issue as any }], ...customFields, id: issue.id, key: issue.key, @@ -225,11 +241,11 @@ export function createIssueEntity({ issue.fields.components && issue.fields.components.map((c) => c.name), priority: issue.fields.priority && issue.fields.priority.name, }; + + // Add event data if requested class is provided if (requestedClass) { - entity._rawData.push({ - name: 'event', - rawData: { requestedClass }, - }); + entity._rawData?.push({ name: 'event', rawData: { requestedClass } }); } return entity; } + From 7932a5253cd26571fb931ff1ae10984485889c37 Mon Sep 17 00:00:00 2001 From: poornima-metron Date: Thu, 27 Jun 2024 17:41:04 +0530 Subject: [PATCH 4/6] added formatting --- src/converters/IssueEntityConverter.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/converters/IssueEntityConverter.ts b/src/converters/IssueEntityConverter.ts index abc4f82..f426f3b 100644 --- a/src/converters/IssueEntityConverter.ts +++ b/src/converters/IssueEntityConverter.ts @@ -248,4 +248,3 @@ export function createIssueEntity({ } return entity; } - From a092c6c1c5f327e5f0c4e6b5b1593a6529430442 Mon Sep 17 00:00:00 2001 From: poornima-metron Date: Thu, 27 Jun 2024 17:48:14 +0530 Subject: [PATCH 5/6] removed unwanted --- src/converters/IssueEntityConverter.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/converters/IssueEntityConverter.ts b/src/converters/IssueEntityConverter.ts index f426f3b..aa6cfa6 100644 --- a/src/converters/IssueEntityConverter.ts +++ b/src/converters/IssueEntityConverter.ts @@ -54,9 +54,9 @@ function getNestedValue(obj: any, path: string): any { return undefined; } - const match = part.match(/^(\w+|\[\d+\])(?:\.(.+))?$/); // Updated regex pattern + const match = part.match(/^(\w+|\[\d+\])(?:\.(.+))?$/); if (match) { - const [, key, rest] = match; + const [, key] = match; if (key.startsWith('[') && key.endsWith(']')) { // Accessing array element const arrayIndex = Number(key.slice(1, -1)); From 275ce686de9bcbf1889275133870ff7814e40bc6 Mon Sep 17 00:00:00 2001 From: poornima-metron Date: Thu, 27 Jun 2024 17:51:59 +0530 Subject: [PATCH 6/6] fixed formatting --- src/converters/IssueEntityConverter.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/converters/IssueEntityConverter.ts b/src/converters/IssueEntityConverter.ts index aa6cfa6..7359317 100644 --- a/src/converters/IssueEntityConverter.ts +++ b/src/converters/IssueEntityConverter.ts @@ -54,7 +54,7 @@ function getNestedValue(obj: any, path: string): any { return undefined; } - const match = part.match(/^(\w+|\[\d+\])(?:\.(.+))?$/); + const match = part.match(/^(\w+|\[\d+\])(?:\.(.+))?$/); if (match) { const [, key] = match; if (key.startsWith('[') && key.endsWith(']')) {