Skip to content
This repository has been archived by the owner on Sep 3, 2024. It is now read-only.

Make servicenow faster - more resilient #51

Merged
merged 3 commits into from
Jun 18, 2024
Merged

Make servicenow faster - more resilient #51

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
6a0fdc1
Servicenow Upgrades
Gonzalo-Avalos-Ribas Jun 18, 2024
efc0b09
Correct snapshot
Gonzalo-Avalos-Ribas Jun 18, 2024
4ea1020
Removed optional chaining from relationships
Gonzalo-Avalos-Ribas Jun 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 4 additions & 23 deletions src/client.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,20 +122,6 @@ export class ServiceNowClient {
const response = await this.request({ url });
const result = response?.data?.result;

const redactedResponse = {
Copy link
Contributor Author

@Gonzalo-Avalos-Ribas Gonzalo-Avalos-Ribas Jun 18, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Removed log

status: response.status,
statusText: response.statusText,
headers: response.headers,
responseLength: response.data?.['length'],
responseType: typeof response.data,
};
this.logger.info(
{
redactedResponse,
},
'Redacted response log',
);

if (Array.isArray(result)) {
const nextLink = getServiceNowNextLink(response?.headers?.link);
return { result, nextLink };
Expand Down Expand Up @@ -206,18 +192,13 @@ export class ServiceNowClient {
>(url);

// For some reason Servicenow API sometimes have string responses for paginated endpoints
if (paginatedResponse?.result && Array.isArray(paginatedResponse.result)) {
if (
paginatedResponse?.result &&
Array.isArray(paginatedResponse.result)
) {
for (const r of paginatedResponse.result) {
await callback(r);
}

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Removed log

this.logger.info(
{
resourceCount: paginatedResponse.result.length,
resource: url,
},
'Received resources for endpoint',
);
}
url = paginatedResponse?.nextLink;
} while (url);
Expand Down
7 changes: 3 additions & 4 deletions src/constants.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,6 @@ export const Steps = {
GROUP_MEMBERS: 'step-group-members',
INCIDENTS: 'step-incidents',
CMDB: 'step-cmdb',
USER_OWNS_CMDB: 'step_user_owns_cmdb',
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Merged this 4 steps into the cmdb step to make the whole process faster

USER_MANAGES_CMDB: 'step_user_manages_cmdb',
GROUP_MANAGES_CMDB: 'step_group_manages_cmdb',
CMDB_ASSIGNED_USER: 'step_cmdb_assigned_user',
};

export const Entities = {
Expand Down Expand Up @@ -49,6 +45,9 @@ export const Entities = {
_type: 'service_now_cmdb_object',
_class: 'Configuration',
disableClassMatch: true,
indexMetadata: {
enabled: false,
},
},
};

Expand Down
1 change: 1 addition & 0 deletions src/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,5 +12,6 @@ export const invocationConfig: IntegrationInvocationConfig<IntegrationConfig> =
validateInvocation,
getStepStartStates,
integrationSteps: integrationSteps,
dependencyGraphOrder: ['last'],
ingestionConfig,
};
2 changes: 1 addition & 1 deletion src/steps/__snapshots__/converters.test.ts.snap
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -153,7 +153,7 @@ exports[`createUserEntity 1`] = `
"_type": "service_now_user",
"active": true,
"createdOn": 1600711037000,
"displayName": "J1 Administrator",
"displayName": "j1-administrator",
"email": "[email protected]",
"id": "5ccbba87db1310109d87a9954b9619db",
"name": "J1 Administrator",
Expand Down
2,151 changes: 2,019 additions & 132 deletions src/steps/cmdb/__recordings__/step-cmdb_3257228988/recording.har
View file
Open in desktop

Large diffs are not rendered by default.

2,319 changes: 0 additions & 2,319 deletions src/steps/cmdb/__recordings__/step_cmdb_assigned_user_1069455055/recording.har
View file
Open in desktop

This file was deleted.

2,100 changes: 0 additions & 2,100 deletions src/steps/cmdb/__recordings__/step_group_manages_cmdb_713613659/recording.har
View file
Open in desktop

This file was deleted.

2,319 changes: 0 additions & 2,319 deletions src/steps/cmdb/__recordings__/step_user_manages_cmdb_4189544667/recording.har
View file
Open in desktop

This file was deleted.

2,319 changes: 0 additions & 2,319 deletions src/steps/cmdb/__recordings__/step_user_owns_cmdb_3809520340/recording.har
View file
Open in desktop

This file was deleted.

69 changes: 4 additions & 65 deletions src/steps/cmdb/index.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,72 +23,11 @@ test(
directory: __dirname,
});

const stepResults = await executeStepWithDependencies(stepTestConfig);
expect(stepResults).toMatchStepMetadata(stepTestConfig);
},
10_000,
);

test(
Steps.CMDB_ASSIGNED_USER,
async () => {
const stepTestConfig = getStepTestConfigForStep(Steps.CMDB_ASSIGNED_USER);

recording = setupServiceNowRecording({
name: Steps.CMDB_ASSIGNED_USER,
directory: __dirname,
});

const stepResults = await executeStepWithDependencies(stepTestConfig);
expect(stepResults).toMatchStepMetadata(stepTestConfig);
},
10_000,
);

test(
Steps.USER_OWNS_CMDB,
async () => {
const stepTestConfig = getStepTestConfigForStep(Steps.USER_OWNS_CMDB);

recording = setupServiceNowRecording({
name: Steps.USER_OWNS_CMDB,
directory: __dirname,
const stepResults = await executeStepWithDependencies({
...stepTestConfig,
dependencyStepIds: [Steps.USERS, Steps.GROUPS],
});

const stepResults = await executeStepWithDependencies(stepTestConfig);
expect(stepResults).toMatchStepMetadata(stepTestConfig);
},
10_000,
);

test(
Steps.USER_MANAGES_CMDB,
async () => {
const stepTestConfig = getStepTestConfigForStep(Steps.USER_MANAGES_CMDB);

recording = setupServiceNowRecording({
name: Steps.USER_MANAGES_CMDB,
directory: __dirname,
});

const stepResults = await executeStepWithDependencies(stepTestConfig);
expect(stepResults).toMatchStepMetadata(stepTestConfig);
},
10_000,
);

test(
Steps.GROUP_MANAGES_CMDB,
async () => {
const stepTestConfig = getStepTestConfigForStep(Steps.GROUP_MANAGES_CMDB);

recording = setupServiceNowRecording({
name: Steps.GROUP_MANAGES_CMDB,
directory: __dirname,
});

const stepResults = await executeStepWithDependencies(stepTestConfig);
expect(stepResults).toMatchStepMetadata(stepTestConfig);
},
10_000,
100_000,
);
205 changes: 71 additions & 134 deletions src/steps/cmdb/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,113 +35,80 @@ export async function fetchCMDB(

await client.iterateTableResources({
table: parent,
limit: 3_000,
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Default for table api is 10_000 so the api is really good at handling large amounts of data. I bumped by x6 to be careful with our memory usage

callback: async (resource: CMDBItem) => {
const sysClassNames = [
resource.sys_class_name,
...(await getAllParents(client, resource.sys_class_name, logger)),
];
if (!jobState.hasKey(resource.sys_id)) {
await jobState.addEntity(createCMDBEntity(resource, sysClassNames));
const cmdbEntity = createCMDBEntity(resource, sysClassNames);
await jobState.addEntity(cmdbEntity);

if (
cmdbEntity.managedBy &&
jobState.hasKey(cmdbEntity.managedBy as string)
) {
await jobState.addRelationship(
createDirectRelationship({
_class: RelationshipClass.MANAGES,
fromKey: cmdbEntity.managedBy as string,
fromType: Entities.USER._type,
toKey: cmdbEntity._key,
toType: cmdbEntity._type,
}),
);
}

if (
cmdbEntity.ownedBy &&
jobState.hasKey(cmdbEntity.ownedBy as string)
) {
await jobState.addRelationship(
createDirectRelationship({
_class: RelationshipClass.OWNS,
fromKey: cmdbEntity.ownedBy as string,
fromType: Entities.USER._type,
toKey: cmdbEntity._key,
toType: cmdbEntity._type,
}),
);
}

if (
cmdbEntity.managedByGroup &&
jobState.hasKey(cmdbEntity.managedByGroup as string)
) {
await jobState.addRelationship(
createDirectRelationship({
_class: RelationshipClass.MANAGES,
fromKey: cmdbEntity.managedByGroup as string,
fromType: Entities.GROUP._type,
toKey: cmdbEntity._key,
toType: cmdbEntity._type,
}),
);
}

if (
cmdbEntity.assignedTo &&
jobState.hasKey(cmdbEntity.assignedTo as string)
) {
await jobState.addRelationship(
createDirectRelationship({
_class: RelationshipClass.ASSIGNED,
toKey: cmdbEntity.assignedTo as string,
toType: Entities.USER._type,
fromKey: cmdbEntity._key,
fromType: cmdbEntity._type,
}),
);
}
}
},
});
}
export async function buildUserManagesCMDB(
context: IntegrationStepExecutionContext<IntegrationConfig>,
) {
const { jobState } = context;
await jobState.iterateEntities(
{ _type: Entities.CMDB_OBJECT._type },
async (cmdbEntity) => {
const userId = (cmdbEntity as any).managedBy;
if (!userId) {
return;
}
const userEntity = await jobState.findEntity(userId);
if (userEntity) {
await jobState.addRelationship(
createDirectRelationship({
_class: RelationshipClass.MANAGES,
from: userEntity,
to: cmdbEntity,
}),
);
}
},
);
}
export async function buildUserOwnsCMDB(
context: IntegrationStepExecutionContext<IntegrationConfig>,
) {
const { jobState } = context;
await jobState.iterateEntities(
{ _type: Entities.CMDB_OBJECT._type },
async (cmdbEntity) => {
const userId = (cmdbEntity as any).ownedBy;
if (!userId) {
return;
}
const userEntity = await jobState.findEntity(userId);
if (userEntity) {
await jobState.addRelationship(
createDirectRelationship({
_class: RelationshipClass.OWNS,
from: userEntity,
to: cmdbEntity,
}),
);
}
},
);
}
export async function buildGroupManagesCMDB(
context: IntegrationStepExecutionContext<IntegrationConfig>,
) {
const { jobState } = context;
await jobState.iterateEntities(
{ _type: Entities.CMDB_OBJECT._type },
async (cmdbEntity) => {
const groupId = (cmdbEntity as any).managedByGroup;
if (!groupId) {
return;
}
const groupEntity = await jobState.findEntity(groupId);
if (groupEntity) {
await jobState.addRelationship(
createDirectRelationship({
_class: RelationshipClass.MANAGES,
from: groupEntity,
to: cmdbEntity,
}),
);
}
},
);
}
export async function buildCMDBAssignedUser(
context: IntegrationStepExecutionContext<IntegrationConfig>,
) {
const { jobState } = context;
await jobState.iterateEntities(
{ _type: Entities.CMDB_OBJECT._type },
async (cmdbEntity) => {
const userId = (cmdbEntity as any).assignedTo;
if (!userId) {
return;
}
const userEntity = await jobState.findEntity(userId);
if (userEntity) {
await jobState.addRelationship(
createDirectRelationship({
_class: RelationshipClass.ASSIGNED,
from: cmdbEntity,
to: userEntity,
}),
);
}
},
);
}

async function getAllParents(
client: ServiceNowClient,
sysClassName: string,
Expand Down Expand Up @@ -198,44 +165,14 @@ export const cmdbIntegrationSteps: Step<
ingestionSourceId: IngestionSources.CMDB_ITEMS,
name: 'CMDB Items',
entities: [Entities.CMDB_OBJECT],
relationships: [],
relationships: [
Relationships.USER_MANAGES_CMDB,
Relationships.USER_OWNS_CMDB,
Relationships.CMDB_ASSIGNED_TO_USER,
Relationships.GROUP_MANAGES_CMDB,
],
dependsOn: [],
dependencyGraphId: 'last',
executionHandler: fetchCMDB,
},
{
id: Steps.USER_MANAGES_CMDB,
ingestionSourceId: IngestionSources.CMDB_ITEMS,
name: 'Build user manages CMDB',
entities: [],
relationships: [Relationships.USER_MANAGES_CMDB],
dependsOn: [Steps.CMDB, Steps.USERS],
executionHandler: buildUserManagesCMDB,
},
{
id: Steps.USER_OWNS_CMDB,
ingestionSourceId: IngestionSources.CMDB_ITEMS,
name: 'Build user owns CMDB',
entities: [],
relationships: [Relationships.USER_OWNS_CMDB],
dependsOn: [Steps.CMDB, Steps.USERS],
executionHandler: buildUserOwnsCMDB,
},
{
id: Steps.CMDB_ASSIGNED_USER,
ingestionSourceId: IngestionSources.CMDB_ITEMS,
name: 'CMDB assigned to user',
entities: [],
relationships: [Relationships.CMDB_ASSIGNED_TO_USER],
dependsOn: [Steps.CMDB, Steps.USERS],
executionHandler: buildCMDBAssignedUser,
},
{
id: Steps.GROUP_MANAGES_CMDB,
ingestionSourceId: IngestionSources.CMDB_ITEMS,
name: 'Group manages CMDB',
entities: [],
relationships: [Relationships.GROUP_MANAGES_CMDB],
dependsOn: [Steps.CMDB, Steps.GROUPS],
executionHandler: buildGroupManagesCMDB,
},
];
Loading
Loading