Fix team members and team repos ingestion; filter batched pull reques…

…ts (#286)
JupiterOne-Archives · Nov 21, 2023 · 6df56e2 · 6df56e2
1 parent ad7c0bf
commit 6df56e2
Show file tree

Hide file tree

Showing 6 changed files with 26 additions and 4 deletions.
diff --git a/src/client.ts b/src/client.ts
@@ -699,6 +699,7 @@ export class APIClient {
 
   public async iterateBatchedPullRequests(
     repoIds: string[],
+    ingestStartDatetime: string,
     iteratee: ResourceIteratee<PullRequestResponse>,
   ): Promise<void> {
     if (!this.graphQLClient) {
@@ -707,6 +708,7 @@ export class APIClient {
     const rateLimit =
       await this.graphQLClient.iterateBatchedPullRequestEntities(
         repoIds,
+        ingestStartDatetime,
         iteratee,
       );
     this.logger.debug(

diff --git a/src/client/GraphQLClient/client.ts b/src/client/GraphQLClient/client.ts
@@ -256,6 +256,7 @@ export class GitHubGraphQLClient {
 
   public async iterateBatchedPullRequests(
     repoIds: string[],
+    ingestStartDatetime: string,
     iteratee: ResourceIteratee<PullRequestResponse>,
   ): Promise<RateLimitStepSummary> {
     const executor = createQueryExecutor(this, this.logger);
@@ -264,6 +265,7 @@ export class GitHubGraphQLClient {
       await BatchedPullRequestsQuery.iteratePullRequests(
         {
           repoIds,
+          ingestStartDatetime,
         },
         executor,
         iteratee,

diff --git a/src/client/GraphQLClient/pullRequestQueries/BatchedPullRequestsQuery.ts b/src/client/GraphQLClient/pullRequestQueries/BatchedPullRequestsQuery.ts
@@ -13,6 +13,7 @@ type QueryState = BaseQueryState;
 
 type QueryParams = {
   repoIds: string[];
+  ingestStartDatetime: string;
 };
 
 const buildQuery: BuildQuery<QueryParams, QueryState> = (
@@ -91,7 +92,15 @@ const iteratePullRequests = async (
   let queryState: QueryState = {};
   const executable = buildQuery(queryParams, queryState);
   const response = await execute(executable);
-  queryState = await processResponseData(response, iteratee);
+  const filterIteratee = async (pullRequest: PullRequestResponse) => {
+    if (
+      new Date(pullRequest.updatedAt) >=
+      new Date(queryParams.ingestStartDatetime)
+    ) {
+      await iteratee(pullRequest);
+    }
+  };
+  queryState = await processResponseData(response, filterIteratee);
 
   const queryCost = queryState?.rateLimit?.cost ?? 0;
 

diff --git a/src/client/OrganizationAccountClient.ts b/src/client/OrganizationAccountClient.ts
@@ -297,13 +297,18 @@ export default class OrganizationAccountClient {
 
   async iterateBatchedPullRequestEntities(
     repoIds: string[],
+    ingestStartDatetime: string, //expect Date.toISOString format
     iteratee: ResourceIteratee<PullRequestResponse>,
   ): Promise<RateLimitStepSummary> {
     if (!this.authorizedForPullRequests) {
       this.logger.info('Account not authorized for ingesting pull requests.');
       return { totalCost: 0 };
     }
-    return await this.v4.iterateBatchedPullRequests(repoIds, iteratee);
+    return await this.v4.iterateBatchedPullRequests(
+      repoIds,
+      ingestStartDatetime,
+      iteratee,
+    );
   }
 
   /**

diff --git a/src/steps/pullRequests.ts b/src/steps/pullRequests.ts
@@ -517,7 +517,11 @@ export async function fetchPrs(
     totalConnectionsById: pullRequestsTotalByRepo,
     threshold: 25,
     batchCb: async (repoKeys) => {
-      await apiClient.iterateBatchedPullRequests(repoKeys, iteratee);
+      await apiClient.iterateBatchedPullRequests(
+        repoKeys,
+        ingestStartDatetime,
+        iteratee,
+      );
     },
     singleCb: async (repoKey) => {
       const repoData = repoTags.get(repoKey);

diff --git a/src/steps/teams.ts b/src/steps/teams.ts
@@ -53,7 +53,7 @@ export async function fetchTeams({
       }),
     );
 
-    teamDataMap.set(teamEntity._key, { name: team.name });
+    teamDataMap.set(teamEntity._key, { name: team.slug });
     if (team.repositories.totalCount) {
       repositoriesTotalByTeam.set(
         teamEntity._key,