Merge pull request #39 from JupiterOne/INT-8363-step-failures

INT-8363 - Upgrading to the latest SDK and handling 401 errors.

.github/workflows/build.yml

@@ -13,7 +13,7 @@ jobs:
         name: Setup Node
         uses: actions/setup-node@v1
         with:
-          node-version: 14.x
+          node-version: 18.x
 
       - name: Check out code repository source code
         uses: actions/checkout@v2
@@ -37,7 +37,7 @@ jobs:
       - name: Setup Node
         uses: actions/setup-node@v1
         with:
-          node-version: 14.x
+          node-version: 18.x
 
       - name: Check out repo
         uses: actions/checkout@v2

package.json

@@ -30,12 +30,12 @@
     "prepush": "yarn lint && yarn type-check && jest --changedSince main"
   },
   "peerDependencies": {
-    "@jupiterone/integration-sdk-core": "^8.0.0"
+    "@jupiterone/integration-sdk-core": "9.8.0"
   },
   "devDependencies": {
-    "@jupiterone/integration-sdk-core": "^8.0.0",
-    "@jupiterone/integration-sdk-dev-tools": "^8.0.0",
-    "@jupiterone/integration-sdk-testing": "^8.0.0"
+    "@jupiterone/integration-sdk-core": "9.8.0",
+    "@jupiterone/integration-sdk-dev-tools": "9.8.0",
+    "@jupiterone/integration-sdk-testing": "9.8.0"
   },
   "dependencies": {
     "node-fetch": "2.6.7"

src/client.ts

@@ -2,6 +2,7 @@ import fetch, { Response } from 'node-fetch';
 import { retry } from '@lifeomic/attempt';
 
 import {
+  IntegrationLogger,
   IntegrationProviderAPIError,
   IntegrationProviderAuthenticationError,
 } from '@jupiterone/integration-sdk-core';
@@ -36,9 +37,12 @@ class ResponseError extends IntegrationProviderAPIError {
 }
 
 export class APIClient {
-  constructor(readonly config: IntegrationConfig) {}
+  constructor(
+    readonly config: IntegrationConfig,
+    readonly logger: IntegrationLogger,
+  ) {}
 
-  private readonly paginateEntitiesPerPage = 30;
+  private readonly paginateEntitiesPerPage = 100;
 
   private authenticationToken: string;
 
@@ -53,9 +57,8 @@ export class APIClient {
   public async initializeAccessToken() {
     const authorizationString =
       this.config.clientId + ':' + this.config.clientSecret;
-    const authorizationEncoded = Buffer.from(authorizationString).toString(
-      'base64',
-    );
+    const authorizationEncoded =
+      Buffer.from(authorizationString).toString('base64');
 
     const result = await retry(
       async () => {
@@ -96,6 +99,10 @@ export class APIClient {
     this.authenticationToken = body.access_token;
   }
 
+  private isErrorRetryable(status: number) {
+    return status === 401 || status === 429;
+  }
+
   private async request(
     uri: string,
     method: 'GET' | 'HEAD' = 'GET',
@@ -123,10 +130,23 @@ export class APIClient {
           delay: 1000,
           factor: 2,
           maxAttempts: 10,
-          handleError: (err, context) => {
+          handleError: async (err, context) => {
             const rateLimitType = err.response.headers.get('X-RateLimit-Type');
-            // only retry on 429 && per second limit
-            if (!(err.status === 429 && rateLimitType === 'QPS')) {
+            if (!this.isErrorRetryable(err.status)) {
+              context.abort();
+            }
+
+            if (err.status === 401) {
+              this.logger.info(
+                'Received a 401 error.  Requesting a new token.',
+              );
+              await this.initializeAccessToken();
+            } else if (err.status === 429 && rateLimitType === 'QPS') {
+              const retryAfter = err.response.headers.get('Retry-After');
+              this.logger.warn(
+                { retryAfter },
+                'Received a daily rate limit error.',
+              );
               context.abort();
             }
           },
@@ -358,6 +378,9 @@ export class APIClient {
   }
 }
 
-export function createAPIClient(config: IntegrationConfig): APIClient {
-  return new APIClient(config);
+export function createAPIClient(
+  config: IntegrationConfig,
+  logger: IntegrationLogger,
+): APIClient {
+  return new APIClient(config, logger);
 }

src/config.ts

@@ -47,6 +47,6 @@ export async function validateInvocation(
     );
   }
 
-  const apiClient = await getOrCreateAPIClient(config);
+  const apiClient = await getOrCreateAPIClient(config, context.logger);
   await apiClient.verifyAuthentication();
 }

src/getOrCreateAPIClient.ts

@@ -1,14 +1,18 @@
 import { IntegrationConfig } from './config';
 import { APIClient } from './client';
-import { IntegrationProviderAuthenticationError } from '@jupiterone/integration-sdk-core';
+import {
+  IntegrationLogger,
+  IntegrationProviderAuthenticationError,
+} from '@jupiterone/integration-sdk-core';
 
 let client: APIClient;
 
 export default async function getOrCreateAPIClient(
   config: IntegrationConfig,
+  logger: IntegrationLogger,
 ): Promise<APIClient> {
   if (!client) {
-    client = new APIClient(config);
+    client = new APIClient(config, logger);
     try {
       await client.initializeAccessToken();
     } catch (err) {

src/index.ts

@@ -7,9 +7,10 @@ import {
   instanceConfigFields,
 } from './config';
 
-export const invocationConfig: IntegrationInvocationConfig<IntegrationConfig> = {
-  instanceConfigFields,
-  validateInvocation,
-  getStepStartStates,
-  integrationSteps,
-};
+export const invocationConfig: IntegrationInvocationConfig<IntegrationConfig> =
+  {
+    instanceConfigFields,
+    validateInvocation,
+    getStepStartStates,
+    integrationSteps,
+  };