Merge pull request #5 from creativice/feature/update-classes-and-pagi…

…nation

Updates classes and adds pagination (#4)

.github/workflows/build.yml

@@ -8,7 +8,7 @@ jobs:
       fail-fast: false
       matrix:
         node-version: [12.x]
-        os: [ubuntu-latest, macos-latest, windows-latest]
+        os: [ubuntu-latest, macos-latest]
 
     steps:
       - id: setup-node

docs/jupiterone.md

@@ -77,11 +77,11 @@ The following entities are created:
 | Resources             | Entity `_type`    | Entity `_class` |
 | --------------------- | ----------------- | --------------- |
 | Kubernetes Container  | `kube_container`  | `Container`     |
-| Kubernetes Deployment | `kube_deployment` | `Configuration` |
+| Kubernetes Deployment | `kube_deployment` | `Deployment`    |
 | Kubernetes Namespace  | `kube_namespace`  | `Group`         |
-| Kubernetes Node       | `kube_node`       | `Group`         |
+| Kubernetes Node       | `kube_node`       | `Host`          |
 | Kubernetes Pod        | `kube_pod`        | `Task`          |
-| Kubernetes ReplicaSet | `kube_replicaset` | `Configuration` |
+| Kubernetes ReplicaSet | `kube_replicaset` | `Deployment`    |
 | Kubernetes Service    | `kube_service`    | `Service`       |
 
 ### Relationships
@@ -90,12 +90,12 @@ The following relationships are created/mapped:
 
 | Source Entity `_type` | Relationship `_class` | Target Entity `_type` |
 | --------------------- | --------------------- | --------------------- |
-| `kube_deployment`     | **HAS**               | `kube_replicaset`     |
-| `kube_namespace`      | **HAS**               | `kube_deployment`     |
-| `kube_namespace`      | **HAS**               | `kube_service`        |
+| `kube_deployment`     | **MANAGES**           | `kube_replicaset`     |
+| `kube_namespace`      | **CONTAINS**          | `kube_deployment`     |
+| `kube_namespace`      | **CONTAINS**          | `kube_service`        |
 | `kube_node`           | **HAS**               | `kube_pod`            |
-| `kube_pod`            | **HAS**               | `kube_container`      |
-| `kube_replicaset`     | **HAS**               | `kube_pod`            |
+| `kube_pod`            | **CONTAINS**          | `kube_container`      |
+| `kube_replicaset`     | **MANAGES**           | `kube_pod`            |
 
 <!--
 ********************************************************************************

package.json

@@ -19,10 +19,10 @@
     "lint": "eslint . --cache --fix --ext .ts,.tsx",
     "format": "prettier --write '**/*.{ts,js,json,css,md,yml}'",
     "type-check": "tsc",
-    "test": "jest",
+    "test": "KUBECONFIG=$(pwd)/test/.kube/config jest",
     "test:ci": "yarn lint && yarn type-check && yarn test",
     "build": "tsc -p tsconfig.dist.json --declaration",
-    "prepush": "yarn lint && yarn type-check && jest --changedSince master",
+    "prepush": "yarn lint && yarn type-check && yarn test --changedSince master",
     "prepack": "yarn build"
   },
   "devDependencies": {

src/kubernetes/client.ts

@@ -5,15 +5,29 @@ export interface ClientOptions {
   config: IntegrationConfig;
 }
 
-export abstract class Client {
+export class Client {
   public kubeConfig: k8s.KubeConfig;
   public config: IntegrationConfig;
+  protected readonly maxPerPage = 50;
 
   constructor(config: IntegrationConfig) {
     this.config = config;
     this.kubeConfig = new k8s.KubeConfig();
   }
 
+  async iterateApi(
+    fn: (nextPageToken?: string) => Promise<any>,
+    callback: (data: any) => Promise<void>,
+  ) {
+    let nextPageToken: string | undefined;
+
+    do {
+      const result = await fn(nextPageToken);
+      nextPageToken = result.body.metadata._continue;
+      await callback(result.body);
+    } while (nextPageToken);
+  }
+
   authenticate() {
     if (this.config.isRunningTest) {
       this.kubeConfig.loadFromDefault();

src/kubernetes/clients/apps.ts

@@ -1,4 +1,5 @@
 import * as k8s from '@kubernetes/client-node';
+import { V1DeploymentList, V1ReplicaSetList } from '@kubernetes/client-node';
 import { IntegrationConfig } from '../../config';
 import { Client } from '../client';
 
@@ -16,21 +17,47 @@ export class AppsClient extends Client {
     namespace: string,
     callback: (data: k8s.V1Deployment) => Promise<void>,
   ): Promise<void> {
-    const resp = await this.client.listNamespacedDeployment(namespace);
-
-    for (const deployment of resp.body.items || []) {
-      await callback(deployment);
-    }
+    await this.iterateApi(
+      async (nextPageToken) => {
+        return this.client.listNamespacedDeployment(
+          namespace,
+          undefined,
+          undefined,
+          nextPageToken,
+          undefined,
+          undefined,
+          this.maxPerPage,
+        );
+      },
+      async (data: V1DeploymentList) => {
+        for (const deployment of data.items || []) {
+          await callback(deployment);
+        }
+      },
+    );
   }
 
   async iterateReplicaSets(
     namespace: string,
     callback: (data: k8s.V1ReplicaSet) => Promise<void>,
   ): Promise<void> {
-    const resp = await this.client.listNamespacedReplicaSet(namespace);
-
-    for (const replicaset of resp.body.items || []) {
-      await callback(replicaset);
-    }
+    await this.iterateApi(
+      async (nextPageToken) => {
+        return this.client.listNamespacedReplicaSet(
+          namespace,
+          undefined,
+          undefined,
+          nextPageToken,
+          undefined,
+          undefined,
+          this.maxPerPage,
+        );
+      },
+      async (data: V1ReplicaSetList) => {
+        for (const replicaSet of data.items || []) {
+          await callback(replicaSet);
+        }
+      },
+    );
   }
 }

src/kubernetes/clients/core.ts

@@ -1,5 +1,11 @@
 import { IntegrationProviderAuthenticationError } from '@jupiterone/integration-sdk-core';
 import * as k8s from '@kubernetes/client-node';
+import {
+  V1NamespaceList,
+  V1NodeList,
+  V1PodList,
+  V1ServiceList,
+} from '@kubernetes/client-node';
 import { IntegrationConfig } from '../../config';
 import { Client } from '../client';
 
@@ -31,50 +37,93 @@ export class CoreClient extends Client {
       const resp = await this.client.readNamespace(namespace as string);
       await callback(resp.body);
     } else {
-      const resp = await this.client.listNamespace();
-
-      for (const namespace of resp.body.items || []) {
-        await callback(namespace);
-      }
+      await this.iterateApi(
+        async (nextPageToken) => {
+          return this.client.listNamespace(
+            undefined,
+            undefined,
+            nextPageToken,
+            undefined,
+            undefined,
+            this.maxPerPage,
+          );
+        },
+        async (data: V1NamespaceList) => {
+          for (const namespace of data.items || []) {
+            await callback(namespace);
+          }
+        },
+      );
     }
   }
 
   async iterateNamespacedPods(
     namespace: string,
     callback: (data: k8s.V1Pod) => Promise<void>,
   ): Promise<void> {
-    const resp = await this.client.listNamespacedPod(namespace);
-
-    for (const pod of resp.body.items || []) {
-      await callback(pod);
-    }
+    await this.iterateApi(
+      async (nextPageToken) => {
+        return this.client.listNamespacedPod(
+          namespace,
+          undefined,
+          undefined,
+          nextPageToken,
+          undefined,
+          undefined,
+          this.maxPerPage,
+        );
+      },
+      async (data: V1PodList) => {
+        for (const pod of data.items) {
+          await callback(pod);
+        }
+      },
+    );
   }
 
   async iterateNodes(
     callback: (data: k8s.V1Node) => Promise<void>,
   ): Promise<void> {
-    const resp = await this.client.listNode();
-    for (const node of resp.body.items || []) {
-      await callback(node);
-    }
-  }
-
-  async iterateServices(
-    callback: (data: k8s.V1Service) => Promise<void>,
-  ): Promise<void> {
-    const resp = await this.client.listServiceForAllNamespaces();
-    for (const service of resp.body.items || []) {
-      await callback(service);
-    }
+    await this.iterateApi(
+      async (nextPageToken) => {
+        return this.client.listNode(
+          undefined,
+          undefined,
+          nextPageToken,
+          undefined,
+          undefined,
+          this.maxPerPage,
+        );
+      },
+      async (data: V1NodeList) => {
+        for (const node of data.items) {
+          await callback(node);
+        }
+      },
+    );
   }
 
   async iterateNamespacedServices(
     namespace: string,
     callback: (data: k8s.V1Service) => Promise<void>,
   ): Promise<void> {
-    const resp = await this.client.listNamespacedService(namespace);
-    for (const service of resp.body.items || []) {
-      await callback(service);
-    }
+    await this.iterateApi(
+      async (nextPageToken) => {
+        return this.client.listNamespacedService(
+          namespace,
+          undefined,
+          undefined,
+          nextPageToken,
+          undefined,
+          undefined,
+          this.maxPerPage,
+        );
+      },
+      async (data: V1ServiceList) => {
+        for (const service of data.items) {
+          await callback(service);
+        }
+      },
+    );
   }
 }

src/steps/constants.ts

@@ -30,17 +30,17 @@ export const Entities: Record<
   },
   DEPLOYMENT: {
     _type: 'kube_deployment',
-    _class: ['Configuration'],
+    _class: ['Deployment'],
     resourceName: 'Kubernetes Deployment',
   },
   REPLICASET: {
     _type: 'kube_replicaset',
-    _class: ['Configuration'],
+    _class: ['Deployment'],
     resourceName: 'Kubernetes ReplicaSet',
   },
   NODE: {
     _type: 'kube_node',
-    _class: ['Group'],
+    _class: ['Host'],
     resourceName: 'Kubernetes Node',
   },
   POD: {
@@ -61,42 +61,42 @@ export const Entities: Record<
 };
 
 export const Relationships: Record<
-  | 'NAMESPACE_HAS_POD'
-  | 'NAMESPACE_HAS_SERVICE'
-  | 'NAMESPACE_HAS_DEPLOYMENT'
-  | 'DEPLOYMENT_HAS_REPLICASET'
-  | 'REPLICASET_HAS_POD'
+  | 'NAMESPACE_CONTAINS_POD'
+  | 'NAMESPACE_CONTAINS_SERVICE'
+  | 'NAMESPACE_CONTAINS_DEPLOYMENT'
+  | 'DEPLOYMENT_MANAGES_REPLICASET'
+  | 'REPLICASET_MANAGES_POD'
   | 'NODE_HAS_POD'
-  | 'POD_HAS_CONTAINER',
+  | 'POD_CONTAINS_CONTAINER',
   StepRelationshipMetadata
 > = {
-  NAMESPACE_HAS_POD: {
-    _type: 'kube_namespace_has_pod',
-    _class: RelationshipClass.HAS,
+  NAMESPACE_CONTAINS_POD: {
+    _type: 'kube_namespace_contains_pod',
+    _class: RelationshipClass.CONTAINS,
     sourceType: Entities.NAMESPACE._type,
     targetType: Entities.POD._type,
   },
-  NAMESPACE_HAS_SERVICE: {
-    _type: 'kube_namespace_has_service',
-    _class: RelationshipClass.HAS,
+  NAMESPACE_CONTAINS_SERVICE: {
+    _type: 'kube_namespace_contains_service',
+    _class: RelationshipClass.CONTAINS,
     sourceType: Entities.NAMESPACE._type,
     targetType: Entities.SERVICE._type,
   },
-  NAMESPACE_HAS_DEPLOYMENT: {
-    _type: 'kube_namespace_has_deployment',
-    _class: RelationshipClass.HAS,
+  NAMESPACE_CONTAINS_DEPLOYMENT: {
+    _type: 'kube_namespace_contains_deployment',
+    _class: RelationshipClass.CONTAINS,
     sourceType: Entities.NAMESPACE._type,
     targetType: Entities.DEPLOYMENT._type,
   },
-  DEPLOYMENT_HAS_REPLICASET: {
-    _type: 'kube_deployment_has_replicaset',
-    _class: RelationshipClass.HAS,
+  DEPLOYMENT_MANAGES_REPLICASET: {
+    _type: 'kube_deployment_manages_replicaset',
+    _class: RelationshipClass.MANAGES,
     sourceType: Entities.DEPLOYMENT._type,
     targetType: Entities.REPLICASET._type,
   },
-  REPLICASET_HAS_POD: {
-    _type: 'kube_replicaset_has_pod',
-    _class: RelationshipClass.HAS,
+  REPLICASET_MANAGES_POD: {
+    _type: 'kube_replicaset_manages_pod',
+    _class: RelationshipClass.MANAGES,
     sourceType: Entities.REPLICASET._type,
     targetType: Entities.POD._type,
   },
@@ -106,9 +106,9 @@ export const Relationships: Record<
     sourceType: Entities.NODE._type,
     targetType: Entities.POD._type,
   },
-  POD_HAS_CONTAINER: {
-    _type: 'kube_pod_has_container',
-    _class: RelationshipClass.HAS,
+  POD_CONTAINS_CONTAINER: {
+    _type: 'kube_pod_contains_container',
+    _class: RelationshipClass.CONTAINS,
     sourceType: Entities.POD._type,
     targetType: Entities.CONTAINER._type,
   },