Skip to content

Commit

Permalink
[Synthtrace][Inventory] K8s entities support (#197077)
Browse files Browse the repository at this point in the history
Part of #196155

New Synthtrace scenario created:
[k8s_entities.ts](https://github.com/elastic/kibana/pull/197077/files#diff-33a935a5fb8848d743e40d89b018ed8ac82ae992c0778cd6cea072d091aa0647)

```
node scripts/synthtrace k8s_entities.ts --clean --live
```

Cluster sample data:
```
 {
        "_index": ".entities.v1.latest.builtin_kubernetes_cluster_ecss_from_ecs_data",
        "_id": "2060900000000000",
        "_score": 1,
        "_source": {
          "entity": {
            "type": "kubernetes_cluster_ecs",
            "id": "2060900000000000",
            "definitionId": "builtin_kubernetes_cluster_ecs",
            "displayName": "cluster_foo",
            "lastSeenTimestamp": "2024-10-21T16:15:17.570Z"
          },
          "orchestrator": {
            "cluster": {
              "name": "cluster_foo"
            }
          },
          "event": {
            "ingested": "2024-10-21T16:15:17.570Z"
          }
        }
      },
      {
        "_index": ".entities.v1.latest.builtin_kubernetes_cluster_semconvs_from_ecs_data",
        "_id": "2060900000000000",
        "_score": 1,
        "_source": {
          "entity": {
            "type": "kubernetes_cluster_semconv",
            "id": "2060900000000000",
            "definitionId": "builtin_kubernetes_cluster_semconv",
            "displayName": "cluster_foo",
            "lastSeenTimestamp": "2024-10-21T16:15:17.570Z"
          },
          "k8s": {
            "cluster": {
              "uid": "cluster_foo"
            }
          },
          "event": {
            "ingested": "2024-10-21T16:15:17.570Z"
          }
        }
      },
```

(cherry picked from commit 5e40320)
  • Loading branch information
cauemarcondes committed Oct 22, 2024
1 parent 5ececbb commit 5d8ecc0
Show file tree
Hide file tree
Showing 14 changed files with 714 additions and 2 deletions.
29 changes: 28 additions & 1 deletion packages/kbn-apm-synthtrace-client/src/lib/entities/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,19 @@ import { Fields } from '../entity';
import { serviceEntity } from './service_entity';
import { hostEntity } from './host_entity';
import { containerEntity } from './container_entity';
import { k8sClusterJobEntity } from './kubernetes/cluster_entity';
import { k8sCronJobEntity } from './kubernetes/cron_job_entity';
import { k8sDaemonSetEntity } from './kubernetes/daemon_set_entity';
import { k8sDeploymentEntity } from './kubernetes/deployment_entity';
import { k8sJobSetEntity } from './kubernetes/job_set_entity';
import { k8sNodeEntity } from './kubernetes/node_entity';
import { k8sPodEntity } from './kubernetes/pod_entity';
import { k8sReplicaSetEntity } from './kubernetes/replica_set';
import { k8sStatefulSetEntity } from './kubernetes/stateful_set';
import { k8sContainerEntity } from './kubernetes/container_entity';

export type EntityDataStreamType = 'metrics' | 'logs' | 'traces';
export type Schema = 'ecs' | 'semconv';

export type EntityFields = Fields &
Partial<{
Expand All @@ -32,4 +43,20 @@ export type EntityFields = Fields &
[key: string]: any;
}>;

export const entities = { serviceEntity, hostEntity, containerEntity };
export const entities = {
serviceEntity,
hostEntity,
containerEntity,
k8s: {
k8sClusterJobEntity,
k8sCronJobEntity,
k8sDaemonSetEntity,
k8sDeploymentEntity,
k8sJobSetEntity,
k8sNodeEntity,
k8sPodEntity,
k8sReplicaSetEntity,
k8sStatefulSetEntity,
k8sContainerEntity,
},
};
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/

import { Schema } from '..';
import { K8sEntity } from '.';

export function k8sClusterJobEntity({
schema,
name,
entityId,
...others
}: {
schema: Schema;
name: string;
entityId: string;
[key: string]: any;
}) {
if (schema === 'ecs') {
return new K8sEntity(schema, {
'entity.type': 'cluster',
'orchestrator.cluster.name': name,
'entity.id': entityId,
...others,
});
}

return new K8sEntity(schema, {
'entity.type': 'cluster',
'k8s.cluster.uid': name,
'entity.id': entityId,
...others,
});
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/

import { Schema } from '..';
import { K8sEntity } from '.';

export function k8sContainerEntity({
schema,
id,
entityId,
...others
}: {
schema: Schema;
id: string;
entityId: string;
[key: string]: any;
}) {
if (schema === 'ecs') {
return new K8sEntity(schema, {
'entity.type': 'container',
'kubernetes.container.id': id,
'entity.id': entityId,
...others,
});
}

return new K8sEntity(schema, {
'entity.type': 'container',
'container.id': id,
'entity.id': entityId,
...others,
});
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/

import { Schema } from '..';
import { K8sEntity } from '.';

export function k8sCronJobEntity({
schema,
name,
uid,
clusterName,
entityId,
...others
}: {
schema: Schema;
name: string;
uid?: string;
clusterName?: string;
entityId: string;
[key: string]: any;
}) {
if (schema === 'ecs') {
return new K8sEntity(schema, {
'entity.type': 'cron_job',
'kubernetes.cronjob.name': name,
'kubernetes.cronjob.uid': uid,
'kubernetes.namespace': clusterName,
'entity.id': entityId,
...others,
});
}

return new K8sEntity(schema, {
'entity.type': 'cron_job',
'k8s.cronjob.name': name,
'k8s.cronjob.uid': uid,
'k8s.cluster.name': clusterName,
'entity.id': entityId,
...others,
});
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/

import { Schema } from '..';
import { K8sEntity } from '.';

export function k8sDaemonSetEntity({
schema,
name,
uid,
clusterName,
entityId,
...others
}: {
schema: Schema;
name: string;
uid?: string;
clusterName?: string;
entityId: string;
[key: string]: any;
}) {
if (schema === 'ecs') {
return new K8sEntity(schema, {
'entity.type': 'daemon_set',
'kubernetes.daemonset.name': name,
'kubernetes.daemonset.uid': uid,
'kubernetes.namespace': clusterName,
'entity.id': entityId,
...others,
});
}

return new K8sEntity(schema, {
'entity.type': 'daemon_set',
'k8s.daemonset.name': name,
'k8s.daemonset.uid': uid,
'k8s.cluster.name': clusterName,
'entity.id': entityId,
...others,
});
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/

import { Schema } from '..';
import { K8sEntity } from '.';

export function k8sDeploymentEntity({
schema,
name,
uid,
clusterName,
entityId,
...others
}: {
schema: Schema;
name: string;
uid?: string;
clusterName?: string;
entityId: string;
[key: string]: any;
}) {
if (schema === 'ecs') {
return new K8sEntity(schema, {
'entity.type': 'deployment',
'kubernetes.deployment.name': name,
'kubernetes.deployment.uid': uid,
'kubernetes.namespace': clusterName,
'entity.id': entityId,
...others,
});
}

return new K8sEntity(schema, {
'entity.type': 'deployment',
'k8s.deployment.name': name,
'k8s.deployment.uid': uid,
'k8s.cluster.name': clusterName,
'entity.id': entityId,
...others,
});
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/

import type { EntityFields, Schema } from '..';
import { Serializable } from '../../serializable';

const identityFieldsMap: Record<Schema, Record<string, string[]>> = {
ecs: {
pod: ['kubernetes.pod.name'],
cluster: ['orchestrator.cluster.name'],
cron_job: ['kubernetes.cronjob.name'],
daemon_set: ['kubernetes.daemonset.name'],
deployment: ['kubernetes.deployment.name'],
job: ['kubernetes.job.name'],
node: ['kubernetes.node.name'],
replica_set: ['kubernetes.replicaset.name'],
stateful_set: ['kubernetes.statefulset.name'],
container: ['kubernetes.container.id'],
},
semconv: {
pod: ['k8s.pod.name'],
cluster: ['k8s.cluster.uid'],
cron_job: ['k8s.cronjob.name'],
daemon_set: ['k8s.daemonset.name'],
deployment: ['k8s.deployment.name'],
job: ['k8s.job.name'],
node: ['k8s.node.uid'],
replica_set: ['k8s.replicaset.name'],
stateful_set: ['k8s.statefulset.name'],
container: ['container.id'],
},
};

export class K8sEntity extends Serializable<EntityFields> {
constructor(schema: Schema, fields: EntityFields) {
const entityType = fields['entity.type'];
if (entityType === undefined) {
throw new Error(`Entity type not defined: ${entityType}`);
}

const entityTypeWithSchema = `kubernetes_${entityType}_${schema}`;
const identityFields = identityFieldsMap[schema][entityType];
if (identityFields === undefined || identityFields.length === 0) {
throw new Error(
`Identity fields not defined for schema: ${schema} and entity type: ${entityType}`
);
}

super({
...fields,
'entity.type': entityTypeWithSchema,
'entity.definitionId': `builtin_${entityTypeWithSchema}`,
'entity.identityFields': identityFields,
'entity.displayName': getDisplayName({ identityFields, fields }),
});
}
}

function getDisplayName({
identityFields,
fields,
}: {
identityFields: string[];
fields: EntityFields;
}) {
return identityFields
.map((field) => fields[field])
.filter((_) => _)
.join(':');
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/

import { Schema } from '..';
import { K8sEntity } from '.';

export function k8sJobSetEntity({
schema,
name,
uid,
clusterName,
entityId,
...others
}: {
schema: Schema;
name: string;
uid?: string;
clusterName?: string;
entityId: string;
[key: string]: any;
}) {
if (schema === 'ecs') {
return new K8sEntity(schema, {
'entity.type': 'job',
'kubernetes.job.name': name,
'kubernetes.job.uid': uid,
'kubernetes.namespace': clusterName,
'entity.id': entityId,
...others,
});
}

return new K8sEntity(schema, {
'entity.type': 'job',
'k8s.job.name': name,
'k8s.job.uid': uid,
'k8s.cluster.name': clusterName,
'entity.id': entityId,
...others,
});
}
Loading

0 comments on commit 5d8ecc0

Please sign in to comment.