From e86aede50c8f01858973fd62b8f7b61d73c238c3 Mon Sep 17 00:00:00 2001 From: f1ames Date: Fri, 1 Sep 2023 10:59:52 +0200 Subject: [PATCH 01/26] feat: add basic policy CRDs --- README.md | 35 ++++++++++++++++++++++++++++++- monokle.policy.crd.yaml | 46 +++++++++++++++++++++++++++++++++++++++++ policy.yaml | 21 +++++++++++++++++++ 3 files changed, 101 insertions(+), 1 deletion(-) create mode 100644 monokle.policy.crd.yaml create mode 100644 policy.yaml diff --git a/README.md b/README.md index f5a2314..91a4bbb 100644 --- a/README.md +++ b/README.md @@ -75,4 +75,37 @@ skaffold dev * https://kubernetes.io/blog/2019/03/21/a-guide-to-kubernetes-admission-controllers/ * https://github.com/stackrox/admission-controller-webhook-demo/tree/master * https://www.witodelnat.eu/blog/2021/local-kubernetes-development -* https://minikube.sigs.k8s.io/docs/tutorials/using_psp/ \ No newline at end of file +* https://minikube.sigs.k8s.io/docs/tutorials/using_psp/ + +## Policy as CRDs + +> https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/ +> https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/ + +1. Start minikube. +1. Apply resource definition: + +```bash +kubectl apply -f monokle.policy.crd.yaml +``` + +3. Test if it was applied correctly: + +```bash +kubectl get crd +kubectl get monoklepolicy +kubectl describe crd monoklepolicies.monokle.com +``` + +4. Create sample policy resource: + +```bash +kubectl apply -f policy.yaml +``` + +5. Test if it was applied correctly: + +```bash +kubectl get monoklepolicy +kubectl describe monoklepolicy policy-sample +``` diff --git a/monokle.policy.crd.yaml b/monokle.policy.crd.yaml new file mode 100644 index 0000000..b14ee38 --- /dev/null +++ b/monokle.policy.crd.yaml @@ -0,0 +1,46 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: monoklepolicies.monokle.com +spec: + group: monokle.com + versions: + - name: v1 + served: true + storage: true + schema: + # For schema see: + # - maps/dicts - https://swagger.io/docs/specification/data-models/dictionaries/ + # - structural schema - https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/#specifying-a-structural-schema + # + # Object values as multitypes: + # Even though it's supported by OpenAPI spec, e.g. https://stackoverflow.com/a/46475776, + # Kubernetes requires "structural" definition # https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/#specifying-a-structural-schema + # which seems to be in opposite to it "does not set description, type, default, additionalProperties, nullable + # within an allOf, anyOf, oneOf or not, with the exception of the two pattern for x-kubernetes-int-or-string: true (see below)." + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + plugins: + type: object + additionalProperties: + type: boolean + rules: + type: object + additionalProperties: true + settings: + type: object + additionalProperties: + type: object + additionalProperties: + type: string + scope: Namespaced + names: + plural: monoklepolicies + singular: monoklepolicy + kind: MonoklePolicy + shortNames: + - mp diff --git a/policy.yaml b/policy.yaml new file mode 100644 index 0000000..8784452 --- /dev/null +++ b/policy.yaml @@ -0,0 +1,21 @@ +apiVersion: monokle.com/v1 +kind: MonoklePolicy +metadata: + name: policy-sample +spec: + plugins: + yaml-syntax: true + open-policy-agent: true + resource-links: true + kubernetes-schema: true + annotations: true + rules: + yaml-syntax/no-bad-alias: "warn" + yaml-syntax/no-bad-directive: false + open-policy-agent/no-last-image: "err" + open-policy-agent/cpu-limit: "err" + open-policy-agent/memory-limit: "err" + open-policy-agent/memory-request: "err" + settings: + kubernetes-schema: + schemaVersion: v1.24.2 \ No newline at end of file From 474756fa7d7694bffbb0827be5e5feb82c7032e1 Mon Sep 17 00:00:00 2001 From: f1ames Date: Mon, 4 Sep 2023 11:29:26 +0200 Subject: [PATCH 02/26] chore: add test script for CRDs informer --- README.md | 14 ++++++++++++++ kac-api/scripts/list-policies.js | 32 ++++++++++++++++++++++++++++++++ kac-api/src/index.ts | 2 +- 3 files changed, 47 insertions(+), 1 deletion(-) create mode 100644 kac-api/scripts/list-policies.js diff --git a/README.md b/README.md index 91a4bbb..8debb93 100644 --- a/README.md +++ b/README.md @@ -109,3 +109,17 @@ kubectl apply -f policy.yaml kubectl get monoklepolicy kubectl describe monoklepolicy policy-sample ``` + +### CRDs Informer + +You can test how watching CRDs changes works by running: + +```bash +node kac-api/scripts/list-policies.js +``` + +and (re)applying sample policy: + +```bash +kubectl apply -f policy.yaml +``` diff --git a/kac-api/scripts/list-policies.js b/kac-api/scripts/list-policies.js new file mode 100644 index 0000000..d96fc2b --- /dev/null +++ b/kac-api/scripts/list-policies.js @@ -0,0 +1,32 @@ +import k8s from '@kubernetes/client-node'; + +const kc = new k8s.KubeConfig() +kc.loadFromDefault() + +const k8sApi = kc.makeApiClient(k8s.CustomObjectsApi) +const listFn = () => k8sApi.listNamespacedCustomObject('monokle.com','v1', 'default', 'monoklepolicies'); +const informer = k8s.makeInformer(kc, '/apis/monokle.com/v1/namespaces/default/monoklepolicies', listFn); + +informer.on('add', (obj) => { + console.log(`Added:`, obj); +}); + +informer.on('update', (obj) => { + console.log(`Updated:`, obj); +}); + +informer.on('delete', (obj) => { + console.log(`Deleted:`, obj); +}); + +informer.on('error', (err) => { + console.error(err); + setTimeout(() => { + console.log('Restarting informer...'); + informer.start(); + }, 5000); +}); + +informer.start().then(() => { + console.log('Informer started'); +}); diff --git a/kac-api/src/index.ts b/kac-api/src/index.ts index a826742..3a432b7 100644 --- a/kac-api/src/index.ts +++ b/kac-api/src/index.ts @@ -125,4 +125,4 @@ function createResourceForValidation(admissionResource: AdmissionRequest): Resou }; return resource; -} \ No newline at end of file +} From ecbce85d2916760629413ddddaf868ec38133f4f Mon Sep 17 00:00:00 2001 From: f1ames Date: Wed, 6 Sep 2023 14:56:21 +0200 Subject: [PATCH 03/26] feat: integrate policy CRDs into webhook [WIP] --- kac-api/src/index.ts | 46 ++++++++++++++++++++++++++++++++++- kac-api/src/utils/informer.ts | 39 +++++++++++++++++++++++++++++ kac-api/src/utils/server.ts | 33 +++++++++++++++++++++++++ 3 files changed, 117 insertions(+), 1 deletion(-) create mode 100644 kac-api/src/utils/informer.ts create mode 100644 kac-api/src/utils/server.ts diff --git a/kac-api/src/index.ts b/kac-api/src/index.ts index 3a432b7..3f85f79 100644 --- a/kac-api/src/index.ts +++ b/kac-api/src/index.ts @@ -3,6 +3,7 @@ import path from "path"; import fastify from "fastify"; import {V1ValidatingWebhookConfiguration, V1ObjectMeta} from "@kubernetes/client-node"; import {createDefaultMonokleValidator, Resource} from "@monokle/validation"; +import { getInformer } from "./utils/informer"; // TODO - some mismatch with types, this is not exactly the type which should be used type AdmissionRequest = V1ValidatingWebhookConfiguration & { @@ -23,7 +24,10 @@ type AdmissionResponse = { } } -const validator = createDefaultMonokleValidator(); + + + + const server = fastify({ https: { @@ -32,6 +36,9 @@ const server = fastify({ } }); + +// Validate endpoint logic should be as thin as possible to reduce impact on resource creation time. +// All the preloading related to validator should be done separately (before or in the meantime). server.post("/validate", async (req, res): Promise => { console.log('request', req.headers, req.body) @@ -126,3 +133,40 @@ function createResourceForValidation(admissionResource: AdmissionRequest): Resou return resource; } + +async function init() { + let awaitValidatorReadiness = Promise.resolve(); + let hasPolicy = false; + + // kube-client get current namespace where webhook is deployed + // https://stackoverflow.com/a/46046153 + + // init validator + const validator = createDefaultMonokleValidator(); + + // init informer + const informer = await getInformer( + // For now the assumption is there is one policy per namespace. + async (obj) => { + const policyConfig = obj.spec; + + hasPolicy = true; + awaitValidatorReadiness = validator.preload(policyConfig); + await awaitValidatorReadiness; + }, + async (obj) => { + const policyConfig = obj.spec; + + hasPolicy = true; + awaitValidatorReadiness = validator.preload(policyConfig); + await awaitValidatorReadiness; + }, + (obj) => { + // if policy was deleted, we fallback to default, disabled policy? + // or just skip validation for new resources (because it doesn't make sense to validate against empty policy anyways) + hasPolicy = false; + } + ); + + // init server +} \ No newline at end of file diff --git a/kac-api/src/utils/informer.ts b/kac-api/src/utils/informer.ts new file mode 100644 index 0000000..b7d384a --- /dev/null +++ b/kac-api/src/utils/informer.ts @@ -0,0 +1,39 @@ +import k8s from '@kubernetes/client-node'; +import { Config } from '@monokle/validation'; + +export type MonoklePolicy = k8s.KubernetesObject & { + spec: Config +}; + +export async function getInformer( + onAdded: k8s.ObjectCallback, + onUpdated: k8s.ObjectCallback, + onDeleted: k8s.ObjectCallback, + namespace = 'default', + onError?: k8s.ErrorCallback +) { + const kc = new k8s.KubeConfig() + kc.loadFromDefault() + + const k8sApi = kc.makeApiClient(k8s.CustomObjectsApi) + const listFn = () => k8sApi.listNamespacedCustomObject('monokle.com','v1', namespace, 'monoklepolicies'); + const informer = k8s.makeInformer(kc, `/apis/monokle.com/v1/namespaces/${namespace}/monoklepolicies`, listFn as any); + + informer.on('add', async (obj) => await onAdded(obj)); + informer.on('update', async (obj) => await onUpdated(obj)); + informer.on('delete', async (obj) => await onDeleted(obj)); + + informer.on('error', (err) => { + if (onError) { + onError(err); + } + + setTimeout(async () => { + await informer.start(); + }, 1000); + }); + + await informer.start(); + + return informer; +} diff --git a/kac-api/src/utils/server.ts b/kac-api/src/utils/server.ts new file mode 100644 index 0000000..d67acf7 --- /dev/null +++ b/kac-api/src/utils/server.ts @@ -0,0 +1,33 @@ +import { MonokleValidator } from "@monokle/validation"; + +export class Server { + private _server: any; // fastify instance + private _shouldValidate: boolean + + constructor( + private readonly _validator: MonokleValidator, + ) { + // init server + this._shouldValidate = false; + } + + get shouldValidate() { + return this._shouldValidate; + } + + set shouldValidate(value: boolean) { + this._shouldValidate = value; + } + + start() { + this.shouldValidate = true; + } + + stop() { + this.shouldValidate = false; + } + + private async _initServer(resource: any) { + + private async _validateResource(resource: any) {} +} \ No newline at end of file From b1b7b3d2f906762c8c117aa58f90144c7e5c6459 Mon Sep 17 00:00:00 2001 From: f1ames Date: Mon, 25 Sep 2023 11:10:34 +0200 Subject: [PATCH 04/26] feat: integrate policy CRDs into webhook --- kac-api/package-lock.json | 183 +++++++++++++---------- kac-api/package.json | 3 +- kac-api/src/index.ts | 194 +++++-------------------- kac-api/src/utils/server.ts | 33 ----- kac-api/src/utils/validation-server.ts | 167 +++++++++++++++++++++ 5 files changed, 313 insertions(+), 267 deletions(-) delete mode 100644 kac-api/src/utils/server.ts create mode 100644 kac-api/src/utils/validation-server.ts diff --git a/kac-api/package-lock.json b/kac-api/package-lock.json index 9eaa45e..458e97e 100644 --- a/kac-api/package-lock.json +++ b/kac-api/package-lock.json @@ -10,7 +10,8 @@ "license": "ISC", "dependencies": { "@kubernetes/client-node": "^0.18.1", - "@monokle/validation": "^0.16.0", + "@monokle/parser": "0.2.0", + "@monokle/validation": "^0.30.1", "fastify": "^3.28.0", "isomorphic-fetch": "^3.0.0", "node-fetch": "^2.6.6", @@ -61,15 +62,31 @@ } }, "node_modules/@kubernetes/client-node/node_modules/@types/node": { - "version": "18.16.12", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.16.12.tgz", - "integrity": "sha512-tIRrjbY9C277MOfP8M3zjMIhtMlUJ6YVqkGgLjz+74jVsdf4/UjC6Hku4+1N0BS0qyC0JAS6tJLUk9H6JUKviQ==" + "version": "18.17.19", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.17.19.tgz", + "integrity": "sha512-+pMhShR3Or5GR0/sp4Da7FnhVmTalWm81M6MkEldbwjETSaPalw138Z4KdpQaistvqQxLB7Cy4xwYdxpbSOs9Q==" + }, + "node_modules/@monokle/parser": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/@monokle/parser/-/parser-0.2.0.tgz", + "integrity": "sha512-gcCWcfOUqTubwi9mJlrOb4X5NW5B6N4/lCR7oIh5pDL75rQUfuHbSbx+BkymZnrGgNNwGX9LMGukdW8A9Y5TMQ==", + "dependencies": { + "lodash": "4.17.21", + "path-browserify": "^1.0.1", + "yaml": "2.2.2" + } + }, + "node_modules/@monokle/types": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/@monokle/types/-/types-0.2.2.tgz", + "integrity": "sha512-GGhl30a1WImxfxWg9Ys7MY1VQi2NU2iCH+gf4kKGZxS6nqAkgU4G1kTTYnxQzGNoM4Zbabh5aRxINsckIV5a+g==" }, "node_modules/@monokle/validation": { - "version": "0.16.0", - "resolved": "https://registry.npmjs.org/@monokle/validation/-/validation-0.16.0.tgz", - "integrity": "sha512-BSuAFzxffNGObHywTpYStYxTzzFYpXekQ8ciQBUZUdTjFCjBCbPMTXjperORTJY0zIPohUmtsse6jfxVcUnw4A==", + "version": "0.30.1", + "resolved": "https://registry.npmjs.org/@monokle/validation/-/validation-0.30.1.tgz", + "integrity": "sha512-gm8FCGQA4VkbSL6DgEk10tDJTbfhqvc81BJ/oHsTMfArT3hjdksYOiNsHj6gzGEFsHXpGHYFqywD1qD1h0jkIA==", "dependencies": { + "@monokle/types": "*", "@open-policy-agent/opa-wasm": "1.8.0", "@rollup/plugin-virtual": "3.0.1", "ajv": "6.12.6", @@ -77,9 +94,9 @@ "isomorphic-fetch": "3.0.0", "lodash": "4.17.21", "node-fetch": "3.3.0", - "react-fast-compare": "3.2.1", "require-from-string": "2.0.2", "rollup": "3.18.0", + "uuid": "9.0.0", "yaml": "2.2.2", "zod": "3.19.1" } @@ -135,19 +152,19 @@ } }, "node_modules/@types/caseless": { - "version": "0.12.2", - "resolved": "https://registry.npmjs.org/@types/caseless/-/caseless-0.12.2.tgz", - "integrity": "sha512-6ckxMjBBD8URvjB6J3NcnuAn5Pkl7t3TizAg+xdlzzQGSPSmBcXf8KoIH0ua/i+tio+ZRUHEXp0HEmvaR4kt0w==" + "version": "0.12.3", + "resolved": "https://registry.npmjs.org/@types/caseless/-/caseless-0.12.3.tgz", + "integrity": "sha512-ZD/NsIJYq/2RH+hY7lXmstfp/v9djGt9ah+xRQ3pcgR79qiKsG4pLl25AI7IcXxVO8dH9GiBE5rAknC0ePntlw==" }, "node_modules/@types/js-yaml": { - "version": "4.0.5", - "resolved": "https://registry.npmjs.org/@types/js-yaml/-/js-yaml-4.0.5.tgz", - "integrity": "sha512-FhpRzf927MNQdRZP0J5DLIdTXhjLYzeUTmLAu69mnVksLH9CJY3IuSeEgbKUki7GQZm0WqDkGzyxju2EZGD2wA==" + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/@types/js-yaml/-/js-yaml-4.0.6.tgz", + "integrity": "sha512-ACTuifTSIIbyksx2HTon3aFtCKWcID7/h3XEmRpDYdMCXxPbl+m9GteOJeaAkiAta/NJaSFuA7ahZ0NkwajDSw==" }, "node_modules/@types/node": { - "version": "20.2.0", - "resolved": "https://registry.npmjs.org/@types/node/-/node-20.2.0.tgz", - "integrity": "sha512-3iD2jaCCziTx04uudpJKwe39QxXgSUnpxXSvRQjRvHPxFQfmfP4NXIm/NURVeNlTCc+ru4WqjYGTmpXrW9uMlw==" + "version": "20.6.5", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.6.5.tgz", + "integrity": "sha512-2qGq5LAOTh9izcc0+F+dToFigBWiK1phKPt7rNhOqJSr35y8rlIBjDwGtFSgAI6MGIhjwOVNSQZVdJsZJ2uR1w==" }, "node_modules/@types/request": { "version": "2.48.8", @@ -161,14 +178,14 @@ } }, "node_modules/@types/tough-cookie": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.2.tgz", - "integrity": "sha512-Q5vtl1W5ue16D+nIaW8JWebSSraJVlK+EthKn7e7UcD4KWsaSJ8BqGPXNaPghgtcn/fhvrN17Tv8ksUsQpiplw==" + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.3.tgz", + "integrity": "sha512-THo502dA5PzG/sfQH+42Lw3fvmYkceefOspdCwpHRul8ik2Jv1K8I5OZz1AT3/rs46kwgMCe9bSBmDLYkkOMGg==" }, "node_modules/@types/ws": { - "version": "8.5.4", - "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.5.4.tgz", - "integrity": "sha512-zdQDHKUgcX/zBc4GrwsE/7dVdAD8JR4EuiAXiiUhhfyIJXXb2+PrGshFyeXWQPMmmZ2XxgaqclgpIC7eTXc1mg==", + "version": "8.5.5", + "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.5.5.tgz", + "integrity": "sha512-lwhs8hktwxSjf9UaZ9tG5M03PGogvFaH8gUgLNbN9HKIg0dvv6q+gkSuJ8HN4/VbyxkuLzCjlN7GquQ0gUJfIg==", "dependencies": { "@types/node": "*" } @@ -459,9 +476,9 @@ ] }, "node_modules/fast-content-type-parse": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/fast-content-type-parse/-/fast-content-type-parse-1.0.0.tgz", - "integrity": "sha512-Xbc4XcysUXcsP5aHUU7Nq3OwvHq97C+WnbkeIefpeYLX+ryzFJlU6OStFJhs6Ol0LkUGpcK+wL0JwfM+FCU5IA==" + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/fast-content-type-parse/-/fast-content-type-parse-1.1.0.tgz", + "integrity": "sha512-fBHHqSTFLVnR61C+gltJuE5GkVQMV0S2nqUO8TJ+5Z3qAKG8vAx4FKai1s5jq/inV1+sREynIWSuQ6HgoSXpDQ==" }, "node_modules/fast-decode-uri-component": { "version": "1.0.1", @@ -493,9 +510,9 @@ } }, "node_modules/fast-redact": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/fast-redact/-/fast-redact-3.2.0.tgz", - "integrity": "sha512-zaTadChr+NekyzallAMXATXLOR8MNx3zqpZ0MUF2aGf4EathnG0f32VLODNlY8IuGY3HoRO2L6/6fSzNsLaHIw==", + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/fast-redact/-/fast-redact-3.3.0.tgz", + "integrity": "sha512-6T5V1QK1u4oF+ATxs1lWUmlEk6P2T9HqJG3e2DnHOdVgZy2rFJBoEnrIedcTXlkAHU/zKC+7KETJ+KGGKwxgMQ==", "engines": { "node": ">=6" } @@ -645,9 +662,9 @@ "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==" }, "node_modules/fsevents": { - "version": "2.3.2", - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", - "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", "hasInstallScript": true, "optional": true, "os": [ @@ -778,9 +795,9 @@ "integrity": "sha512-Yljz7ffyPbrLpLngrMtZ7NduUgVvi6wG9RJ9IUcyCd59YQ911PBJphODUcbOVbqYfxe1wuYf/LJ8PauMRwsM/g==" }, "node_modules/jose": { - "version": "4.14.4", - "resolved": "https://registry.npmjs.org/jose/-/jose-4.14.4.tgz", - "integrity": "sha512-j8GhLiKmUAh+dsFXlX1aJCbt5KMibuKb+d7j1JaOJG6s2UjX1PQlW+OKB/sD4a/5ZYF4RcmYmLSndOoU3Lt/3g==", + "version": "4.14.6", + "resolved": "https://registry.npmjs.org/jose/-/jose-4.14.6.tgz", + "integrity": "sha512-EqJPEUlZD0/CSUMubKtMaYUOtWe91tZXTWMJZoKSbLk+KtdhNdcvppH8lA9XwVu2V4Ailvsj0GBZJ2ZwDjfesQ==", "optional": true, "funding": { "url": "https://github.com/sponsors/panva" @@ -999,14 +1016,22 @@ } }, "node_modules/node-fetch": { - "version": "2.6.6", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.6.tgz", - "integrity": "sha512-Z8/6vRlTUChSdIgMa51jxQ4lrw/Jy5SOW10ObaA47/RElsAN2c5Pn8bTgFGWn/ibwzXTE8qwr1Yzx28vsecXEA==", + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", "dependencies": { "whatwg-url": "^5.0.0" }, "engines": { "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } } }, "node_modules/oauth-sign": { @@ -1044,12 +1069,12 @@ } }, "node_modules/openid-client": { - "version": "5.4.2", - "resolved": "https://registry.npmjs.org/openid-client/-/openid-client-5.4.2.tgz", - "integrity": "sha512-lIhsdPvJ2RneBm3nGBBhQchpe3Uka//xf7WPHTIglery8gnckvW7Bd9IaQzekzXJvWthCMyi/xVEyGW0RFPytw==", + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/openid-client/-/openid-client-5.5.0.tgz", + "integrity": "sha512-Y7Xl8BgsrkzWLHkVDYuroM67hi96xITyEDSkmWaGUiNX6CkcXC3XyQGdv5aWZ6dukVKBFVQCADi9gCavOmU14w==", "optional": true, "dependencies": { - "jose": "^4.14.1", + "jose": "^4.14.4", "lru-cache": "^6.0.0", "object-hash": "^2.2.0", "oidc-token-hash": "^5.0.3" @@ -1076,6 +1101,11 @@ "tslib": "^2.0.3" } }, + "node_modules/path-browserify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-browserify/-/path-browserify-1.0.1.tgz", + "integrity": "sha512-b7uo2UCUOYZcnF/3ID0lulOJi/bafxa1xPe7ZPsammBSpjSWQkjNxlt635YGS2MiR9GjvuXCtz2emr3jbsz98g==" + }, "node_modules/path-case": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/path-case/-/path-case-3.0.4.tgz", @@ -1182,11 +1212,6 @@ "resolved": "https://registry.npmjs.org/quick-format-unescaped/-/quick-format-unescaped-4.0.4.tgz", "integrity": "sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg==" }, - "node_modules/react-fast-compare": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/react-fast-compare/-/react-fast-compare-3.2.1.tgz", - "integrity": "sha512-xTYf9zFim2pEif/Fw16dBiXpe0hoy5PxcD8+OwBnTtNLfIm3g6WxhKNurY+6OmdH1u6Ta/W/Vl6vjbYP1MFnDg==" - }, "node_modules/request": { "version": "2.88.2", "resolved": "https://registry.npmjs.org/request/-/request-2.88.2.tgz", @@ -1231,6 +1256,15 @@ "node": ">= 0.12" } }, + "node_modules/request/node_modules/uuid": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", + "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==", + "deprecated": "Please upgrade to version 7 or higher. Older versions may use Math.random() in certain circumstances, which is known to be problematic. See https://v8.dev/blog/math-random for details.", + "bin": { + "uuid": "bin/uuid" + } + }, "node_modules/require-from-string": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", @@ -1333,9 +1367,9 @@ "integrity": "sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw==" }, "node_modules/semver": { - "version": "7.5.1", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.1.tgz", - "integrity": "sha512-Wvss5ivl8TMRZXXESstBA4uR5iXgEN/VC5/sOcuXdVLzcdkz4HWetIoRfG5gb5X+ij/G9rw9YoGn3QoQ8OCSpw==", + "version": "7.5.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz", + "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==", "dependencies": { "lru-cache": "^6.0.0" }, @@ -1385,9 +1419,9 @@ } }, "node_modules/sprintf-js": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.2.tgz", - "integrity": "sha512-VE0SOVEHCk7Qc8ulkWw3ntAzXuqf7S2lvwQaDLRnUeIEaKNQJzV6BwmLKhOqT61aGhfUMrXeaBk+oDGCzvhcug==" + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.3.tgz", + "integrity": "sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==" }, "node_modules/sshpk": { "version": "1.17.0", @@ -1428,9 +1462,9 @@ "deprecated": "Package no longer supported. Contact Support at https://www.npmjs.com/support for more info." }, "node_modules/tar": { - "version": "6.1.15", - "resolved": "https://registry.npmjs.org/tar/-/tar-6.1.15.tgz", - "integrity": "sha512-/zKt9UyngnxIT/EAGYuxaMYgOIJiP81ab9ZfkILq4oNLPFX50qyYmu7jRj9qeXoxmJHjGlbH0+cm2uy1WCs10A==", + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/tar/-/tar-6.2.0.tgz", + "integrity": "sha512-/Wo7DcT0u5HUV486xg675HtjNd3BXZ6xDbzsCUZPt5iw8bTQ63bP0Raut3mvro9u+CUyq7YQd8Cx55fsZXxqLQ==", "dependencies": { "chownr": "^2.0.0", "fs-minipass": "^2.0.0", @@ -1488,9 +1522,9 @@ "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" }, "node_modules/tslib": { - "version": "2.5.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.1.tgz", - "integrity": "sha512-KaI6gPil5m9vF7DKaoXxx1ia9fxS4qG5YveErRRVknPDXXriu5M8h48YRjB6h5ZUOKuAKlSJYb0GaDe8I39fRw==" + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz", + "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==" }, "node_modules/tunnel-agent": { "version": "0.6.0", @@ -1509,23 +1543,21 @@ "integrity": "sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA==" }, "node_modules/type-fest": { - "version": "3.10.0", - "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-3.10.0.tgz", - "integrity": "sha512-hmAPf1datm+gt3c2mvu0sJyhFy6lTkIGf0GzyaZWxRLnabQfPUqg6tF95RPg6sLxKI7nFLGdFxBcf2/7+GXI+A==", + "version": "3.13.1", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-3.13.1.tgz", + "integrity": "sha512-tLq3bSNx+xSpwvAJnzrK0Ep5CLNWjvFTOp71URMaAEWBfRb9nnJiBoUe0tF8bI4ZFO3omgBR6NvnbzVUT3Ly4g==", "engines": { "node": ">=14.16" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" - }, - "peerDependencies": { - "typescript": ">=4.7.0" } }, "node_modules/typescript": { "version": "4.9.5", "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.9.5.tgz", "integrity": "sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g==", + "dev": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -1564,12 +1596,11 @@ } }, "node_modules/uuid": { - "version": "3.4.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", - "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==", - "deprecated": "Please upgrade to version 7 or higher. Older versions may use Math.random() in certain circumstances, which is known to be problematic. See https://v8.dev/blog/math-random for details.", + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.0.tgz", + "integrity": "sha512-MXcSTerfPa4uqyzStbRoTgt5XIe3x5+42+q1sDuy3R5MDk66URdLMOZe5aPX/SQd+kuYAh0FdP/pO28IkQyTeg==", "bin": { - "uuid": "bin/uuid" + "uuid": "dist/bin/uuid" } }, "node_modules/verror": { @@ -1599,9 +1630,9 @@ "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" }, "node_modules/whatwg-fetch": { - "version": "3.6.2", - "resolved": "https://registry.npmjs.org/whatwg-fetch/-/whatwg-fetch-3.6.2.tgz", - "integrity": "sha512-bJlen0FcuU/0EMLrdbJ7zOnW6ITZLrZMIarMUVmdKtsGvZna8vxKYaexICWPfZ8qwf9fzNq+UEIZrnSaApt6RA==" + "version": "3.6.19", + "resolved": "https://registry.npmjs.org/whatwg-fetch/-/whatwg-fetch-3.6.19.tgz", + "integrity": "sha512-d67JP4dHSbm2TrpFj8AbO8DnL1JXL5J9u0Kq2xW6d0TFDbCA3Muhdt8orXC22utleTVj7Prqt82baN6RBvnEgw==" }, "node_modules/whatwg-url": { "version": "5.0.0", @@ -1618,9 +1649,9 @@ "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" }, "node_modules/ws": { - "version": "8.13.0", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.13.0.tgz", - "integrity": "sha512-x9vcZYTrFPC7aSIbj7sRCYo7L/Xb8Iy+pW0ng0wt2vCJv7M9HOMy0UoN3rr+IFC7hb7vXoqS+P9ktyLLLhO+LA==", + "version": "8.14.2", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.14.2.tgz", + "integrity": "sha512-wEBG1ftX4jcglPxgFCMJmZ2PLtSbJ2Peg6TmpJFTbe9GZYOQCDPdMYu/Tm0/bGZkw8paZnJY45J4K2PZrLYq8g==", "engines": { "node": ">=10.0.0" }, diff --git a/kac-api/package.json b/kac-api/package.json index 12fce0e..900299a 100644 --- a/kac-api/package.json +++ b/kac-api/package.json @@ -13,7 +13,8 @@ "license": "ISC", "dependencies": { "@kubernetes/client-node": "^0.18.1", - "@monokle/validation": "^0.16.0", + "@monokle/validation": "^0.30.1", + "@monokle/parser": "0.2.0", "fastify": "^3.28.0", "isomorphic-fetch": "^3.0.0", "node-fetch": "^2.6.6", diff --git a/kac-api/src/index.ts b/kac-api/src/index.ts index 3f85f79..f091477 100644 --- a/kac-api/src/index.ts +++ b/kac-api/src/index.ts @@ -1,172 +1,52 @@ -import {readFileSync} from "fs"; -import path from "path"; -import fastify from "fastify"; -import {V1ValidatingWebhookConfiguration, V1ObjectMeta} from "@kubernetes/client-node"; -import {createDefaultMonokleValidator, Resource} from "@monokle/validation"; -import { getInformer } from "./utils/informer"; +import {AnnotationSuppressor, FingerprintSuppressor, MonokleValidator, RemotePluginLoader, SchemaLoader, DisabledFixer, ResourceParser} from "@monokle/validation"; +import { MonoklePolicy, getInformer } from "./utils/informer"; +import { ValidationServer } from "./utils/validation-server"; -// TODO - some mismatch with types, this is not exactly the type which should be used -type AdmissionRequest = V1ValidatingWebhookConfiguration & { - request?: V1ObjectMeta & { - object?: Resource - } -} +// Refs: +// +// kube-client get current namespace where webhook is deployed +// https://stackoverflow.com/a/46046153 -type AdmissionResponse = { - kind: string, - apiVersion: string, - response: { - uid: string, - allowed: boolean, - status: { - message: string - } - } -} +const INITIAL_NAMESPACE = 'default'; +(async() => { + // VALIDATOR + let awaitValidatorReadiness = Promise.resolve(); - - - - -const server = fastify({ - https: { - key: readFileSync(path.join('/run/secrets/tls', 'tls.key')), - cert: readFileSync(path.join('/run/secrets/tls', 'tls.crt')) - } -}); - - -// Validate endpoint logic should be as thin as possible to reduce impact on resource creation time. -// All the preloading related to validator should be done separately (before or in the meantime). -server.post("/validate", async (req, res): Promise => { - console.log('request', req.headers, req.body) - - const body = req.body as AdmissionRequest; - - const response = { - kind: body?.kind || '', - apiVersion: body?.apiVersion || '', - response: { - uid: body?.request?.uid || "", - allowed: true, - status: { - message: "OK" - } - } - } - - // Dev workaround - always return true for webhok server to not block hot-reload - if (body.request?.name?.startsWith('webhook-server-')) { - console.log('Allowing webhook server to pass', response); - - return response; - } - - await validator.preload({ - plugins: { - "kubernetes-schema": true, - "open-policy-agent": true, + const validator = new MonokleValidator( + { + loader: new RemotePluginLoader(), + parser: new ResourceParser(), + schemaLoader: new SchemaLoader(), + suppressors: [new AnnotationSuppressor(), new FingerprintSuppressor()], + fixer: new DisabledFixer(), }, - }); + {} + ); - const resourceForValidation = createResourceForValidation(body); - const validationResponse = await validator.validate({ resources: [resourceForValidation] }); + // INFORMER + let activePolicy: MonoklePolicy | null = null; - const warnings = []; - const errors = []; - for (const result of validationResponse.runs) { - for (const item of result.results) { - if (item.level === "warning") { - warnings.push(item); - } else if (item.level === "error") { - errors.push(item); - } - } + const onPolicy = async (policy: MonoklePolicy) => { + activePolicy = policy; + awaitValidatorReadiness = validator.preload(policy.spec); + await awaitValidatorReadiness; } - if (errors.length > 0 || warnings.length > 0) { - const warningsList = warnings.map((e) => `${e.ruleId}: ${e.message.text}`).join("\n"); - const errorsList = errors.map((e) => `${e.ruleId}: ${e.message.text}`).join("\n"); - const message = []; - - if (errors.length > 0) { - message.push(`\n${errors.length} errors found:\n${errorsList}\n`); - } - - if (warnings.length > 0) { - message.push(`\n${warnings.length} warnings found:\n${warningsList}\n`); - } - - message.push("\nYou can use Monokle (https://monokle.io/) to validate and fix those errors easily!"); - - response.response.allowed = false; - response.response.status.message = message.join(""); + const onPolicyRemoval = async () => { + activePolicy = null; + awaitValidatorReadiness = validator.preload({}); + await awaitValidatorReadiness; } - console.log('response', resourceForValidation, validationResponse, response); - - return response; -}); - -server.listen({port: 8443, host: '0.0.0.0' }, (err, address) => { - if (err) { - console.error(err); - process.exit(1); + const onError = (err: any) => { + console.log('ERROR:INFROMER', err); } - console.log(`Server listening at ${address}`); -}); - -function createResourceForValidation(admissionResource: AdmissionRequest): Resource { - const resource = { - id: admissionResource.request?.uid || '', - fileId: '', - filePath: '', - fileOffset: 0, - name: admissionResource.request?.name || '', - apiVersion: admissionResource.request?.object?.apiVersion || '', - kind: admissionResource.request?.object?.kind || '', - namespace: admissionResource.request?.namespace || '', - content: admissionResource.request?.object || {}, - text: '' - }; - - return resource; -} -async function init() { - let awaitValidatorReadiness = Promise.resolve(); - let hasPolicy = false; - - // kube-client get current namespace where webhook is deployed - // https://stackoverflow.com/a/46046153 - - // init validator - const validator = createDefaultMonokleValidator(); - - // init informer - const informer = await getInformer( - // For now the assumption is there is one policy per namespace. - async (obj) => { - const policyConfig = obj.spec; + const informer = getInformer(onPolicy, onPolicy, onPolicyRemoval, INITIAL_NAMESPACE, onError); - hasPolicy = true; - awaitValidatorReadiness = validator.preload(policyConfig); - await awaitValidatorReadiness; - }, - async (obj) => { - const policyConfig = obj.spec; - - hasPolicy = true; - awaitValidatorReadiness = validator.preload(policyConfig); - await awaitValidatorReadiness; - }, - (obj) => { - // if policy was deleted, we fallback to default, disabled policy? - // or just skip validation for new resources (because it doesn't make sense to validate against empty policy anyways) - hasPolicy = false; - } - ); + // SERVER + const server = new ValidationServer(validator); - // init server -} \ No newline at end of file + await server.start(); +})(); diff --git a/kac-api/src/utils/server.ts b/kac-api/src/utils/server.ts deleted file mode 100644 index d67acf7..0000000 --- a/kac-api/src/utils/server.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { MonokleValidator } from "@monokle/validation"; - -export class Server { - private _server: any; // fastify instance - private _shouldValidate: boolean - - constructor( - private readonly _validator: MonokleValidator, - ) { - // init server - this._shouldValidate = false; - } - - get shouldValidate() { - return this._shouldValidate; - } - - set shouldValidate(value: boolean) { - this._shouldValidate = value; - } - - start() { - this.shouldValidate = true; - } - - stop() { - this.shouldValidate = false; - } - - private async _initServer(resource: any) { - - private async _validateResource(resource: any) {} -} \ No newline at end of file diff --git a/kac-api/src/utils/validation-server.ts b/kac-api/src/utils/validation-server.ts new file mode 100644 index 0000000..78e9c91 --- /dev/null +++ b/kac-api/src/utils/validation-server.ts @@ -0,0 +1,167 @@ +import fastify from "fastify"; +import path from "path"; +import { readFileSync } from "fs"; +import { MonokleValidator, Resource } from "@monokle/validation"; +import {V1ValidatingWebhookConfiguration, V1ObjectMeta} from "@kubernetes/client-node"; + +export type ValidationServerOptions = { + port: number; + host: string; +}; + +type AdmissionRequest = V1ValidatingWebhookConfiguration & { + request?: V1ObjectMeta & { + object?: Resource + } +} + +type AdmissionResponse = { + kind: string, + apiVersion: string, + response: { + uid: string, + allowed: boolean, + status: { + message: string + } + } +} + +export class ValidationServer { + private _server: ReturnType; + private _shouldValidate: boolean + + constructor( + private readonly _validator: MonokleValidator, + private readonly _options: ValidationServerOptions = { + port: 8443, + host: '0.0.0.0' + } + ) { + this._shouldValidate = false; + + this._server = fastify({ + https: { + key: readFileSync(path.join('/run/secrets/tls', 'tls.key')), + cert: readFileSync(path.join('/run/secrets/tls', 'tls.crt')) + } + }); + + this._initRouting(); + } + + get shouldValidate() { + return this._shouldValidate; + } + + set shouldValidate(value: boolean) { + this._shouldValidate = value; + } + + async start() { + return new Promise((resolve, reject) => { + this._server.listen({port: this._options.port, host: this._options.host}, (err, address) => { + if (err) { + reject(err); + } + + console.log(`Server listening at ${address}`); + + this.shouldValidate = true; + + resolve(address); + }); + + }); + } + + async stop() { + this.shouldValidate = false; + + if (this._server) { + await this._server.close(); + } + } + + private async _initRouting() { + this._server.post("/validate", async (req, res): Promise => { + console.log('request', req.headers, req.body) + + const body = req.body as AdmissionRequest; + + const response = { + kind: body?.kind || '', + apiVersion: body?.apiVersion || '', + response: { + uid: body?.request?.uid || "", + allowed: true, + status: { + message: "OK" + } + } + } + + // Dev workaround - always return true for webhook server to not block hot-reload + if (body.request?.name?.startsWith('webhook-server-')) { + console.log('Allowing webhook server to pass', response); + + return response; + } + + const resourceForValidation = this._createResourceForValidation(body); + const validationResponse = await this._validator.validate({ resources: [resourceForValidation] }); + + const warnings = []; + const errors = []; + for (const result of validationResponse.runs) { + for (const item of result.results) { + if (item.level === "warning") { + warnings.push(item); + } else if (item.level === "error") { + errors.push(item); + } + } + } + + if (errors.length > 0 || warnings.length > 0) { + const warningsList = warnings.map((e) => `${e.ruleId}: ${e.message.text}`).join("\n"); + const errorsList = errors.map((e) => `${e.ruleId}: ${e.message.text}`).join("\n"); + const message = []; + + if (errors.length > 0) { + message.push(`\n${errors.length} errors found:\n${errorsList}\n`); + } + + if (warnings.length > 0) { + message.push(`\n${warnings.length} warnings found:\n${warningsList}\n`); + } + + message.push("\nYou can use Monokle (https://monokle.io/) to validate and fix those errors easily!"); + + response.response.allowed = false; + response.response.status.message = message.join(""); + } + + console.log('response', resourceForValidation, validationResponse, response); + + return response; + }); + } + + private _createResourceForValidation(admissionResource: AdmissionRequest): Resource { + const resource = { + id: admissionResource.request?.uid || '', + fileId: '', + filePath: '', + fileOffset: 0, + name: admissionResource.request?.name || '', + apiVersion: admissionResource.request?.object?.apiVersion || '', + kind: admissionResource.request?.object?.kind || '', + namespace: admissionResource.request?.namespace || '', + content: admissionResource.request?.object || {}, + text: '' + }; + + return resource; + } +} From 404ab2617afcc697a04ef5955e3d2838a96435a2 Mon Sep 17 00:00:00 2001 From: f1ames Date: Mon, 25 Sep 2023 17:25:22 +0200 Subject: [PATCH 05/26] fix: fix deployment script for local testing --- README.md | 11 +++++++++- deploy.sh | 2 ++ deployment/webhook.yaml.template | 25 ++++++++++++++++++++++- kac-api/src/index.ts | 11 ++++++---- kac-api/src/utils/informer.ts | 28 ++++++++++++++++++++++++-- kac-api/src/utils/validation-server.ts | 1 + policy.yaml | 2 +- 7 files changed, 71 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 8debb93..bae5cdb 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,7 @@ validatingwebhookconfiguration.admissionregistration.k8s.io/demo-webhook 1 You can try to create sample resource and see webhook response: ```bash -kubectl -n webhook-demo create -f examples/pod-with-conflict.yaml +kubectl -n webhook-demo create -f examples/pod-warning.yaml ``` ### Iterating @@ -70,6 +70,15 @@ skaffold dev **Important**: Skaffold will recreate deployment on every change so make sure that webhook pod doesn't get rejected by it's previous version (via Validation Admission Controller). +You can also do manual clean-up and run `./deploy.sh` script again: + +```bash +kubectl delete all -n webhook-demo --all && \ +kubectl delete validatingwebhookconfiguration.admissionregistration.k8s.io/demo-webhook -n webhook-demo && \ +kubectl delete namespace webhook-demo && \ +kubectl delete crd monoklepolicies.monokle.com +``` + ## Refs * https://kubernetes.io/blog/2019/03/21/a-guide-to-kubernetes-admission-controllers/ diff --git a/deploy.sh b/deploy.sh index e6082eb..b40466f 100755 --- a/deploy.sh +++ b/deploy.sh @@ -48,6 +48,8 @@ cp deployment/deployment.yaml.template deployment.yaml skaffold run --namespace webhook-demo # kubectl apply -f deployment.yaml sleep 2 +kubectl apply -f monokle.policy.crd.yaml +sleep 2 kubectl apply -f webhook.yaml # skaffold dev diff --git a/deployment/webhook.yaml.template b/deployment/webhook.yaml.template index 6940037..b9a91e1 100644 --- a/deployment/webhook.yaml.template +++ b/deployment/webhook.yaml.template @@ -16,4 +16,27 @@ webhooks: - operations: [ "CREATE" ] apiGroups: [""] apiVersions: ["v1"] - resources: ["pods"] \ No newline at end of file + resources: ["pods"] +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: crds-monokle-list +rules: +- apiGroups: ["monokle.com"] + resources: ["monoklepolicies"] + verbs: ["list", "watch", "get"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: crds-monokle-list +subjects: +- kind: ServiceAccount + name: default + namespace: webhook-demo +roleRef: + kind: ClusterRole + name: crds-monokle-list + apiGroup: rbac.authorization.k8s.io \ No newline at end of file diff --git a/kac-api/src/index.ts b/kac-api/src/index.ts index f091477..520c8b4 100644 --- a/kac-api/src/index.ts +++ b/kac-api/src/index.ts @@ -1,13 +1,14 @@ import {AnnotationSuppressor, FingerprintSuppressor, MonokleValidator, RemotePluginLoader, SchemaLoader, DisabledFixer, ResourceParser} from "@monokle/validation"; -import { MonoklePolicy, getInformer } from "./utils/informer"; -import { ValidationServer } from "./utils/validation-server"; +import { MonoklePolicy, getInformer } from "./utils/informer.js"; +import { ValidationServer } from "./utils/validation-server.js"; // Refs: // // kube-client get current namespace where webhook is deployed // https://stackoverflow.com/a/46046153 -const INITIAL_NAMESPACE = 'default'; +// const INITIAL_NAMESPACE = 'default'; +const INITIAL_NAMESPACE = 'webhook-demo'; (async() => { // VALIDATOR @@ -28,19 +29,21 @@ const INITIAL_NAMESPACE = 'default'; let activePolicy: MonoklePolicy | null = null; const onPolicy = async (policy: MonoklePolicy) => { + console.log('POLICY:UPDATE', policy); activePolicy = policy; awaitValidatorReadiness = validator.preload(policy.spec); await awaitValidatorReadiness; } const onPolicyRemoval = async () => { + console.log('POLICY:REMOVAL'); activePolicy = null; awaitValidatorReadiness = validator.preload({}); await awaitValidatorReadiness; } const onError = (err: any) => { - console.log('ERROR:INFROMER', err); + console.log('ERROR:INFORMER', err); } const informer = getInformer(onPolicy, onPolicy, onPolicyRemoval, INITIAL_NAMESPACE, onError); diff --git a/kac-api/src/utils/informer.ts b/kac-api/src/utils/informer.ts index b7d384a..1925e5a 100644 --- a/kac-api/src/utils/informer.ts +++ b/kac-api/src/utils/informer.ts @@ -12,8 +12,32 @@ export async function getInformer( namespace = 'default', onError?: k8s.ErrorCallback ) { - const kc = new k8s.KubeConfig() - kc.loadFromDefault() + let informer: Awaited> | null = null; + + let tries = 0; + while (!informer) { + try { + tries++; + informer = await createInformer(onAdded, onUpdated, onDeleted, namespace, onError); + return informer; + } catch (err: any) { + console.error(`INFORMER:CREATION:ERROR, try ${tries}`, err.message, err.body); + await new Promise((resolve) => setTimeout(resolve, 1000)); + } + } + + return informer; +} + +async function createInformer( + onAdded: k8s.ObjectCallback, + onUpdated: k8s.ObjectCallback, + onDeleted: k8s.ObjectCallback, + namespace = 'default', + onError?: k8s.ErrorCallback +) { + const kc = new k8s.KubeConfig(); + kc.loadFromCluster(); const k8sApi = kc.makeApiClient(k8s.CustomObjectsApi) const listFn = () => k8sApi.listNamespacedCustomObject('monokle.com','v1', namespace, 'monoklepolicies'); diff --git a/kac-api/src/utils/validation-server.ts b/kac-api/src/utils/validation-server.ts index 78e9c91..40430f8 100644 --- a/kac-api/src/utils/validation-server.ts +++ b/kac-api/src/utils/validation-server.ts @@ -41,6 +41,7 @@ export class ValidationServer { this._shouldValidate = false; this._server = fastify({ + // @TODO do not require certs when running locally (for testing outside K8s cluster) https: { key: readFileSync(path.join('/run/secrets/tls', 'tls.key')), cert: readFileSync(path.join('/run/secrets/tls', 'tls.crt')) diff --git a/policy.yaml b/policy.yaml index 8784452..724aea3 100644 --- a/policy.yaml +++ b/policy.yaml @@ -18,4 +18,4 @@ spec: open-policy-agent/memory-request: "err" settings: kubernetes-schema: - schemaVersion: v1.24.2 \ No newline at end of file + schemaVersion: v1.28.2 \ No newline at end of file From 28a1b30e26721ad4b33379b7f47e5b2ad00db04c Mon Sep 17 00:00:00 2001 From: f1ames Date: Tue, 26 Sep 2023 10:48:12 +0200 Subject: [PATCH 06/26] fix: read namespace name from within webhook pod --- kac-api/src/index.ts | 13 +++++-------- kac-api/src/utils/helpers.ts | 17 +++++++++++++++++ 2 files changed, 22 insertions(+), 8 deletions(-) create mode 100644 kac-api/src/utils/helpers.ts diff --git a/kac-api/src/index.ts b/kac-api/src/index.ts index 520c8b4..0af4046 100644 --- a/kac-api/src/index.ts +++ b/kac-api/src/index.ts @@ -1,16 +1,13 @@ import {AnnotationSuppressor, FingerprintSuppressor, MonokleValidator, RemotePluginLoader, SchemaLoader, DisabledFixer, ResourceParser} from "@monokle/validation"; import { MonoklePolicy, getInformer } from "./utils/informer.js"; import { ValidationServer } from "./utils/validation-server.js"; +import { DEFAULT_NAMESPACE, getNamespace } from "./utils/helpers.js"; -// Refs: -// -// kube-client get current namespace where webhook is deployed -// https://stackoverflow.com/a/46046153 +(async() => { + const currentNamespace = getNamespace() || DEFAULT_NAMESPACE; -// const INITIAL_NAMESPACE = 'default'; -const INITIAL_NAMESPACE = 'webhook-demo'; + console.log(`Admission Controller namespace ${currentNamespace}`); -(async() => { // VALIDATOR let awaitValidatorReadiness = Promise.resolve(); @@ -46,7 +43,7 @@ const INITIAL_NAMESPACE = 'webhook-demo'; console.log('ERROR:INFORMER', err); } - const informer = getInformer(onPolicy, onPolicy, onPolicyRemoval, INITIAL_NAMESPACE, onError); + const informer = getInformer(onPolicy, onPolicy, onPolicyRemoval, currentNamespace, onError); // SERVER const server = new ValidationServer(validator); diff --git a/kac-api/src/utils/helpers.ts b/kac-api/src/utils/helpers.ts new file mode 100644 index 0000000..1a40e67 --- /dev/null +++ b/kac-api/src/utils/helpers.ts @@ -0,0 +1,17 @@ +import { readFileSync } from "fs"; + +export const DEFAULT_NAMESPACE = 'default'; + +export function getNamespace() { + try { + // Get current namespace from inside a pod - https://stackoverflow.com/a/46046153 + const namespace = readFileSync('/var/run/secrets/kubernetes.io/serviceaccount/namespace', 'utf8'); + if (namespace.trim()) { + return namespace.trim(); + } + } + catch (err: any) { + console.error('Failed to read namespace from service account', err); + return null + } +} From c7b92b07edff041465aff9d37eb57ec5b7ffd183 Mon Sep 17 00:00:00 2001 From: f1ames Date: Tue, 26 Sep 2023 14:31:48 +0200 Subject: [PATCH 07/26] refactor: improve logging and cleanup code --- kac-api/package-lock.json | 239 +++++++++++++++++++++++-- kac-api/package.json | 3 +- kac-api/src/index.ts | 34 ++-- kac-api/src/utils/helpers.ts | 15 +- kac-api/src/utils/informer.ts | 15 +- kac-api/src/utils/validation-server.ts | 27 +-- 6 files changed, 276 insertions(+), 57 deletions(-) diff --git a/kac-api/package-lock.json b/kac-api/package-lock.json index 458e97e..cda7e46 100644 --- a/kac-api/package-lock.json +++ b/kac-api/package-lock.json @@ -15,6 +15,7 @@ "fastify": "^3.28.0", "isomorphic-fetch": "^3.0.0", "node-fetch": "^2.6.6", + "pino": "^8.15.1", "type-fest": "^3.10.0" }, "devDependencies": { @@ -190,6 +191,17 @@ "@types/node": "*" } }, + "node_modules/abort-controller": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", + "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", + "dependencies": { + "event-target-shim": "^5.0.0" + }, + "engines": { + "node": ">=6.5" + } + }, "node_modules/abstract-logging": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/abstract-logging/-/abstract-logging-2.0.1.tgz", @@ -278,6 +290,25 @@ "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, "node_modules/bcrypt-pbkdf": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", @@ -295,6 +326,29 @@ "concat-map": "0.0.1" } }, + "node_modules/buffer": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", + "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.2.1" + } + }, "node_modules/byline": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/byline/-/byline-5.0.0.tgz", @@ -462,6 +516,22 @@ "safer-buffer": "^2.1.0" } }, + "node_modules/event-target-shim": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", + "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==", + "engines": { + "node": ">=6" + } + }, + "node_modules/events": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", + "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", + "engines": { + "node": ">=0.8.x" + } + }, "node_modules/extend": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", @@ -545,6 +615,37 @@ "tiny-lru": "^8.0.1" } }, + "node_modules/fastify/node_modules/pino": { + "version": "6.14.0", + "resolved": "https://registry.npmjs.org/pino/-/pino-6.14.0.tgz", + "integrity": "sha512-iuhEDel3Z3hF9Jfe44DPXR8l07bhjuFY3GMHIXbjnY9XcafbyDDwl2sN2vw2GjMPf5Nkoe+OFao7ffn9SXaKDg==", + "dependencies": { + "fast-redact": "^3.0.0", + "fast-safe-stringify": "^2.0.8", + "flatstr": "^1.0.12", + "pino-std-serializers": "^3.1.0", + "process-warning": "^1.0.0", + "quick-format-unescaped": "^4.0.3", + "sonic-boom": "^1.0.2" + }, + "bin": { + "pino": "bin.js" + } + }, + "node_modules/fastify/node_modules/pino-std-serializers": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/pino-std-serializers/-/pino-std-serializers-3.2.0.tgz", + "integrity": "sha512-EqX4pwDPrt3MuOAAUBMU0Tk5kR/YcCM5fNPEzgCO2zJ5HfX0vbiH9HbJglnyeQsN96Kznae6MWD47pZB5avTrg==" + }, + "node_modules/fastify/node_modules/sonic-boom": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-1.4.1.tgz", + "integrity": "sha512-LRHh/A8tpW7ru89lrlkU4AszXt1dbwSjVWguGrmlxE7tawVmDBlI1PILMkXAxJTwqhgsEeTHzj36D5CmHgQmNg==", + "dependencies": { + "atomic-sleep": "^1.0.0", + "flatstr": "^1.0.12" + } + }, "node_modules/fastq": { "version": "1.15.0", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.15.0.tgz", @@ -745,6 +846,25 @@ "npm": ">=1.3.7" } }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, "node_modules/inflight": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", @@ -1060,6 +1180,11 @@ "node": "^10.13.0 || >=12.0.0" } }, + "node_modules/on-exit-leak-free": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/on-exit-leak-free/-/on-exit-leak-free-2.1.0.tgz", + "integrity": "sha512-VuCaZZAjReZ3vUwgOB8LxAosIurDiAW0s13rI1YwmaP++jvcxP77AWoQvenZebpCA2m8WC1/EosPYPMjnRAp/w==" + }, "node_modules/once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", @@ -1129,26 +1254,52 @@ "integrity": "sha512-7EAHlyLHI56VEIdK57uwHdHKIaAGbnXPiw0yWbarQZOKaKpvUIgW0jWRVLiatnM+XXlSwsanIBH/hzGMJulMow==" }, "node_modules/pino": { - "version": "6.14.0", - "resolved": "https://registry.npmjs.org/pino/-/pino-6.14.0.tgz", - "integrity": "sha512-iuhEDel3Z3hF9Jfe44DPXR8l07bhjuFY3GMHIXbjnY9XcafbyDDwl2sN2vw2GjMPf5Nkoe+OFao7ffn9SXaKDg==", + "version": "8.15.1", + "resolved": "https://registry.npmjs.org/pino/-/pino-8.15.1.tgz", + "integrity": "sha512-Cp4QzUQrvWCRJaQ8Lzv0mJzXVk4z2jlq8JNKMGaixC2Pz5L4l2p95TkuRvYbrEbe85NQsDKrAd4zalf7Ml6WiA==", "dependencies": { - "fast-redact": "^3.0.0", - "fast-safe-stringify": "^2.0.8", - "flatstr": "^1.0.12", - "pino-std-serializers": "^3.1.0", - "process-warning": "^1.0.0", + "atomic-sleep": "^1.0.0", + "fast-redact": "^3.1.1", + "on-exit-leak-free": "^2.1.0", + "pino-abstract-transport": "v1.1.0", + "pino-std-serializers": "^6.0.0", + "process-warning": "^2.0.0", "quick-format-unescaped": "^4.0.3", - "sonic-boom": "^1.0.2" + "real-require": "^0.2.0", + "safe-stable-stringify": "^2.3.1", + "sonic-boom": "^3.1.0", + "thread-stream": "^2.0.0" }, "bin": { "pino": "bin.js" } }, + "node_modules/pino-abstract-transport": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/pino-abstract-transport/-/pino-abstract-transport-1.1.0.tgz", + "integrity": "sha512-lsleG3/2a/JIWUtf9Q5gUNErBqwIu1tUKTT3dUzaf5DySw9ra1wcqKjJjLX1VTY64Wk1eEOYsVGSaGfCK85ekA==", + "dependencies": { + "readable-stream": "^4.0.0", + "split2": "^4.0.0" + } + }, "node_modules/pino-std-serializers": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/pino-std-serializers/-/pino-std-serializers-3.2.0.tgz", - "integrity": "sha512-EqX4pwDPrt3MuOAAUBMU0Tk5kR/YcCM5fNPEzgCO2zJ5HfX0vbiH9HbJglnyeQsN96Kznae6MWD47pZB5avTrg==" + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/pino-std-serializers/-/pino-std-serializers-6.2.2.tgz", + "integrity": "sha512-cHjPPsE+vhj/tnhCy/wiMh3M3z3h/j15zHQX+S9GkTBgqJuTuJzYJ4gUyACLhDaJ7kk9ba9iRDmbH2tJU03OiA==" + }, + "node_modules/pino/node_modules/process-warning": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/process-warning/-/process-warning-2.2.0.tgz", + "integrity": "sha512-/1WZ8+VQjR6avWOgHeEPd7SDQmFQ1B5mC1eRXsCm5TarlNmx/wCsa5GEaxGm05BORRtyG/Ex/3xq3TuRvq57qg==" + }, + "node_modules/process": { + "version": "0.11.10", + "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz", + "integrity": "sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==", + "engines": { + "node": ">= 0.6.0" + } }, "node_modules/process-warning": { "version": "1.0.0", @@ -1212,6 +1363,29 @@ "resolved": "https://registry.npmjs.org/quick-format-unescaped/-/quick-format-unescaped-4.0.4.tgz", "integrity": "sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg==" }, + "node_modules/readable-stream": { + "version": "4.4.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.4.2.tgz", + "integrity": "sha512-Lk/fICSyIhodxy1IDK2HazkeGjSmezAWX2egdtJnYhtzKEsBPJowlI6F6LPb5tqIQILrMbx22S5o3GuJavPusA==", + "dependencies": { + "abort-controller": "^3.0.0", + "buffer": "^6.0.3", + "events": "^3.3.0", + "process": "^0.11.10", + "string_decoder": "^1.3.0" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + } + }, + "node_modules/real-require": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/real-require/-/real-require-0.2.0.tgz", + "integrity": "sha512-57frrGM/OCTLqLOAh0mhVA9VBMHd+9U7Zb2THMGdBUoZVOtGbJzjxsYGDJ3A9AYYCP4hn6y1TVbaOfzWtm5GFg==", + "engines": { + "node": ">= 12.13.0" + } + }, "node_modules/request": { "version": "2.88.2", "resolved": "https://registry.npmjs.org/request/-/request-2.88.2.tgz", @@ -1356,6 +1530,14 @@ "ret": "~0.2.0" } }, + "node_modules/safe-stable-stringify": { + "version": "2.4.3", + "resolved": "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.4.3.tgz", + "integrity": "sha512-e2bDA2WJT0wxseVd4lsDP4+3ONX6HpMXQa1ZhFQ7SU+GjvORCmShbCMltrtIDfkYhVHrOcPtj+KhmDBdPdZD1g==", + "engines": { + "node": ">=10" + } + }, "node_modules/safer-buffer": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", @@ -1410,12 +1592,19 @@ } }, "node_modules/sonic-boom": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-1.4.1.tgz", - "integrity": "sha512-LRHh/A8tpW7ru89lrlkU4AszXt1dbwSjVWguGrmlxE7tawVmDBlI1PILMkXAxJTwqhgsEeTHzj36D5CmHgQmNg==", + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-3.3.0.tgz", + "integrity": "sha512-LYxp34KlZ1a2Jb8ZQgFCK3niIHzibdwtwNUWKg0qQRzsDoJ3Gfgkf8KdBTFU3SkejDEIlWwnSnpVdOZIhFMl/g==", "dependencies": { - "atomic-sleep": "^1.0.0", - "flatstr": "^1.0.12" + "atomic-sleep": "^1.0.0" + } + }, + "node_modules/split2": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", + "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", + "engines": { + "node": ">= 10.x" } }, "node_modules/sprintf-js": { @@ -1455,6 +1644,14 @@ "node": ">= 0.10.0" } }, + "node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, "node_modules/string-similarity": { "version": "4.0.4", "resolved": "https://registry.npmjs.org/string-similarity/-/string-similarity-4.0.4.tgz", @@ -1477,6 +1674,14 @@ "node": ">=10" } }, + "node_modules/thread-stream": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/thread-stream/-/thread-stream-2.4.0.tgz", + "integrity": "sha512-xZYtOtmnA63zj04Q+F9bdEay5r47bvpo1CaNqsKi7TpoJHcotUez8Fkfo2RJWpW91lnnaApdpRbVwCWsy+ifcw==", + "dependencies": { + "real-require": "^0.2.0" + } + }, "node_modules/tiny-lru": { "version": "8.0.2", "resolved": "https://registry.npmjs.org/tiny-lru/-/tiny-lru-8.0.2.tgz", diff --git a/kac-api/package.json b/kac-api/package.json index 900299a..78e65d1 100644 --- a/kac-api/package.json +++ b/kac-api/package.json @@ -13,11 +13,12 @@ "license": "ISC", "dependencies": { "@kubernetes/client-node": "^0.18.1", - "@monokle/validation": "^0.30.1", "@monokle/parser": "0.2.0", + "@monokle/validation": "^0.30.1", "fastify": "^3.28.0", "isomorphic-fetch": "^3.0.0", "node-fetch": "^2.6.6", + "pino": "^8.15.1", "type-fest": "^3.10.0" }, "devDependencies": { diff --git a/kac-api/src/index.ts b/kac-api/src/index.ts index 0af4046..9eac191 100644 --- a/kac-api/src/index.ts +++ b/kac-api/src/index.ts @@ -1,12 +1,24 @@ -import {AnnotationSuppressor, FingerprintSuppressor, MonokleValidator, RemotePluginLoader, SchemaLoader, DisabledFixer, ResourceParser} from "@monokle/validation"; -import { MonoklePolicy, getInformer } from "./utils/informer.js"; -import { ValidationServer } from "./utils/validation-server.js"; -import { DEFAULT_NAMESPACE, getNamespace } from "./utils/helpers.js"; +import pino from 'pino'; +import {AnnotationSuppressor, FingerprintSuppressor, MonokleValidator, RemotePluginLoader, SchemaLoader, DisabledFixer, ResourceParser} from '@monokle/validation'; +import {MonoklePolicy, getInformer} from './utils/informer.js'; +import {ValidationServer} from './utils/validation-server.js'; +import {DEFAULT_NAMESPACE, getNamespace} from './utils/helpers.js'; (async() => { - const currentNamespace = getNamespace() || DEFAULT_NAMESPACE; + const logger = pino({ + name: 'Monokle', + level: 'trace', + }); - console.log(`Admission Controller namespace ${currentNamespace}`); + let currentNamespace: string; + try { + currentNamespace = getNamespace() || DEFAULT_NAMESPACE; + } catch (err: any) { + logger.error('Failed to get current namespace', err); + process.exit(1); + } + + logger.debug({namespace: currentNamespace}); // VALIDATOR let awaitValidatorReadiness = Promise.resolve(); @@ -26,27 +38,27 @@ import { DEFAULT_NAMESPACE, getNamespace } from "./utils/helpers.js"; let activePolicy: MonoklePolicy | null = null; const onPolicy = async (policy: MonoklePolicy) => { - console.log('POLICY:UPDATE', policy); + logger.info({msg: 'Informer: Policy updated', policy}); activePolicy = policy; awaitValidatorReadiness = validator.preload(policy.spec); await awaitValidatorReadiness; } const onPolicyRemoval = async () => { - console.log('POLICY:REMOVAL'); + logger.info('Informer: Policy removed'); activePolicy = null; awaitValidatorReadiness = validator.preload({}); await awaitValidatorReadiness; } const onError = (err: any) => { - console.log('ERROR:INFORMER', err); + logger.error({msg: 'Informer: Error', err}); } - const informer = getInformer(onPolicy, onPolicy, onPolicyRemoval, currentNamespace, onError); + const informer = getInformer(onPolicy, onPolicy, onPolicyRemoval, onError, currentNamespace); // SERVER - const server = new ValidationServer(validator); + const server = new ValidationServer(validator, logger); await server.start(); })(); diff --git a/kac-api/src/utils/helpers.ts b/kac-api/src/utils/helpers.ts index 1a40e67..f9fa790 100644 --- a/kac-api/src/utils/helpers.ts +++ b/kac-api/src/utils/helpers.ts @@ -1,17 +1,8 @@ -import { readFileSync } from "fs"; +import {readFileSync} from "fs"; export const DEFAULT_NAMESPACE = 'default'; export function getNamespace() { - try { - // Get current namespace from inside a pod - https://stackoverflow.com/a/46046153 - const namespace = readFileSync('/var/run/secrets/kubernetes.io/serviceaccount/namespace', 'utf8'); - if (namespace.trim()) { - return namespace.trim(); - } - } - catch (err: any) { - console.error('Failed to read namespace from service account', err); - return null - } + // Get current namespace from inside a pod - https://stackoverflow.com/a/46046153 + return readFileSync('/var/run/secrets/kubernetes.io/serviceaccount/namespace', 'utf8').trim(); } diff --git a/kac-api/src/utils/informer.ts b/kac-api/src/utils/informer.ts index 1925e5a..e292cf3 100644 --- a/kac-api/src/utils/informer.ts +++ b/kac-api/src/utils/informer.ts @@ -1,16 +1,18 @@ import k8s from '@kubernetes/client-node'; -import { Config } from '@monokle/validation'; +import {Config} from '@monokle/validation'; export type MonoklePolicy = k8s.KubernetesObject & { spec: Config }; +const ERROR_RESTART_INTERVAL = 500; + export async function getInformer( onAdded: k8s.ObjectCallback, onUpdated: k8s.ObjectCallback, onDeleted: k8s.ObjectCallback, + onError: k8s.ErrorCallback, namespace = 'default', - onError?: k8s.ErrorCallback ) { let informer: Awaited> | null = null; @@ -21,8 +23,11 @@ export async function getInformer( informer = await createInformer(onAdded, onUpdated, onDeleted, namespace, onError); return informer; } catch (err: any) { - console.error(`INFORMER:CREATION:ERROR, try ${tries}`, err.message, err.body); - await new Promise((resolve) => setTimeout(resolve, 1000)); + if (onError) { + onError(err); + } + + await new Promise((resolve) => setTimeout(resolve, ERROR_RESTART_INTERVAL)); } } @@ -54,7 +59,7 @@ async function createInformer( setTimeout(async () => { await informer.start(); - }, 1000); + }, ERROR_RESTART_INTERVAL); }); await informer.start(); diff --git a/kac-api/src/utils/validation-server.ts b/kac-api/src/utils/validation-server.ts index 40430f8..14fe3a9 100644 --- a/kac-api/src/utils/validation-server.ts +++ b/kac-api/src/utils/validation-server.ts @@ -1,7 +1,8 @@ import fastify from "fastify"; +import pino from 'pino'; import path from "path"; -import { readFileSync } from "fs"; -import { MonokleValidator, Resource } from "@monokle/validation"; +import {readFileSync} from "fs"; +import {MonokleValidator, Resource} from "@monokle/validation"; import {V1ValidatingWebhookConfiguration, V1ObjectMeta} from "@kubernetes/client-node"; export type ValidationServerOptions = { @@ -9,13 +10,13 @@ export type ValidationServerOptions = { host: string; }; -type AdmissionRequest = V1ValidatingWebhookConfiguration & { +export type AdmissionRequest = V1ValidatingWebhookConfiguration & { request?: V1ObjectMeta & { object?: Resource } -} +}; -type AdmissionResponse = { +export type AdmissionResponse = { kind: string, apiVersion: string, response: { @@ -25,7 +26,7 @@ type AdmissionResponse = { message: string } } -} +}; export class ValidationServer { private _server: ReturnType; @@ -33,6 +34,7 @@ export class ValidationServer { constructor( private readonly _validator: MonokleValidator, + private readonly _logger: ReturnType, private readonly _options: ValidationServerOptions = { port: 8443, host: '0.0.0.0' @@ -66,7 +68,7 @@ export class ValidationServer { reject(err); } - console.log(`Server listening at ${address}`); + this._logger.info(`Server listening at ${address}`); this.shouldValidate = true; @@ -85,8 +87,10 @@ export class ValidationServer { } private async _initRouting() { - this._server.post("/validate", async (req, res): Promise => { - console.log('request', req.headers, req.body) + this._server.post("/validate", async (req, _res): Promise => { + + this._logger.debug({request: req}) + this._logger.trace({requestBody: req.body}); const body = req.body as AdmissionRequest; @@ -104,7 +108,7 @@ export class ValidationServer { // Dev workaround - always return true for webhook server to not block hot-reload if (body.request?.name?.startsWith('webhook-server-')) { - console.log('Allowing webhook server to pass', response); + this._logger.debug({msg: 'Allowing webhook server to pass', response}); return response; } @@ -143,7 +147,8 @@ export class ValidationServer { response.response.status.message = message.join(""); } - console.log('response', resourceForValidation, validationResponse, response); + this._logger.debug({response}); + this._logger.trace({resourceForValidation, validationResponse}); return response; }); From a723b2ebafa416ebbe1283c303ae121c20233493 Mon Sep 17 00:00:00 2001 From: f1ames Date: Wed, 27 Sep 2023 10:53:20 +0200 Subject: [PATCH 08/26] chore: update README file --- README.md | 110 ++++++++++++++++++++------------------------------ README.old.md | 98 -------------------------------------------- 2 files changed, 43 insertions(+), 165 deletions(-) delete mode 100644 README.old.md diff --git a/README.md b/README.md index bae5cdb..0b937da 100644 --- a/README.md +++ b/README.md @@ -1,19 +1,19 @@ -# K8s Admission Controller Demo +# Monokle Admission Controller -This was heavily inspired by https://github.com/stackrox/admission-controller-webhook-demo. +Monokle Admission Controller is an admission controller for validating resources in the cluster. -And since it is hackaton PoC there was massive amount of duct tape applied in some places ;) +## Development -## Prerequisites +### Prerequisites * Minikube (or any other K8s cluster running) * kubectl * Skaffold * nodejs -## Running +### Running -### Minikube +#### Minikube Start Minikube: @@ -21,46 +21,67 @@ Start Minikube: minikube start --uuid 00000000-0000-0000-0000-000000000001 --extra-config=apiserver.enable-admission-plugins=ValidatingAdmissionWebhook ``` -Every resource will be deployed to `webhook-demo` namespace, to watch it you can run: +#### Deploying ```bash -watch kubectl -n webhook-demo get all,ValidatingWebhookConfiguration,MutatingWebhookConfiguration +./deploy.sh ``` -### Deploying +Every resource will be deployed to `webhook-demo` namespace, to watch it you can run: ```bash -./deploy.sh +watch kubectl -n webhook-demo get all,CustomResourceDefinition,ValidatingWebhookConfiguration,MutatingWebhookConfiguration ``` After it runs, the result should be something like: ```bash NAME READY STATUS RESTARTS AGE -pod/webhook-server-55dd5d6f44-lwwnw 1/1 Running 0 11s +pod/webhook-server-677556956c-f7hcq 1/1 Running 0 3m54s NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE -service/webhook-server ClusterIP 10.96.18.123 443/TCP 11s +service/webhook-server ClusterIP 10.105.249.5 443/TCP 3m54s NAME READY UP-TO-DATE AVAILABLE AGE -deployment.apps/webhook-server 1/1 1 1 11s +deployment.apps/webhook-server 1/1 1 1 3m54s NAME DESIRED CURRENT READY AGE -replicaset.apps/webhook-server-55dd5d6f44 1 1 1 11s +replicaset.apps/webhook-server-677556956c 1 1 1 3m54s + +NAME CREATED AT +customresourcedefinition.apiextensions.k8s.io/monoklepolicies.monokle.com 2023-09-27T08:45:13Z NAME WEBHOOKS AGE -validatingwebhookconfiguration.admissionregistration.k8s.io/demo-webhook 1 6s +validatingwebhookconfiguration.admissionregistration.k8s.io/demo-webhook 1 3m46s +``` + +For getting info about CRDs: + +```bash +kubectl get crd +kubectl describe crd monoklepolicies.monokle.com + +kubectl get monoklepolicy -n webhook-demo +kubectl describe monoklepolicy policy-sample -n webhook-demo ``` -### Testing +#### Testing -You can try to create sample resource and see webhook response: +First you need to create policy resource, for example: + +```bash +kubectl -n webhook-demo apply -f policy.yaml +``` + +> Admission controller will still work without policy resource but then it will be like running validation with all plugins disabled. + +Then you can try to create sample resource and see webhook response: ```bash kubectl -n webhook-demo create -f examples/pod-warning.yaml ``` -### Iterating +#### Iterating After everything is running you can allow hot-reload by running: @@ -70,7 +91,7 @@ skaffold dev **Important**: Skaffold will recreate deployment on every change so make sure that webhook pod doesn't get rejected by it's previous version (via Validation Admission Controller). -You can also do manual clean-up and run `./deploy.sh` script again: +You can also do manual clean-up and re-run `./deploy.sh` script again: ```bash kubectl delete all -n webhook-demo --all && \ @@ -79,56 +100,11 @@ kubectl delete namespace webhook-demo && \ kubectl delete crd monoklepolicies.monokle.com ``` -## Refs +### Refs * https://kubernetes.io/blog/2019/03/21/a-guide-to-kubernetes-admission-controllers/ * https://github.com/stackrox/admission-controller-webhook-demo/tree/master * https://www.witodelnat.eu/blog/2021/local-kubernetes-development * https://minikube.sigs.k8s.io/docs/tutorials/using_psp/ - -## Policy as CRDs - -> https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/ -> https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/ - -1. Start minikube. -1. Apply resource definition: - -```bash -kubectl apply -f monokle.policy.crd.yaml -``` - -3. Test if it was applied correctly: - -```bash -kubectl get crd -kubectl get monoklepolicy -kubectl describe crd monoklepolicies.monokle.com -``` - -4. Create sample policy resource: - -```bash -kubectl apply -f policy.yaml -``` - -5. Test if it was applied correctly: - -```bash -kubectl get monoklepolicy -kubectl describe monoklepolicy policy-sample -``` - -### CRDs Informer - -You can test how watching CRDs changes works by running: - -```bash -node kac-api/scripts/list-policies.js -``` - -and (re)applying sample policy: - -```bash -kubectl apply -f policy.yaml -``` +* https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/ +* https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/ diff --git a/README.old.md b/README.old.md deleted file mode 100644 index ed85948..0000000 --- a/README.old.md +++ /dev/null @@ -1,98 +0,0 @@ -# Kubernetes Admission Controller Webhook Demo - -This repository contains a small HTTP server that can be used as a Kubernetes -[MutatingAdmissionWebhook](https://kubernetes.io/docs/admin/admission-controllers/#mutatingadmissionwebhook-beta-in-19). - -The logic of this demo webhook is fairly simple: it enforces more secure defaults for running -containers as non-root user. While it is still possible to run containers as root, the webhook -ensures that this is only possible if the setting `runAsNonRoot` is *explicitly* set to `false` -in the `securityContext` of the Pod. If no value is set for `runAsNonRoot`, a default of `true` -is applied, and the user ID defaults to `1234`. - -## Prerequisites - -A cluster on which this example can be tested must be running Kubernetes 1.9.0 or above, -with the `admissionregistration.k8s.io/v1beta1` API enabled. You can verify that by observing that the -following command produces a non-empty output: -``` -kubectl api-versions | grep admissionregistration.k8s.io/v1beta1 -``` -In addition, the `MutatingAdmissionWebhook` admission controller should be added and listed in the admission-control -flag of `kube-apiserver`. - -For building the image, [GNU make](https://www.gnu.org/software/make/) and [Go](https://golang.org) are required. - -## Deploying the Webhook Server - -1. Bring up a Kubernetes cluster satisfying the above prerequisites, and make -sure it is active (i.e., either via the configuration in the default location, or by setting -the `KUBECONFIG` environment variable). -2. Run `./deploy.sh`. This will create a CA, a certificate and private key for the webhook server, -and deploy the resources in the newly created `webhook-demo` namespace in your Kubernetes cluster. - - -## Verify - -1. The `webhook-server` pod in the `webhook-demo` namespace should be running: -``` -$ kubectl -n webhook-demo get pods -NAME READY STATUS RESTARTS AGE -webhook-server-6f976f7bf-hssc9 1/1 Running 0 35m -``` - -2. A `MutatingWebhookConfiguration` named `demo-webhook` should exist: -``` -$ kubectl get mutatingwebhookconfigurations -NAME AGE -demo-webhook 36m -``` - -3. Deploy [a pod](examples/pod-with-defaults.yaml) that neither sets `runAsNonRoot` nor `runAsUser`: -``` -$ kubectl create -f examples/pod-with-defaults.yaml -``` -Verify that the pod has default values in its security context filled in: -``` -$ kubectl get pod/pod-with-defaults -o yaml -... - securityContext: - runAsNonRoot: true - runAsUser: 1234 -... -``` -Also, check the logs that the pod had in fact been running as a non-root user: -``` -$ kubectl logs pod-with-defaults -I am running as user 1234 -``` - -4. Deploy [a pod](examples/pod-with-override.yaml) that explicitly sets `runAsNonRoot` to `false`, allowing it to run as the -`root` user: -``` -$ kubectl create -f examples/pod-with-override.yaml -$ kubectl get pod/pod-with-override -o yaml -... - securityContext: - runAsNonRoot: false -... -$ kubectl logs pod-with-override -I am running as user 0 -``` - -5. Attempt to deploy [a pod](examples/pod-with-conflict.yaml) that has a conflicting setting: `runAsNonRoot` set to `true`, but `runAsUser` set to 0 (root). -The admission controller should block the creation of that pod. -``` -$ kubectl create -f examples/pod-with-conflict.yaml -Error from server (InternalError): error when creating "examples/pod-with-conflict.yaml": Internal error -occurred: admission webhook "webhook-server.webhook-demo.svc" denied the request: runAsNonRoot specified, -but runAsUser set to 0 (the root user) -``` - -## Build the Image from Sources (optional) - -An image can be built by running `make`. -If you want to modify the webhook server for testing purposes, be sure to set and export -the shell environment variable `IMAGE` to an image tag for which you have push access. You can then -build and push the image by running `make push-image`. Also make sure to change the image tag -in `deployment/deployment.yaml.template`, and if necessary, add image pull secrets. - From 7cc0f97fedb5d8c24d948463235118d7a1fbb827 Mon Sep 17 00:00:00 2001 From: f1ames Date: Wed, 27 Sep 2023 11:37:10 +0200 Subject: [PATCH 09/26] refactor: rework project structure --- .gitignore | 4 +- Dockerfile | 6 +-- README.md | 6 +-- {kac-api => admission-webhook}/.gitignore | 0 .../package-lock.json | 4 +- {kac-api => admission-webhook}/package.json | 2 +- {kac-api => admission-webhook}/src/index.ts | 0 .../src/utils/helpers.ts | 0 .../src/utils/informer.ts | 0 .../src/utils/validation-server.ts | 0 {kac-api => admission-webhook}/tsconfig.json | 0 deployment/webhook.yaml.template | 42 ------------------- policy.yaml => examples/policy-sample.yaml | 0 .../manifests/crd.yaml | 0 k8s/manifests/service-account.yaml | 27 ++++++++++++ skaffold.yaml => k8s/skaffold.yaml | 4 +- .../templates}/deployment.yaml.template | 2 +- k8s/templates/webhook.yaml.template | 19 +++++++++ kac-api/scripts/list-policies.js | 32 -------------- deploy.sh => scripts/deploy.sh | 20 ++++----- {deployment => scripts}/generate-keys.sh | 0 21 files changed, 70 insertions(+), 98 deletions(-) rename {kac-api => admission-webhook}/.gitignore (100%) rename {kac-api => admission-webhook}/package-lock.json (99%) rename {kac-api => admission-webhook}/package.json (95%) rename {kac-api => admission-webhook}/src/index.ts (100%) rename {kac-api => admission-webhook}/src/utils/helpers.ts (100%) rename {kac-api => admission-webhook}/src/utils/informer.ts (100%) rename {kac-api => admission-webhook}/src/utils/validation-server.ts (100%) rename {kac-api => admission-webhook}/tsconfig.json (100%) delete mode 100644 deployment/webhook.yaml.template rename policy.yaml => examples/policy-sample.yaml (100%) rename monokle.policy.crd.yaml => k8s/manifests/crd.yaml (100%) create mode 100644 k8s/manifests/service-account.yaml rename skaffold.yaml => k8s/skaffold.yaml (78%) rename {deployment => k8s/templates}/deployment.yaml.template (96%) create mode 100644 k8s/templates/webhook.yaml.template delete mode 100644 kac-api/scripts/list-policies.js rename deploy.sh => scripts/deploy.sh (76%) rename {deployment => scripts}/generate-keys.sh (100%) diff --git a/.gitignore b/.gitignore index 951fdc2..89f5237 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,3 @@ /.idea/ -webhook.yaml -deployment.yaml \ No newline at end of file +k8s/manifests/webhook.yaml +k8s/manifests/deployment.yaml \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index e4f4bbf..dd231e0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,11 @@ FROM node:18.16.0-bullseye-slim WORKDIR /workdir -COPY kac-api/package*.json ./ +COPY admission-webhook/package*.json ./ RUN npm ci --ignore-scripts -COPY ./kac-api/src ./src -COPY ./kac-api/tsconfig.json ./ +COPY ./admission-webhook/src ./src +COPY ./admission-webhook/tsconfig.json ./ CMD ["npm", "run", "prod"] EXPOSE 8443 diff --git a/README.md b/README.md index 0b937da..04b5ca6 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ minikube start --uuid 00000000-0000-0000-0000-000000000001 --extra-config=apiser #### Deploying ```bash -./deploy.sh +./scripts/deploy.sh ``` Every resource will be deployed to `webhook-demo` namespace, to watch it you can run: @@ -70,7 +70,7 @@ kubectl describe monoklepolicy policy-sample -n webhook-demo First you need to create policy resource, for example: ```bash -kubectl -n webhook-demo apply -f policy.yaml +kubectl -n webhook-demo apply -f examples/policy-sample.yaml ``` > Admission controller will still work without policy resource but then it will be like running validation with all plugins disabled. @@ -86,7 +86,7 @@ kubectl -n webhook-demo create -f examples/pod-warning.yaml After everything is running you can allow hot-reload by running: ```bash -skaffold dev +skaffold dev -f k8s/skaffold.yaml ``` **Important**: Skaffold will recreate deployment on every change so make sure that webhook pod doesn't get rejected by it's previous version (via Validation Admission Controller). diff --git a/kac-api/.gitignore b/admission-webhook/.gitignore similarity index 100% rename from kac-api/.gitignore rename to admission-webhook/.gitignore diff --git a/kac-api/package-lock.json b/admission-webhook/package-lock.json similarity index 99% rename from kac-api/package-lock.json rename to admission-webhook/package-lock.json index cda7e46..603da6c 100644 --- a/kac-api/package-lock.json +++ b/admission-webhook/package-lock.json @@ -1,11 +1,11 @@ { - "name": "kac2", + "name": "admission-webhook", "version": "1.0.0", "lockfileVersion": 3, "requires": true, "packages": { "": { - "name": "kac2", + "name": "admission-webhook", "version": "1.0.0", "license": "ISC", "dependencies": { diff --git a/kac-api/package.json b/admission-webhook/package.json similarity index 95% rename from kac-api/package.json rename to admission-webhook/package.json index 78e65d1..f760246 100644 --- a/kac-api/package.json +++ b/admission-webhook/package.json @@ -1,5 +1,5 @@ { - "name": "kac2", + "name": "admission-webhook", "version": "1.0.0", "description": "", "main": "dist/index.js", diff --git a/kac-api/src/index.ts b/admission-webhook/src/index.ts similarity index 100% rename from kac-api/src/index.ts rename to admission-webhook/src/index.ts diff --git a/kac-api/src/utils/helpers.ts b/admission-webhook/src/utils/helpers.ts similarity index 100% rename from kac-api/src/utils/helpers.ts rename to admission-webhook/src/utils/helpers.ts diff --git a/kac-api/src/utils/informer.ts b/admission-webhook/src/utils/informer.ts similarity index 100% rename from kac-api/src/utils/informer.ts rename to admission-webhook/src/utils/informer.ts diff --git a/kac-api/src/utils/validation-server.ts b/admission-webhook/src/utils/validation-server.ts similarity index 100% rename from kac-api/src/utils/validation-server.ts rename to admission-webhook/src/utils/validation-server.ts diff --git a/kac-api/tsconfig.json b/admission-webhook/tsconfig.json similarity index 100% rename from kac-api/tsconfig.json rename to admission-webhook/tsconfig.json diff --git a/deployment/webhook.yaml.template b/deployment/webhook.yaml.template deleted file mode 100644 index b9a91e1..0000000 --- a/deployment/webhook.yaml.template +++ /dev/null @@ -1,42 +0,0 @@ -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingWebhookConfiguration -metadata: - name: demo-webhook -webhooks: - - name: webhook-server.webhook-demo.svc - sideEffects: None - admissionReviewVersions: ["v1", "v1beta1"] - clientConfig: - service: - name: webhook-server - namespace: webhook-demo - path: "/validate" - caBundle: ${CA_PEM_B64} - rules: - - operations: [ "CREATE" ] - apiGroups: [""] - apiVersions: ["v1"] - resources: ["pods"] ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: crds-monokle-list -rules: -- apiGroups: ["monokle.com"] - resources: ["monoklepolicies"] - verbs: ["list", "watch", "get"] - ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: crds-monokle-list -subjects: -- kind: ServiceAccount - name: default - namespace: webhook-demo -roleRef: - kind: ClusterRole - name: crds-monokle-list - apiGroup: rbac.authorization.k8s.io \ No newline at end of file diff --git a/policy.yaml b/examples/policy-sample.yaml similarity index 100% rename from policy.yaml rename to examples/policy-sample.yaml diff --git a/monokle.policy.crd.yaml b/k8s/manifests/crd.yaml similarity index 100% rename from monokle.policy.crd.yaml rename to k8s/manifests/crd.yaml diff --git a/k8s/manifests/service-account.yaml b/k8s/manifests/service-account.yaml new file mode 100644 index 0000000..693e69f --- /dev/null +++ b/k8s/manifests/service-account.yaml @@ -0,0 +1,27 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: crds-monokle-list +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: crds-monokle-list +rules: +- apiGroups: ["monokle.com"] + resources: ["monoklepolicies"] + verbs: ["list", "watch", "get"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: crds-monokle-list +subjects: +- kind: ServiceAccount + name: default + namespace: webhook-demo +roleRef: + kind: ClusterRole + name: crds-monokle-list + apiGroup: rbac.authorization.k8s.io \ No newline at end of file diff --git a/skaffold.yaml b/k8s/skaffold.yaml similarity index 78% rename from skaffold.yaml rename to k8s/skaffold.yaml index 934df1c..db22fb8 100644 --- a/skaffold.yaml +++ b/k8s/skaffold.yaml @@ -4,12 +4,12 @@ metadata: name: webhook-server build: artifacts: - - image: kac-api + - image: admission-webhook docker: dockerfile: Dockerfile manifests: rawYaml: - - deployment.yaml + - ./k8s/manifests/deployment.yaml portForward: - resourceType: service resourceName: webhook-server diff --git a/deployment/deployment.yaml.template b/k8s/templates/deployment.yaml.template similarity index 96% rename from deployment/deployment.yaml.template rename to k8s/templates/deployment.yaml.template index 0eab556..c8587cf 100644 --- a/deployment/deployment.yaml.template +++ b/k8s/templates/deployment.yaml.template @@ -20,7 +20,7 @@ spec: # runAsUser: 1234 containers: - name: server - image: kac-api + image: admission-webhook #imagePullPolicy: Always ports: - containerPort: 8443 diff --git a/k8s/templates/webhook.yaml.template b/k8s/templates/webhook.yaml.template new file mode 100644 index 0000000..6940037 --- /dev/null +++ b/k8s/templates/webhook.yaml.template @@ -0,0 +1,19 @@ +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: demo-webhook +webhooks: + - name: webhook-server.webhook-demo.svc + sideEffects: None + admissionReviewVersions: ["v1", "v1beta1"] + clientConfig: + service: + name: webhook-server + namespace: webhook-demo + path: "/validate" + caBundle: ${CA_PEM_B64} + rules: + - operations: [ "CREATE" ] + apiGroups: [""] + apiVersions: ["v1"] + resources: ["pods"] \ No newline at end of file diff --git a/kac-api/scripts/list-policies.js b/kac-api/scripts/list-policies.js deleted file mode 100644 index d96fc2b..0000000 --- a/kac-api/scripts/list-policies.js +++ /dev/null @@ -1,32 +0,0 @@ -import k8s from '@kubernetes/client-node'; - -const kc = new k8s.KubeConfig() -kc.loadFromDefault() - -const k8sApi = kc.makeApiClient(k8s.CustomObjectsApi) -const listFn = () => k8sApi.listNamespacedCustomObject('monokle.com','v1', 'default', 'monoklepolicies'); -const informer = k8s.makeInformer(kc, '/apis/monokle.com/v1/namespaces/default/monoklepolicies', listFn); - -informer.on('add', (obj) => { - console.log(`Added:`, obj); -}); - -informer.on('update', (obj) => { - console.log(`Updated:`, obj); -}); - -informer.on('delete', (obj) => { - console.log(`Deleted:`, obj); -}); - -informer.on('error', (err) => { - console.error(err); - setTimeout(() => { - console.log('Restarting informer...'); - informer.start(); - }, 5000); -}); - -informer.start().then(() => { - console.log('Informer started'); -}); diff --git a/deploy.sh b/scripts/deploy.sh similarity index 76% rename from deploy.sh rename to scripts/deploy.sh index b40466f..0a2b0af 100755 --- a/deploy.sh +++ b/scripts/deploy.sh @@ -20,8 +20,10 @@ set -euo pipefail -basedir="$(dirname "$0")/deployment" +basedir="$(dirname "$0")" keydir="$(mktemp -d)" +templdir="${basedir}/../k8s/templates" +resdir="${basedir}/../k8s/manifests" # Generate keys into a temporary directory. echo "Generating TLS keys ..." @@ -37,22 +39,20 @@ kubectl -n webhook-demo create secret tls webhook-server-tls \ --cert "${keydir}/webhook-server-tls.crt" \ --key "${keydir}/webhook-server-tls.key" -rm -f webhook.yaml deployment.yaml +rm -f "${resdir}/webhook.yaml" "${resdir}/deployment.yaml" # Read the PEM-encoded CA certificate, base64 encode it, and replace the `${CA_PEM_B64}` placeholder in the YAML # template with it. Then, create the Kubernetes resources. ca_pem_b64="$(openssl base64 -A <"${keydir}/ca.crt")" -sed -e 's@${CA_PEM_B64}@'"$ca_pem_b64"'@g' <"${basedir}/webhook.yaml.template" > webhook.yaml -cp deployment/deployment.yaml.template deployment.yaml +sed -e 's@${CA_PEM_B64}@'"$ca_pem_b64"'@g' <"${templdir}/webhook.yaml.template" > "${resdir}/webhook.yaml" +cp "${templdir}/deployment.yaml.template" "${resdir}/deployment.yaml" -skaffold run --namespace webhook-demo +skaffold run -n webhook-demo -f k8s/skaffold.yaml # kubectl apply -f deployment.yaml sleep 2 -kubectl apply -f monokle.policy.crd.yaml -sleep 2 -kubectl apply -f webhook.yaml - -# skaffold dev +kubectl apply -f "${resdir}/crd.yaml" +kubectl apply -f "${resdir}/service-account.yaml" +kubectl apply -f "${resdir}/webhook.yaml" # Delete the key directory to prevent abuse (DO NOT USE THESE KEYS ANYWHERE ELSE). rm -rf "$keydir" diff --git a/deployment/generate-keys.sh b/scripts/generate-keys.sh similarity index 100% rename from deployment/generate-keys.sh rename to scripts/generate-keys.sh From f4cbd766187ea45b376aa4e078ef3dbfcd60efb1 Mon Sep 17 00:00:00 2001 From: f1ames Date: Wed, 27 Sep 2023 14:37:20 +0200 Subject: [PATCH 10/26] fix: add minor code adjustements --- README.md | 6 +++--- admission-webhook/src/utils/validation-server.ts | 1 + k8s/manifests/crd.yaml | 6 +++--- k8s/manifests/service-account.yaml | 2 +- k8s/templates/webhook.yaml.template | 3 ++- 5 files changed, 10 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 04b5ca6..f0696f3 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ NAME DESIRED CURRENT READY AGE replicaset.apps/webhook-server-677556956c 1 1 1 3m54s NAME CREATED AT -customresourcedefinition.apiextensions.k8s.io/monoklepolicies.monokle.com 2023-09-27T08:45:13Z +customresourcedefinition.apiextensions.k8s.io/policies.monokle.com 2023-09-27T08:45:13Z NAME WEBHOOKS AGE validatingwebhookconfiguration.admissionregistration.k8s.io/demo-webhook 1 3m46s @@ -59,7 +59,7 @@ For getting info about CRDs: ```bash kubectl get crd -kubectl describe crd monoklepolicies.monokle.com +kubectl describe crd policies.monokle.com kubectl get monoklepolicy -n webhook-demo kubectl describe monoklepolicy policy-sample -n webhook-demo @@ -97,7 +97,7 @@ You can also do manual clean-up and re-run `./deploy.sh` script again: kubectl delete all -n webhook-demo --all && \ kubectl delete validatingwebhookconfiguration.admissionregistration.k8s.io/demo-webhook -n webhook-demo && \ kubectl delete namespace webhook-demo && \ -kubectl delete crd monoklepolicies.monokle.com +kubectl delete crd policies.monokle.com ``` ### Refs diff --git a/admission-webhook/src/utils/validation-server.ts b/admission-webhook/src/utils/validation-server.ts index 14fe3a9..943f557 100644 --- a/admission-webhook/src/utils/validation-server.ts +++ b/admission-webhook/src/utils/validation-server.ts @@ -106,6 +106,7 @@ export class ValidationServer { } } + // @TODO should not be a part of production code // Dev workaround - always return true for webhook server to not block hot-reload if (body.request?.name?.startsWith('webhook-server-')) { this._logger.debug({msg: 'Allowing webhook server to pass', response}); diff --git a/k8s/manifests/crd.yaml b/k8s/manifests/crd.yaml index b14ee38..47fe557 100644 --- a/k8s/manifests/crd.yaml +++ b/k8s/manifests/crd.yaml @@ -1,7 +1,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - name: monoklepolicies.monokle.com + name: policies.monokle.com spec: group: monokle.com versions: @@ -39,8 +39,8 @@ spec: type: string scope: Namespaced names: - plural: monoklepolicies - singular: monoklepolicy + plural: policies + singular: policy kind: MonoklePolicy shortNames: - mp diff --git a/k8s/manifests/service-account.yaml b/k8s/manifests/service-account.yaml index 693e69f..4e87970 100644 --- a/k8s/manifests/service-account.yaml +++ b/k8s/manifests/service-account.yaml @@ -9,7 +9,7 @@ metadata: name: crds-monokle-list rules: - apiGroups: ["monokle.com"] - resources: ["monoklepolicies"] + resources: ["policies"] verbs: ["list", "watch", "get"] --- diff --git a/k8s/templates/webhook.yaml.template b/k8s/templates/webhook.yaml.template index 6940037..23f63e9 100644 --- a/k8s/templates/webhook.yaml.template +++ b/k8s/templates/webhook.yaml.template @@ -16,4 +16,5 @@ webhooks: - operations: [ "CREATE" ] apiGroups: [""] apiVersions: ["v1"] - resources: ["pods"] \ No newline at end of file + resources: ["pods"] + scope: "Namespaced" \ No newline at end of file From 80249e1588d8d7c59cd6af82c87c75224e1ed4d0 Mon Sep 17 00:00:00 2001 From: f1ames Date: Wed, 27 Sep 2023 15:19:14 +0200 Subject: [PATCH 11/26] fix: rework admission controller webhook to not be namespace bound --- admission-webhook/src/index.ts | 84 ++++++++++--------- admission-webhook/src/utils/helpers.ts | 8 -- admission-webhook/src/utils/informer.ts | 10 +-- .../src/utils/validation-server.ts | 18 +++- 4 files changed, 65 insertions(+), 55 deletions(-) delete mode 100644 admission-webhook/src/utils/helpers.ts diff --git a/admission-webhook/src/index.ts b/admission-webhook/src/index.ts index 9eac191..e574071 100644 --- a/admission-webhook/src/index.ts +++ b/admission-webhook/src/index.ts @@ -1,64 +1,70 @@ import pino from 'pino'; import {AnnotationSuppressor, FingerprintSuppressor, MonokleValidator, RemotePluginLoader, SchemaLoader, DisabledFixer, ResourceParser} from '@monokle/validation'; +import {Config} from '@monokle/validation'; import {MonoklePolicy, getInformer} from './utils/informer.js'; import {ValidationServer} from './utils/validation-server.js'; -import {DEFAULT_NAMESPACE, getNamespace} from './utils/helpers.js'; -(async() => { - const logger = pino({ - name: 'Monokle', - level: 'trace', - }); +const logger = pino({ + name: 'Monokle', + level: 'trace', +}); - let currentNamespace: string; - try { - currentNamespace = getNamespace() || DEFAULT_NAMESPACE; - } catch (err: any) { - logger.error('Failed to get current namespace', err); - process.exit(1); - } +const policies: Map = new Map(); +const validators: Map = new Map(); - logger.debug({namespace: currentNamespace}); +(async() => { + // INFORMER + const onPolicy = async (policy: MonoklePolicy) => { + logger.info({msg: 'Informer: Policy updated', policy}); - // VALIDATOR - let awaitValidatorReadiness = Promise.resolve(); + const policyNamespace = policy.metadata?.namespace; + if (!policyNamespace) { + logger.error({msg: 'Informer: Policy namespace is empty', metadata: policy.metadata}); + return; + } - const validator = new MonokleValidator( - { - loader: new RemotePluginLoader(), - parser: new ResourceParser(), - schemaLoader: new SchemaLoader(), - suppressors: [new AnnotationSuppressor(), new FingerprintSuppressor()], - fixer: new DisabledFixer(), - }, - {} - ); + policies.set(policyNamespace, policy.spec); - // INFORMER - let activePolicy: MonoklePolicy | null = null; + if (!validators.has(policyNamespace)) { + validators.set( + policyNamespace, + new MonokleValidator( + { + loader: new RemotePluginLoader(), + parser: new ResourceParser(), + schemaLoader: new SchemaLoader(), + suppressors: [new AnnotationSuppressor(), new FingerprintSuppressor()], + fixer: new DisabledFixer(), + }, + {} + ) + ); + } - const onPolicy = async (policy: MonoklePolicy) => { - logger.info({msg: 'Informer: Policy updated', policy}); - activePolicy = policy; - awaitValidatorReadiness = validator.preload(policy.spec); - await awaitValidatorReadiness; + await validators.get(policyNamespace)!.preload(policy.spec); } - const onPolicyRemoval = async () => { + const onPolicyRemoval = async (policy: MonoklePolicy) => { logger.info('Informer: Policy removed'); - activePolicy = null; - awaitValidatorReadiness = validator.preload({}); - await awaitValidatorReadiness; + + const policyNamespace = policy.metadata?.namespace; + if (!policyNamespace) { + logger.error({msg: 'Informer: Policy namespace is empty', metadata: policy.metadata}); + return; + } + + policies.delete(policyNamespace); + validators.delete(policyNamespace); } const onError = (err: any) => { logger.error({msg: 'Informer: Error', err}); } - const informer = getInformer(onPolicy, onPolicy, onPolicyRemoval, onError, currentNamespace); + const informer = await getInformer(onPolicy, onPolicy, onPolicyRemoval, onError); // SERVER - const server = new ValidationServer(validator, logger); + const server = new ValidationServer(validators, logger); await server.start(); })(); diff --git a/admission-webhook/src/utils/helpers.ts b/admission-webhook/src/utils/helpers.ts deleted file mode 100644 index f9fa790..0000000 --- a/admission-webhook/src/utils/helpers.ts +++ /dev/null @@ -1,8 +0,0 @@ -import {readFileSync} from "fs"; - -export const DEFAULT_NAMESPACE = 'default'; - -export function getNamespace() { - // Get current namespace from inside a pod - https://stackoverflow.com/a/46046153 - return readFileSync('/var/run/secrets/kubernetes.io/serviceaccount/namespace', 'utf8').trim(); -} diff --git a/admission-webhook/src/utils/informer.ts b/admission-webhook/src/utils/informer.ts index e292cf3..94386f3 100644 --- a/admission-webhook/src/utils/informer.ts +++ b/admission-webhook/src/utils/informer.ts @@ -11,8 +11,7 @@ export async function getInformer( onAdded: k8s.ObjectCallback, onUpdated: k8s.ObjectCallback, onDeleted: k8s.ObjectCallback, - onError: k8s.ErrorCallback, - namespace = 'default', + onError?: k8s.ErrorCallback, ) { let informer: Awaited> | null = null; @@ -20,7 +19,7 @@ export async function getInformer( while (!informer) { try { tries++; - informer = await createInformer(onAdded, onUpdated, onDeleted, namespace, onError); + informer = await createInformer(onAdded, onUpdated, onDeleted, onError); return informer; } catch (err: any) { if (onError) { @@ -38,15 +37,14 @@ async function createInformer( onAdded: k8s.ObjectCallback, onUpdated: k8s.ObjectCallback, onDeleted: k8s.ObjectCallback, - namespace = 'default', onError?: k8s.ErrorCallback ) { const kc = new k8s.KubeConfig(); kc.loadFromCluster(); const k8sApi = kc.makeApiClient(k8s.CustomObjectsApi) - const listFn = () => k8sApi.listNamespacedCustomObject('monokle.com','v1', namespace, 'monoklepolicies'); - const informer = k8s.makeInformer(kc, `/apis/monokle.com/v1/namespaces/${namespace}/monoklepolicies`, listFn as any); + const listFn = () => k8sApi.listClusterCustomObject('monokle.com','v1', 'policies'); + const informer = k8s.makeInformer(kc, `/apis/monokle.com/v1/policies`, listFn as any); informer.on('add', async (obj) => await onAdded(obj)); informer.on('update', async (obj) => await onUpdated(obj)); diff --git a/admission-webhook/src/utils/validation-server.ts b/admission-webhook/src/utils/validation-server.ts index 943f557..cdab3b1 100644 --- a/admission-webhook/src/utils/validation-server.ts +++ b/admission-webhook/src/utils/validation-server.ts @@ -33,7 +33,7 @@ export class ValidationServer { private _shouldValidate: boolean constructor( - private readonly _validator: MonokleValidator, + private readonly _validators: Map, private readonly _logger: ReturnType, private readonly _options: ValidationServerOptions = { port: 8443, @@ -93,6 +93,7 @@ export class ValidationServer { this._logger.trace({requestBody: req.body}); const body = req.body as AdmissionRequest; + const namespace = body.request?.namespace; const response = { kind: body?.kind || '', @@ -106,6 +107,19 @@ export class ValidationServer { } } + if (!namespace) { + this._logger.error({msg: 'No namespace found', metadata: body.request}); + + return response; + } + + const validator = this._validators.get(namespace); + if (!validator) { + this._logger.info({msg: 'No validator found for namespace', namespace}); + + return response; + } + // @TODO should not be a part of production code // Dev workaround - always return true for webhook server to not block hot-reload if (body.request?.name?.startsWith('webhook-server-')) { @@ -115,7 +129,7 @@ export class ValidationServer { } const resourceForValidation = this._createResourceForValidation(body); - const validationResponse = await this._validator.validate({ resources: [resourceForValidation] }); + const validationResponse = await validator.validate({ resources: [resourceForValidation] }); const warnings = []; const errors = []; From 6f908b3eba1d5a59c6fefd2791206b8b522b1b52 Mon Sep 17 00:00:00 2001 From: f1ames Date: Thu, 28 Sep 2023 16:17:09 +0200 Subject: [PATCH 12/26] feat: introduce MonoklePolicyBinding [WIP] --- admission-webhook/src/index.ts | 111 ++++++++++-------- admission-webhook/src/utils/get-informer.ts | 47 ++++++++ admission-webhook/src/utils/informer.ts | 66 ----------- admission-webhook/src/utils/policy-manager.ts | 79 +++++++++++++ examples/policy-binding-sample.yaml | 7 ++ examples/policy-sample.yaml | 2 +- k8s/manifests/monokle-policy-binding-crd.yaml | 29 +++++ .../{crd.yaml => monokle-policy-crd.yaml} | 4 +- k8s/manifests/service-account.yaml | 10 +- scripts/deploy.sh | 3 +- 10 files changed, 234 insertions(+), 124 deletions(-) create mode 100644 admission-webhook/src/utils/get-informer.ts delete mode 100644 admission-webhook/src/utils/informer.ts create mode 100644 admission-webhook/src/utils/policy-manager.ts create mode 100644 examples/policy-binding-sample.yaml create mode 100644 k8s/manifests/monokle-policy-binding-crd.yaml rename k8s/manifests/{crd.yaml => monokle-policy-crd.yaml} (97%) diff --git a/admission-webhook/src/index.ts b/admission-webhook/src/index.ts index e574071..68cb5c3 100644 --- a/admission-webhook/src/index.ts +++ b/admission-webhook/src/index.ts @@ -1,70 +1,83 @@ import pino from 'pino'; import {AnnotationSuppressor, FingerprintSuppressor, MonokleValidator, RemotePluginLoader, SchemaLoader, DisabledFixer, ResourceParser} from '@monokle/validation'; -import {Config} from '@monokle/validation'; -import {MonoklePolicy, getInformer} from './utils/informer.js'; -import {ValidationServer} from './utils/validation-server.js'; +import {getInformer} from './utils/get-informer.js'; +import {MonoklePolicy, MonoklePolicyBinding, PolicyManager} from './utils/policy-manager.js'; const logger = pino({ name: 'Monokle', level: 'trace', }); -const policies: Map = new Map(); -const validators: Map = new Map(); - (async() => { - // INFORMER - const onPolicy = async (policy: MonoklePolicy) => { - logger.info({msg: 'Informer: Policy updated', policy}); - - const policyNamespace = policy.metadata?.namespace; - if (!policyNamespace) { - logger.error({msg: 'Informer: Policy namespace is empty', metadata: policy.metadata}); - return; + const policyInformer = await getInformer( + 'monokle.io', + 'v1', + 'policies', + (err: any) => { + logger.error({msg: 'Informer: Policies: Error', err}); } + ); - policies.set(policyNamespace, policy.spec); - - if (!validators.has(policyNamespace)) { - validators.set( - policyNamespace, - new MonokleValidator( - { - loader: new RemotePluginLoader(), - parser: new ResourceParser(), - schemaLoader: new SchemaLoader(), - suppressors: [new AnnotationSuppressor(), new FingerprintSuppressor()], - fixer: new DisabledFixer(), - }, - {} - ) - ); + const bindingsInformer = await getInformer( + 'monokle.io', + 'v1', + 'policybindings', + (err: any) => { + logger.error({msg: 'Informer: Bindings: Error', err}); } + ); - await validators.get(policyNamespace)!.preload(policy.spec); - } + const policyManager = new PolicyManager(policyInformer, bindingsInformer, logger); - const onPolicyRemoval = async (policy: MonoklePolicy) => { - logger.info('Informer: Policy removed'); + // POLICY MANAGER - const policyNamespace = policy.metadata?.namespace; - if (!policyNamespace) { - logger.error({msg: 'Informer: Policy namespace is empty', metadata: policy.metadata}); - return; - } + // // INFORMER + // const onPolicy = async (policy: MonoklePolicy) => { + // logger.info({msg: 'Informer: Policy updated', policy}); + + // const policyNamespace = policy.metadata?.namespace; + // if (!policyNamespace) { + // logger.error({msg: 'Informer: Policy namespace is empty', metadata: policy.metadata}); + // return; + // } + + // policies.set(policyNamespace, policy.spec); + + // if (!validators.has(policyNamespace)) { + // validators.set( + // policyNamespace, + // new MonokleValidator( + // { + // loader: new RemotePluginLoader(), + // parser: new ResourceParser(), + // schemaLoader: new SchemaLoader(), + // suppressors: [new AnnotationSuppressor(), new FingerprintSuppressor()], + // fixer: new DisabledFixer(), + // }, + // {} + // ) + // ); + // } + + // await validators.get(policyNamespace)!.preload(policy.spec); + // } + + // const onPolicyRemoval = async (policy: MonoklePolicy) => { + // logger.info('Informer: Policy removed'); - policies.delete(policyNamespace); - validators.delete(policyNamespace); - } + // const policyNamespace = policy.metadata?.namespace; + // if (!policyNamespace) { + // logger.error({msg: 'Informer: Policy namespace is empty', metadata: policy.metadata}); + // return; + // } - const onError = (err: any) => { - logger.error({msg: 'Informer: Error', err}); - } + // policies.delete(policyNamespace); + // validators.delete(policyNamespace); + // } - const informer = await getInformer(onPolicy, onPolicy, onPolicyRemoval, onError); - // SERVER - const server = new ValidationServer(validators, logger); + // // SERVER + // const server = new ValidationServer(validators, logger); - await server.start(); + // await server.start(); })(); diff --git a/admission-webhook/src/utils/get-informer.ts b/admission-webhook/src/utils/get-informer.ts new file mode 100644 index 0000000..34d950e --- /dev/null +++ b/admission-webhook/src/utils/get-informer.ts @@ -0,0 +1,47 @@ +import k8s from '@kubernetes/client-node'; + +export type Informer = k8s.Informer & k8s.ObjectCache; + +const ERROR_RESTART_INTERVAL = 500; + +export async function getInformer(group: string, version: string, plural: string, onError?: k8s.ErrorCallback) { + let informer: Informer | null = null; + + let tries = 0; + while (!informer) { + try { + tries++; + informer = await createInformer(group, version, plural, onError); + return informer; + } catch (err: any) { + if (onError) { + onError(err); + } + + await new Promise((resolve) => setTimeout(resolve, ERROR_RESTART_INTERVAL)); + } + } + + return informer; +} + +async function createInformer(group: string, version: string, plural: string, onError?: k8s.ErrorCallback) { + const kc = new k8s.KubeConfig(); + kc.loadFromCluster(); + + const k8sApi = kc.makeApiClient(k8s.CustomObjectsApi) + const listFn = () => k8sApi.listClusterCustomObject(group, version, plural); + const informer = k8s.makeInformer(kc, `/apis/${group}/${version}/${plural}`, listFn as any); + + informer.on('error', (err) => { + if (onError) { + onError(err); + } + + setTimeout(async () => { + await informer.start(); + }, ERROR_RESTART_INTERVAL); + }); + + return informer; +} diff --git a/admission-webhook/src/utils/informer.ts b/admission-webhook/src/utils/informer.ts deleted file mode 100644 index 94386f3..0000000 --- a/admission-webhook/src/utils/informer.ts +++ /dev/null @@ -1,66 +0,0 @@ -import k8s from '@kubernetes/client-node'; -import {Config} from '@monokle/validation'; - -export type MonoklePolicy = k8s.KubernetesObject & { - spec: Config -}; - -const ERROR_RESTART_INTERVAL = 500; - -export async function getInformer( - onAdded: k8s.ObjectCallback, - onUpdated: k8s.ObjectCallback, - onDeleted: k8s.ObjectCallback, - onError?: k8s.ErrorCallback, -) { - let informer: Awaited> | null = null; - - let tries = 0; - while (!informer) { - try { - tries++; - informer = await createInformer(onAdded, onUpdated, onDeleted, onError); - return informer; - } catch (err: any) { - if (onError) { - onError(err); - } - - await new Promise((resolve) => setTimeout(resolve, ERROR_RESTART_INTERVAL)); - } - } - - return informer; -} - -async function createInformer( - onAdded: k8s.ObjectCallback, - onUpdated: k8s.ObjectCallback, - onDeleted: k8s.ObjectCallback, - onError?: k8s.ErrorCallback -) { - const kc = new k8s.KubeConfig(); - kc.loadFromCluster(); - - const k8sApi = kc.makeApiClient(k8s.CustomObjectsApi) - const listFn = () => k8sApi.listClusterCustomObject('monokle.com','v1', 'policies'); - const informer = k8s.makeInformer(kc, `/apis/monokle.com/v1/policies`, listFn as any); - - informer.on('add', async (obj) => await onAdded(obj)); - informer.on('update', async (obj) => await onUpdated(obj)); - informer.on('delete', async (obj) => await onDeleted(obj)); - - informer.on('error', (err) => { - if (onError) { - onError(err); - } - - setTimeout(async () => { - await informer.start(); - }, ERROR_RESTART_INTERVAL); - }); - - await informer.start(); - - return informer; -} diff --git a/admission-webhook/src/utils/policy-manager.ts b/admission-webhook/src/utils/policy-manager.ts new file mode 100644 index 0000000..25c5bfa --- /dev/null +++ b/admission-webhook/src/utils/policy-manager.ts @@ -0,0 +1,79 @@ +import pino from 'pino'; +import {KubernetesObject} from '@kubernetes/client-node'; +import {Config} from '@monokle/validation'; +import {Informer} from './get-informer'; + +export type MonoklePolicy = KubernetesObject & { + spec: Config +} + +export type MonoklePolicyBinding = KubernetesObject & { + spec: { + policyName: string + validationActions: 'Warn' + } +} + +export class PolicyManager { + private _policies = new Map(); // Map + private _bindings = new Map(); // Map // @TODO use policyName as key instead of bindingName? + + constructor( + private readonly _policyInformer: Informer, + private readonly _bindingInformer: Informer, + private readonly _logger: ReturnType, + ) { + this._policyInformer.on('add', this.onPolicy); + this._policyInformer.on('update', this.onPolicy); + this._policyInformer.on('delete', this.onPolicyRemoval); + + this._bindingInformer.on('add', this.onBinding); + this._bindingInformer.on('update', this.onBinding); + this._bindingInformer.on('delete', this.onBindingRemoval); + } + + async start() { + await this._policyInformer.start(); + await this._bindingInformer.start(); + } + + getMatchingPolicies() { // @TODO pass resource data so it can be matched according to matchResources definition (when it's implemented) + if (this._bindings.size === 0) { + return []; + } + + return Array.from(this._bindings.values()).map((binding) => { + const policy = this._policies.get(binding.spec.policyName); + + if (!policy) { + this._logger.error({msg: 'Binding is pointing to missing policy', binding}); + } + + return policy; + }).filter((policy) => policy !== undefined); + } + + private onPolicy(policy: MonoklePolicy) { + this._logger.debug({msg: 'Policy updated', policy}); + + this._policies.set(policy.metadata!.name!, policy); + } + + private onPolicyRemoval(policy: MonoklePolicy) { + this._logger.debug({msg: 'Policy updated', policy}); + + this._policies.delete(policy.metadata!.name!); + } + + private onBinding(binding: MonoklePolicyBinding) { + this._logger.debug({msg: 'Binding updated', binding}); + + this._bindings.set(binding.metadata!.name!, binding); + } + + private onBindingRemoval(binding: MonoklePolicyBinding) { + this._logger.debug({msg: 'Binding updated', binding}); + + this._bindings.delete(binding.metadata!.name!); + } +} \ No newline at end of file diff --git a/examples/policy-binding-sample.yaml b/examples/policy-binding-sample.yaml new file mode 100644 index 0000000..0049bed --- /dev/null +++ b/examples/policy-binding-sample.yaml @@ -0,0 +1,7 @@ +apiVersion: monokle.com/v1alpha1 +kind: MonoklePolicyBinding +metadata: + name: policy-sample-binding +spec: + policyName: "policy-sample" + validationActions: [Warn] \ No newline at end of file diff --git a/examples/policy-sample.yaml b/examples/policy-sample.yaml index 724aea3..4fb272a 100644 --- a/examples/policy-sample.yaml +++ b/examples/policy-sample.yaml @@ -1,4 +1,4 @@ -apiVersion: monokle.com/v1 +apiVersion: monokle.com/v1alpha1 kind: MonoklePolicy metadata: name: policy-sample diff --git a/k8s/manifests/monokle-policy-binding-crd.yaml b/k8s/manifests/monokle-policy-binding-crd.yaml new file mode 100644 index 0000000..f950641 --- /dev/null +++ b/k8s/manifests/monokle-policy-binding-crd.yaml @@ -0,0 +1,29 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: policybindings.monokle.com +spec: + group: monokle.com + versions: + - name: v1alpha1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + policyName: + type: string + validationActions: + type: string + enum: [Warn] + scope: Cluster + names: + plural: policybindings + singular: policybinding + kind: MonoklePolicyBinding + shortNames: + - mpb \ No newline at end of file diff --git a/k8s/manifests/crd.yaml b/k8s/manifests/monokle-policy-crd.yaml similarity index 97% rename from k8s/manifests/crd.yaml rename to k8s/manifests/monokle-policy-crd.yaml index 47fe557..25a5f31 100644 --- a/k8s/manifests/crd.yaml +++ b/k8s/manifests/monokle-policy-crd.yaml @@ -5,7 +5,7 @@ metadata: spec: group: monokle.com versions: - - name: v1 + - name: v1alpha1 served: true storage: true schema: @@ -37,7 +37,7 @@ spec: type: object additionalProperties: type: string - scope: Namespaced + scope: Cluster names: plural: policies singular: policy diff --git a/k8s/manifests/service-account.yaml b/k8s/manifests/service-account.yaml index 4e87970..2552766 100644 --- a/k8s/manifests/service-account.yaml +++ b/k8s/manifests/service-account.yaml @@ -1,27 +1,27 @@ apiVersion: v1 kind: ServiceAccount metadata: - name: crds-monokle-list + name: monokle-policies --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: crds-monokle-list + name: monokle-policies rules: - apiGroups: ["monokle.com"] - resources: ["policies"] + resources: ["policies", "policybindings"] verbs: ["list", "watch", "get"] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: crds-monokle-list + name: monokle-policies subjects: - kind: ServiceAccount name: default namespace: webhook-demo roleRef: kind: ClusterRole - name: crds-monokle-list + name: monokle-policies apiGroup: rbac.authorization.k8s.io \ No newline at end of file diff --git a/scripts/deploy.sh b/scripts/deploy.sh index 0a2b0af..9493787 100755 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -50,7 +50,8 @@ cp "${templdir}/deployment.yaml.template" "${resdir}/deployment.yaml" skaffold run -n webhook-demo -f k8s/skaffold.yaml # kubectl apply -f deployment.yaml sleep 2 -kubectl apply -f "${resdir}/crd.yaml" +kubectl apply -f "${resdir}/monokle-policy-crd.yaml" +kubectl apply -f "${resdir}/monokle-policy-binding-crd.yaml" kubectl apply -f "${resdir}/service-account.yaml" kubectl apply -f "${resdir}/webhook.yaml" From e65bf621bac42f07f4a8af5c296f13584b779f60 Mon Sep 17 00:00:00 2001 From: f1ames Date: Thu, 28 Sep 2023 16:32:44 +0200 Subject: [PATCH 13/26] refactor: draft ValidatorManager class --- admission-webhook/src/index.ts | 56 ++----------------- .../src/utils/validation-server.ts | 14 ++--- .../src/utils/validator-manager.ts | 25 +++++++++ 3 files changed, 38 insertions(+), 57 deletions(-) create mode 100644 admission-webhook/src/utils/validator-manager.ts diff --git a/admission-webhook/src/index.ts b/admission-webhook/src/index.ts index 68cb5c3..ff49e3d 100644 --- a/admission-webhook/src/index.ts +++ b/admission-webhook/src/index.ts @@ -1,7 +1,8 @@ import pino from 'pino'; -import {AnnotationSuppressor, FingerprintSuppressor, MonokleValidator, RemotePluginLoader, SchemaLoader, DisabledFixer, ResourceParser} from '@monokle/validation'; import {getInformer} from './utils/get-informer.js'; import {MonoklePolicy, MonoklePolicyBinding, PolicyManager} from './utils/policy-manager.js'; +import {ValidatorManager} from './utils/validator-manager.js'; +import {ValidationServer} from './utils/validation-server.js'; const logger = pino({ name: 'Monokle', @@ -28,56 +29,11 @@ const logger = pino({ ); const policyManager = new PolicyManager(policyInformer, bindingsInformer, logger); + const validatorManager = new ValidatorManager(policyManager); - // POLICY MANAGER + await policyManager.start(); - // // INFORMER - // const onPolicy = async (policy: MonoklePolicy) => { - // logger.info({msg: 'Informer: Policy updated', policy}); + const server = new ValidationServer(validatorManager, logger); - // const policyNamespace = policy.metadata?.namespace; - // if (!policyNamespace) { - // logger.error({msg: 'Informer: Policy namespace is empty', metadata: policy.metadata}); - // return; - // } - - // policies.set(policyNamespace, policy.spec); - - // if (!validators.has(policyNamespace)) { - // validators.set( - // policyNamespace, - // new MonokleValidator( - // { - // loader: new RemotePluginLoader(), - // parser: new ResourceParser(), - // schemaLoader: new SchemaLoader(), - // suppressors: [new AnnotationSuppressor(), new FingerprintSuppressor()], - // fixer: new DisabledFixer(), - // }, - // {} - // ) - // ); - // } - - // await validators.get(policyNamespace)!.preload(policy.spec); - // } - - // const onPolicyRemoval = async (policy: MonoklePolicy) => { - // logger.info('Informer: Policy removed'); - - // const policyNamespace = policy.metadata?.namespace; - // if (!policyNamespace) { - // logger.error({msg: 'Informer: Policy namespace is empty', metadata: policy.metadata}); - // return; - // } - - // policies.delete(policyNamespace); - // validators.delete(policyNamespace); - // } - - - // // SERVER - // const server = new ValidationServer(validators, logger); - - // await server.start(); + await server.start(); })(); diff --git a/admission-webhook/src/utils/validation-server.ts b/admission-webhook/src/utils/validation-server.ts index cdab3b1..2763ec2 100644 --- a/admission-webhook/src/utils/validation-server.ts +++ b/admission-webhook/src/utils/validation-server.ts @@ -2,8 +2,9 @@ import fastify from "fastify"; import pino from 'pino'; import path from "path"; import {readFileSync} from "fs"; -import {MonokleValidator, Resource} from "@monokle/validation"; +import {Resource} from "@monokle/validation"; import {V1ValidatingWebhookConfiguration, V1ObjectMeta} from "@kubernetes/client-node"; +import {ValidatorManager} from "./validator-manager"; export type ValidationServerOptions = { port: number; @@ -33,7 +34,7 @@ export class ValidationServer { private _shouldValidate: boolean constructor( - private readonly _validators: Map, + private readonly _validators: ValidatorManager, private readonly _logger: ReturnType, private readonly _options: ValidationServerOptions = { port: 8443, @@ -113,10 +114,8 @@ export class ValidationServer { return response; } - const validator = this._validators.get(namespace); - if (!validator) { - this._logger.info({msg: 'No validator found for namespace', namespace}); - + const validators = this._validators.getMatchingValidators(); + if (validators.length === 0) { return response; } @@ -129,7 +128,8 @@ export class ValidationServer { } const resourceForValidation = this._createResourceForValidation(body); - const validationResponse = await validator.validate({ resources: [resourceForValidation] }); + // @TODO iterate over validators and run them all + const validationResponse = await validators[0].validate({ resources: [resourceForValidation] }); const warnings = []; const errors = []; diff --git a/admission-webhook/src/utils/validator-manager.ts b/admission-webhook/src/utils/validator-manager.ts new file mode 100644 index 0000000..28d7590 --- /dev/null +++ b/admission-webhook/src/utils/validator-manager.ts @@ -0,0 +1,25 @@ +import {MonokleValidator} from "@monokle/validation"; +import {PolicyManager} from "./policy-manager"; + +export class ValidatorManager { + private _validators = new Map(); // Map + + constructor( + private readonly _policyManager: PolicyManager, + ) { + // @TODO implement this._policyManager.on(policyUpdated, this._reloadValidator) + // We should preload configuration here instead of in getMatchingValidators since + // it would affect performance of the admission webhook response time + } + + getMatchingValidators(): MonokleValidator[] { + const matchingPolicies = this._policyManager.getMatchingPolicies(); + + if (matchingPolicies.length === 0) { + return []; + } + + // @TODO + return []; + } +} \ No newline at end of file From e6ce712b22294f5ce950ec6837847d89eab0424d Mon Sep 17 00:00:00 2001 From: f1ames Date: Fri, 29 Sep 2023 13:50:52 +0200 Subject: [PATCH 14/26] refactor: add support for policy binding [WIP] --- README.md | 3 +- admission-webhook/src/index.ts | 19 +++--- admission-webhook/src/utils/get-informer.ts | 46 +++++++++----- admission-webhook/src/utils/policy-manager.ts | 60 ++++++++++++------- .../src/utils/validation-server.ts | 28 ++++++--- .../src/utils/validator-manager.ts | 42 +++++++++++-- k8s/manifests/service-account.yaml | 2 +- k8s/templates/deployment.yaml.template | 6 ++ scripts/deploy.sh | 13 +++- 9 files changed, 153 insertions(+), 66 deletions(-) diff --git a/README.md b/README.md index f0696f3..3fe75ae 100644 --- a/README.md +++ b/README.md @@ -97,7 +97,8 @@ You can also do manual clean-up and re-run `./deploy.sh` script again: kubectl delete all -n webhook-demo --all && \ kubectl delete validatingwebhookconfiguration.admissionregistration.k8s.io/demo-webhook -n webhook-demo && \ kubectl delete namespace webhook-demo && \ -kubectl delete crd policies.monokle.com +kubectl delete crd policies.monokle.com && \ +kubectl delete crd policybindings.monokle.com ``` ### Refs diff --git a/admission-webhook/src/index.ts b/admission-webhook/src/index.ts index ff49e3d..66934d3 100644 --- a/admission-webhook/src/index.ts +++ b/admission-webhook/src/index.ts @@ -4,31 +4,34 @@ import {MonoklePolicy, MonoklePolicyBinding, PolicyManager} from './utils/policy import {ValidatorManager} from './utils/validator-manager.js'; import {ValidationServer} from './utils/validation-server.js'; +const LOG_LEVEL = (process.env.MONOKLE_LOG_LEVEL || 'warn').toLowerCase(); +const IGNORED_NAMESPACES = (process.env.MONOKLE_IGNORE_NAMESPACES || '').split(','); + const logger = pino({ name: 'Monokle', - level: 'trace', + level: LOG_LEVEL, }); (async() => { const policyInformer = await getInformer( - 'monokle.io', - 'v1', + 'monokle.com', + 'v1alpha1', 'policies', (err: any) => { - logger.error({msg: 'Informer: Policies: Error', err}); + logger.error({msg: 'Informer: Policies: Error', err: err.message, body: err.body}); } ); const bindingsInformer = await getInformer( - 'monokle.io', - 'v1', + 'monokle.com', + 'v1alpha1', 'policybindings', (err: any) => { - logger.error({msg: 'Informer: Bindings: Error', err}); + logger.error({msg: 'Informer: Bindings: Error', err: err.message, body: err.body}); } ); - const policyManager = new PolicyManager(policyInformer, bindingsInformer, logger); + const policyManager = new PolicyManager(policyInformer, bindingsInformer, IGNORED_NAMESPACES, logger); const validatorManager = new ValidatorManager(policyManager); await policyManager.start(); diff --git a/admission-webhook/src/utils/get-informer.ts b/admission-webhook/src/utils/get-informer.ts index 34d950e..9112f12 100644 --- a/admission-webhook/src/utils/get-informer.ts +++ b/admission-webhook/src/utils/get-informer.ts @@ -2,27 +2,41 @@ import k8s from '@kubernetes/client-node'; export type Informer = k8s.Informer & k8s.ObjectCache; +export type InformerWrapper = { + informer: Informer, + start: () => Promise +} + const ERROR_RESTART_INTERVAL = 500; -export async function getInformer(group: string, version: string, plural: string, onError?: k8s.ErrorCallback) { - let informer: Informer | null = null; - - let tries = 0; - while (!informer) { - try { - tries++; - informer = await createInformer(group, version, plural, onError); - return informer; - } catch (err: any) { - if (onError) { - onError(err); - } +export async function getInformer( + group: string, version: string, plural: string, onError?: k8s.ErrorCallback +): Promise> { + const informer = await createInformer(group, version, plural, onError); + const start = createInformerStarter(informer, onError); + + return {informer, start} +} - await new Promise((resolve) => setTimeout(resolve, ERROR_RESTART_INTERVAL)); +function createInformerStarter(informer: Informer, onError?: k8s.ErrorCallback) { // not sure if can try to start informer multiple times + return async () => { + let tries = 0; + let started = false; + + while (!started) { + try { + tries++; + await informer.start(); + started = true; + } catch (err: any) { + if (onError) { + onError(err); + } + + await new Promise((resolve) => setTimeout(resolve, ERROR_RESTART_INTERVAL)); + } } } - - return informer; } async function createInformer(group: string, version: string, plural: string, onError?: k8s.ErrorCallback) { diff --git a/admission-webhook/src/utils/policy-manager.ts b/admission-webhook/src/utils/policy-manager.ts index 25c5bfa..4f43cc4 100644 --- a/admission-webhook/src/utils/policy-manager.ts +++ b/admission-webhook/src/utils/policy-manager.ts @@ -1,17 +1,24 @@ import pino from 'pino'; import {KubernetesObject} from '@kubernetes/client-node'; import {Config} from '@monokle/validation'; -import {Informer} from './get-informer'; +import {InformerWrapper} from './get-informer'; export type MonoklePolicy = KubernetesObject & { spec: Config } +export type MonoklePolicyBindingConfiguration = { + policyName: string + validationActions: ['Warn'] +} + export type MonoklePolicyBinding = KubernetesObject & { - spec: { - policyName: string - validationActions: 'Warn' - } + spec: MonoklePolicyBindingConfiguration +} + +export type MonokleApplicablePolicy = { + policy: Config, + binding: MonoklePolicyBindingConfiguration } export class PolicyManager { @@ -19,17 +26,18 @@ export class PolicyManager { private _bindings = new Map(); // Map // @TODO use policyName as key instead of bindingName? constructor( - private readonly _policyInformer: Informer, - private readonly _bindingInformer: Informer, + private readonly _policyInformer: InformerWrapper, + private readonly _bindingInformer: InformerWrapper, + private readonly _ignoreNamespaces: string[], private readonly _logger: ReturnType, ) { - this._policyInformer.on('add', this.onPolicy); - this._policyInformer.on('update', this.onPolicy); - this._policyInformer.on('delete', this.onPolicyRemoval); + this._policyInformer.informer.on('add', this.onPolicy.bind(this)); + this._policyInformer.informer.on('update', this.onPolicy.bind(this)); + this._policyInformer.informer.on('delete', this.onPolicyRemoval.bind(this)); - this._bindingInformer.on('add', this.onBinding); - this._bindingInformer.on('update', this.onBinding); - this._bindingInformer.on('delete', this.onBindingRemoval); + this._bindingInformer.informer.on('add', this.onBinding.bind(this)); + this._bindingInformer.informer.on('update', this.onBinding.bind(this)); + this._bindingInformer.informer.on('delete', this.onBindingRemoval.bind(this)); } async start() { @@ -37,20 +45,26 @@ export class PolicyManager { await this._bindingInformer.start(); } - getMatchingPolicies() { // @TODO pass resource data so it can be matched according to matchResources definition (when it's implemented) + getMatchingPolicies(): MonokleApplicablePolicy[] { // @TODO pass resource data so it can be matched according to matchResources definition (when it's implemented) if (this._bindings.size === 0) { return []; } - return Array.from(this._bindings.values()).map((binding) => { - const policy = this._policies.get(binding.spec.policyName); - - if (!policy) { - this._logger.error({msg: 'Binding is pointing to missing policy', binding}); - } - - return policy; - }).filter((policy) => policy !== undefined); + return Array.from(this._bindings.values()) + .map((binding) => { + const policy = this._policies.get(binding.spec.policyName); + + if (!policy) { + this._logger.error({msg: 'Binding is pointing to missing policy', binding}); + return null; + } + + return { + policy: policy.spec, + binding: binding.spec + } + }) + .filter((policy) => policy !== null) as MonokleApplicablePolicy[]; } private onPolicy(policy: MonoklePolicy) { diff --git a/admission-webhook/src/utils/validation-server.ts b/admission-webhook/src/utils/validation-server.ts index 2763ec2..a8e885d 100644 --- a/admission-webhook/src/utils/validation-server.ts +++ b/admission-webhook/src/utils/validation-server.ts @@ -128,17 +128,27 @@ export class ValidationServer { } const resourceForValidation = this._createResourceForValidation(body); - // @TODO iterate over validators and run them all - const validationResponse = await validators[0].validate({ resources: [resourceForValidation] }); + const validationResponses = await Promise.all(validators.map(async (validator) => { + return { + result: await validator.validator.validate({ resources: [resourceForValidation] }), + policy: validator.policy + }; + } + )); + + // @TODO each result may have different `validationActions` (defined in bindings) so it should be handled + // it can by be grouping results by action and then performing action for each group const warnings = []; const errors = []; - for (const result of validationResponse.runs) { - for (const item of result.results) { - if (item.level === "warning") { - warnings.push(item); - } else if (item.level === "error") { - errors.push(item); + for (const validationResponse of validationResponses) { + for (const result of validationResponse.result.runs) { + for (const item of result.results) { + if (item.level === "warning") { + warnings.push(item); + } else if (item.level === "error") { + errors.push(item); + } } } } @@ -163,7 +173,7 @@ export class ValidationServer { } this._logger.debug({response}); - this._logger.trace({resourceForValidation, validationResponse}); + this._logger.trace({resourceForValidation, validationResponses}); return response; }); diff --git a/admission-webhook/src/utils/validator-manager.ts b/admission-webhook/src/utils/validator-manager.ts index 28d7590..ae9c190 100644 --- a/admission-webhook/src/utils/validator-manager.ts +++ b/admission-webhook/src/utils/validator-manager.ts @@ -1,5 +1,10 @@ -import {MonokleValidator} from "@monokle/validation"; -import {PolicyManager} from "./policy-manager"; +import {AnnotationSuppressor, Config, DisabledFixer, FingerprintSuppressor, MonokleValidator, RemotePluginLoader, ResourceParser, SchemaLoader} from "@monokle/validation"; +import {MonokleApplicablePolicy, PolicyManager} from "./policy-manager"; + +export type MonokleApplicableValidator = { + validator: MonokleValidator, + policy: MonokleApplicablePolicy +} export class ValidatorManager { private _validators = new Map(); // Map @@ -12,14 +17,41 @@ export class ValidatorManager { // it would affect performance of the admission webhook response time } - getMatchingValidators(): MonokleValidator[] { + getMatchingValidators(): MonokleApplicableValidator[] { const matchingPolicies = this._policyManager.getMatchingPolicies(); if (matchingPolicies.length === 0) { return []; } - // @TODO - return []; + return matchingPolicies.map((policy) => { + if (!this._validators.has(policy.binding.policyName)) { + this.setupValidator(policy.binding.policyName, policy.policy); + } + + return { + validator: this._validators.get(policy.binding.policyName)!, + policy + } + }); + } + + private async setupValidator(policyName: string, policy: Config) { + if (this._validators.has(policyName)) { + this._validators.get(policyName)!.preload(policy); + } else { + const validator = new MonokleValidator( + { + loader: new RemotePluginLoader(), + parser: new ResourceParser(), + schemaLoader: new SchemaLoader(), + suppressors: [new AnnotationSuppressor(), new FingerprintSuppressor()], + fixer: new DisabledFixer(), + }, + policy + ); + + this._validators.set(policyName, validator); + } } } \ No newline at end of file diff --git a/k8s/manifests/service-account.yaml b/k8s/manifests/service-account.yaml index 2552766..347a55a 100644 --- a/k8s/manifests/service-account.yaml +++ b/k8s/manifests/service-account.yaml @@ -19,7 +19,7 @@ metadata: name: monokle-policies subjects: - kind: ServiceAccount - name: default + name: monokle-policies namespace: webhook-demo roleRef: kind: ClusterRole diff --git a/k8s/templates/deployment.yaml.template b/k8s/templates/deployment.yaml.template index c8587cf..d10c15d 100644 --- a/k8s/templates/deployment.yaml.template +++ b/k8s/templates/deployment.yaml.template @@ -29,10 +29,16 @@ spec: - name: webhook-tls-certs mountPath: /run/secrets/tls readOnly: true + env: + - name: MONOKLE_LOG_LEVEL + value: DEBUG + - name: MONOKLE_IGNORE_NAMESPACES + value: 'kube-node-lease,kube-public,kube-system' volumes: - name: webhook-tls-certs secret: secretName: webhook-server-tls + serviceAccountName: monokle-policies --- apiVersion: v1 kind: Service diff --git a/scripts/deploy.sh b/scripts/deploy.sh index 9493787..1c60b8d 100755 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -47,12 +47,19 @@ ca_pem_b64="$(openssl base64 -A <"${keydir}/ca.crt")" sed -e 's@${CA_PEM_B64}@'"$ca_pem_b64"'@g' <"${templdir}/webhook.yaml.template" > "${resdir}/webhook.yaml" cp "${templdir}/deployment.yaml.template" "${resdir}/deployment.yaml" +# Cluster-wide +kubectl apply -f "${resdir}/monokle-policy-crd.yaml" +kubectl apply -f "${resdir}/monokle-policy-binding-crd.yaml" + +# Namespaced +kubectl apply -f "${resdir}/service-account.yaml" -n webhook-demo + skaffold run -n webhook-demo -f k8s/skaffold.yaml # kubectl apply -f deployment.yaml sleep 2 -kubectl apply -f "${resdir}/monokle-policy-crd.yaml" -kubectl apply -f "${resdir}/monokle-policy-binding-crd.yaml" -kubectl apply -f "${resdir}/service-account.yaml" +# kubectl apply -f "${resdir}/monokle-policy-crd.yaml" +# kubectl apply -f "${resdir}/monokle-policy-binding-crd.yaml" +# kubectl apply -f "${resdir}/service-account.yaml" kubectl apply -f "${resdir}/webhook.yaml" # Delete the key directory to prevent abuse (DO NOT USE THESE KEYS ANYWHERE ELSE). From 13a95157be17926b884349b30c06313f69e683ad Mon Sep 17 00:00:00 2001 From: f1ames Date: Fri, 29 Sep 2023 15:48:49 +0200 Subject: [PATCH 15/26] fix: fix policy binding CRD schema --- README.md | 1 + k8s/manifests/monokle-policy-binding-crd.yaml | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3fe75ae..b4aa14a 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,7 @@ First you need to create policy resource, for example: ```bash kubectl -n webhook-demo apply -f examples/policy-sample.yaml +kubectl -n webhook-demo apply -f examples/policy-binding-sample.yaml ``` > Admission controller will still work without policy resource but then it will be like running validation with all plugins disabled. diff --git a/k8s/manifests/monokle-policy-binding-crd.yaml b/k8s/manifests/monokle-policy-binding-crd.yaml index f950641..5d536f1 100644 --- a/k8s/manifests/monokle-policy-binding-crd.yaml +++ b/k8s/manifests/monokle-policy-binding-crd.yaml @@ -18,8 +18,10 @@ spec: policyName: type: string validationActions: - type: string - enum: [Warn] + type: array + items: + type: string + enum: [Warn] scope: Cluster names: plural: policybindings From 15b48d3616a8b8507eb8e60d7bab989bcd12ce0b Mon Sep 17 00:00:00 2001 From: f1ames Date: Mon, 2 Oct 2023 11:41:55 +0200 Subject: [PATCH 16/26] feat: add namespace filtering --- README.md | 29 ++++++--- admission-webhook/src/index.ts | 2 +- admission-webhook/src/utils/policy-manager.ts | 63 +++++++++++++++++-- .../src/utils/validation-server.ts | 32 ++++++++-- .../src/utils/validator-manager.ts | 37 +++++++---- examples/policy-binding-sample.yaml | 2 +- examples/policy-binding-scoped-sample.yaml | 11 ++++ examples/policy-sample-2.yaml | 8 +++ k8s/manifests/monokle-policy-binding-crd.yaml | 10 +++ 9 files changed, 161 insertions(+), 33 deletions(-) create mode 100644 examples/policy-binding-scoped-sample.yaml create mode 100644 examples/policy-sample-2.yaml diff --git a/README.md b/README.md index b4aa14a..f5dde2a 100644 --- a/README.md +++ b/README.md @@ -60,9 +60,7 @@ For getting info about CRDs: ```bash kubectl get crd kubectl describe crd policies.monokle.com - -kubectl get monoklepolicy -n webhook-demo -kubectl describe monoklepolicy policy-sample -n webhook-demo +kubectl describe crd policybindings.monokle.com ``` #### Testing @@ -70,16 +68,33 @@ kubectl describe monoklepolicy policy-sample -n webhook-demo First you need to create policy resource, for example: ```bash -kubectl -n webhook-demo apply -f examples/policy-sample.yaml -kubectl -n webhook-demo apply -f examples/policy-binding-sample.yaml +kubectl apply -f examples/policy-sample.yaml +kubectl apply -f examples/policy-sample-2.yaml +``` + +Then it needs to be bind to be used for validation. Either without scope (globally to all, but ignored namespaces) or with `matchResource` field: + +```bash +kubectl apply -f examples/policy-binding-sample.yaml +kubectl apply -f examples/policy-binding-scoped-sample.yaml ``` -> Admission controller will still work without policy resource but then it will be like running validation with all plugins disabled. +You can inspect deployed policies with: + +```bash +kubectl get policy +kubectl describe policy + +kubectl get policybinding +kubectl describe policybinding +``` Then you can try to create sample resource and see webhook response: ```bash -kubectl -n webhook-demo create -f examples/pod-warning.yaml +kubectl apply -f examples/pod-valid.yaml +kubectl apply -f examples/pod-warning.yaml +kubectl apply -f examples/pod-errors.yaml ``` #### Iterating diff --git a/admission-webhook/src/index.ts b/admission-webhook/src/index.ts index 66934d3..7f8f730 100644 --- a/admission-webhook/src/index.ts +++ b/admission-webhook/src/index.ts @@ -32,7 +32,7 @@ const logger = pino({ ); const policyManager = new PolicyManager(policyInformer, bindingsInformer, IGNORED_NAMESPACES, logger); - const validatorManager = new ValidatorManager(policyManager); + const validatorManager = new ValidatorManager(policyManager, logger); await policyManager.start(); diff --git a/admission-webhook/src/utils/policy-manager.ts b/admission-webhook/src/utils/policy-manager.ts index 4f43cc4..90e9506 100644 --- a/admission-webhook/src/utils/policy-manager.ts +++ b/admission-webhook/src/utils/policy-manager.ts @@ -1,7 +1,9 @@ +import {EventEmitter} from "events"; import pino from 'pino'; import {KubernetesObject} from '@kubernetes/client-node'; import {Config} from '@monokle/validation'; import {InformerWrapper} from './get-informer'; +import {AdmissionRequestObject} from "./validation-server"; export type MonoklePolicy = KubernetesObject & { spec: Config @@ -10,6 +12,11 @@ export type MonoklePolicy = KubernetesObject & { export type MonoklePolicyBindingConfiguration = { policyName: string validationActions: ['Warn'] + matchResources?: { + namespaceSelector?: { + matchLabels?: Record + } + } } export type MonoklePolicyBinding = KubernetesObject & { @@ -21,9 +28,9 @@ export type MonokleApplicablePolicy = { binding: MonoklePolicyBindingConfiguration } -export class PolicyManager { +export class PolicyManager extends EventEmitter{ private _policies = new Map(); // Map - private _bindings = new Map(); // Map // @TODO use policyName as key instead of bindingName? + private _bindings = new Map(); // Map constructor( private readonly _policyInformer: InformerWrapper, @@ -31,6 +38,8 @@ export class PolicyManager { private readonly _ignoreNamespaces: string[], private readonly _logger: ReturnType, ) { + super(); + this._policyInformer.informer.on('add', this.onPolicy.bind(this)); this._policyInformer.informer.on('update', this.onPolicy.bind(this)); this._policyInformer.informer.on('delete', this.onPolicyRemoval.bind(this)); @@ -45,11 +54,17 @@ export class PolicyManager { await this._bindingInformer.start(); } - getMatchingPolicies(): MonokleApplicablePolicy[] { // @TODO pass resource data so it can be matched according to matchResources definition (when it's implemented) + getMatchingPolicies(resource: AdmissionRequestObject, namespace: string): MonokleApplicablePolicy[] { + this._logger.debug({policies: this._policies.size, bindings: this._bindings.size}); + if (this._bindings.size === 0) { return []; } + if (this._ignoreNamespaces.includes(namespace)) { + return []; + } + return Array.from(this._bindings.values()) .map((binding) => { const policy = this._policies.get(binding.spec.policyName); @@ -59,6 +74,10 @@ export class PolicyManager { return null; } + if (binding.spec.matchResources && !this.isResourceMatching(binding, resource)) { + return null; + } + return { policy: policy.spec, binding: binding.spec @@ -71,23 +90,55 @@ export class PolicyManager { this._logger.debug({msg: 'Policy updated', policy}); this._policies.set(policy.metadata!.name!, policy); + + this.emit('policyUpdated', policy); } private onPolicyRemoval(policy: MonoklePolicy) { - this._logger.debug({msg: 'Policy updated', policy}); + this._logger.debug({msg: 'Policy removed', policy}); this._policies.delete(policy.metadata!.name!); + + this.emit('policyRemoved', policy); } private onBinding(binding: MonoklePolicyBinding) { this._logger.debug({msg: 'Binding updated', binding}); this._bindings.set(binding.metadata!.name!, binding); + + this.emit('bindingUpdated', binding); } private onBindingRemoval(binding: MonoklePolicyBinding) { - this._logger.debug({msg: 'Binding updated', binding}); + this._logger.debug({msg: 'Binding removed', binding}); this._bindings.delete(binding.metadata!.name!); + + this.emit('bindingRemoved', binding); } -} \ No newline at end of file + + private isResourceMatching(binding: MonoklePolicyBinding, resource: AdmissionRequestObject): boolean { + const namespaceMatchLabels = binding.spec.matchResources?.namespaceSelector?.matchLabels; + + this._logger.trace({ + msg: 'Checking if resource matches binding', + namespaceMatchLabels, + resourceMetadata: resource.metadata.labels + }); + + if (!namespaceMatchLabels) { + return true; + } + + for (const key of Object.keys(namespaceMatchLabels)) { + if (resource.metadata.labels?.[key] !== namespaceMatchLabels[key]) { + if (!(key === 'namespace' && resource.metadata.namespace === namespaceMatchLabels[key])) { + return false; + } + } + } + + return true; + } +} diff --git a/admission-webhook/src/utils/validation-server.ts b/admission-webhook/src/utils/validation-server.ts index a8e885d..b9bf668 100644 --- a/admission-webhook/src/utils/validation-server.ts +++ b/admission-webhook/src/utils/validation-server.ts @@ -3,7 +3,7 @@ import pino from 'pino'; import path from "path"; import {readFileSync} from "fs"; import {Resource} from "@monokle/validation"; -import {V1ValidatingWebhookConfiguration, V1ObjectMeta} from "@kubernetes/client-node"; +import {V1ObjectMeta} from "@kubernetes/client-node"; import {ValidatorManager} from "./validator-manager"; export type ValidationServerOptions = { @@ -11,9 +11,22 @@ export type ValidationServerOptions = { host: string; }; -export type AdmissionRequest = V1ValidatingWebhookConfiguration & { - request?: V1ObjectMeta & { - object?: Resource +export type AdmissionRequestObject = { + apiVersion: string + kind: string + metadata: V1ObjectMeta + spec: any + status: any +}; + +export type AdmissionRequest = { + apiVersion: string + kind: string + request: { + name: string + namespace: string + uid: string + object: AdmissionRequestObject } }; @@ -44,7 +57,6 @@ export class ValidationServer { this._shouldValidate = false; this._server = fastify({ - // @TODO do not require certs when running locally (for testing outside K8s cluster) https: { key: readFileSync(path.join('/run/secrets/tls', 'tls.key')), cert: readFileSync(path.join('/run/secrets/tls', 'tls.crt')) @@ -110,11 +122,19 @@ export class ValidationServer { if (!namespace) { this._logger.error({msg: 'No namespace found', metadata: body.request}); + return response; + } + const resource = body.request?.object; + if (!resource) { + this._logger.error({msg: 'No resource found', metadata: body.request}); return response; } - const validators = this._validators.getMatchingValidators(); + const validators = this._validators.getMatchingValidators(resource, namespace); + + this._logger.debug({msg: 'Matching validators', count: validators.length}); + if (validators.length === 0) { return response; } diff --git a/admission-webhook/src/utils/validator-manager.ts b/admission-webhook/src/utils/validator-manager.ts index ae9c190..aca87fa 100644 --- a/admission-webhook/src/utils/validator-manager.ts +++ b/admission-webhook/src/utils/validator-manager.ts @@ -1,5 +1,7 @@ +import pino from 'pino'; import {AnnotationSuppressor, Config, DisabledFixer, FingerprintSuppressor, MonokleValidator, RemotePluginLoader, ResourceParser, SchemaLoader} from "@monokle/validation"; -import {MonokleApplicablePolicy, PolicyManager} from "./policy-manager"; +import {MonokleApplicablePolicy, MonoklePolicy, PolicyManager} from "./policy-manager"; +import {AdmissionRequestObject} from './validation-server'; export type MonokleApplicableValidator = { validator: MonokleValidator, @@ -11,14 +13,19 @@ export class ValidatorManager { constructor( private readonly _policyManager: PolicyManager, + private readonly _logger: ReturnType, ) { - // @TODO implement this._policyManager.on(policyUpdated, this._reloadValidator) - // We should preload configuration here instead of in getMatchingValidators since - // it would affect performance of the admission webhook response time + this._policyManager.on('policyUpdated', async (policy: MonoklePolicy) => { + await this.setupValidator(policy.metadata!.name!, policy.spec); + }); + + this._policyManager.on('policyRemoved', async (policy: MonoklePolicy) => { + await this._validators.delete(policy.metadata!.name!); + }); } - getMatchingValidators(): MonokleApplicableValidator[] { - const matchingPolicies = this._policyManager.getMatchingPolicies(); + getMatchingValidators(resource: AdmissionRequestObject, namespace: string): MonokleApplicableValidator[] { + const matchingPolicies = this._policyManager.getMatchingPolicies(resource, namespace); if (matchingPolicies.length === 0) { return []; @@ -26,19 +33,22 @@ export class ValidatorManager { return matchingPolicies.map((policy) => { if (!this._validators.has(policy.binding.policyName)) { - this.setupValidator(policy.binding.policyName, policy.policy); + // This should not happen and means there is a bug in other place in the code. Raise warning and skip. + // Do not create validator instance here to keep this function sync and to keep processing time low. + this._logger.warn({msg: 'ValidatorManager: Validator not found', policyName: policy.binding.policyName}); + return null; } return { validator: this._validators.get(policy.binding.policyName)!, policy } - }); + }).filter((validator) => validator !== null) as MonokleApplicableValidator[]; } private async setupValidator(policyName: string, policy: Config) { if (this._validators.has(policyName)) { - this._validators.get(policyName)!.preload(policy); + await this._validators.get(policyName)!.preload(policy); } else { const validator = new MonokleValidator( { @@ -47,11 +57,14 @@ export class ValidatorManager { schemaLoader: new SchemaLoader(), suppressors: [new AnnotationSuppressor(), new FingerprintSuppressor()], fixer: new DisabledFixer(), - }, - policy + } ); + // Run separately (instead of passing config to constructor) to make sure that validator + // is ready when 'setupValidator' function call fulfills. + await validator.preload(policy); + this._validators.set(policyName, validator); } } -} \ No newline at end of file +} diff --git a/examples/policy-binding-sample.yaml b/examples/policy-binding-sample.yaml index 0049bed..681a397 100644 --- a/examples/policy-binding-sample.yaml +++ b/examples/policy-binding-sample.yaml @@ -1,7 +1,7 @@ apiVersion: monokle.com/v1alpha1 kind: MonoklePolicyBinding metadata: - name: policy-sample-binding + name: policy-binding-sample spec: policyName: "policy-sample" validationActions: [Warn] \ No newline at end of file diff --git a/examples/policy-binding-scoped-sample.yaml b/examples/policy-binding-scoped-sample.yaml new file mode 100644 index 0000000..f5591fd --- /dev/null +++ b/examples/policy-binding-scoped-sample.yaml @@ -0,0 +1,11 @@ +apiVersion: monokle.com/v1alpha1 +kind: MonoklePolicyBinding +metadata: + name: policy-binding-scoped-sample +spec: + policyName: "policy-sample-2" + validationActions: [Warn] + matchResources: + namespaceSelector: + matchLabels: + namespace: default \ No newline at end of file diff --git a/examples/policy-sample-2.yaml b/examples/policy-sample-2.yaml new file mode 100644 index 0000000..2c15f5f --- /dev/null +++ b/examples/policy-sample-2.yaml @@ -0,0 +1,8 @@ +apiVersion: monokle.com/v1alpha1 +kind: MonoklePolicy +metadata: + name: policy-sample-2 +spec: + plugins: + open-policy-agent: true + pod-security-standards: true \ No newline at end of file diff --git a/k8s/manifests/monokle-policy-binding-crd.yaml b/k8s/manifests/monokle-policy-binding-crd.yaml index 5d536f1..d7567d0 100644 --- a/k8s/manifests/monokle-policy-binding-crd.yaml +++ b/k8s/manifests/monokle-policy-binding-crd.yaml @@ -22,6 +22,16 @@ spec: items: type: string enum: [Warn] + matchResources: + type: object + properties: + namespaceSelector: + type: object + properties: + matchLabels: + type: object + additionalProperties: + type: string scope: Cluster names: plural: policybindings From bf7b56ffe1770268527164785ab14dad9b0819c2 Mon Sep 17 00:00:00 2001 From: f1ames Date: Mon, 2 Oct 2023 14:25:46 +0200 Subject: [PATCH 17/26] refactor: deploy admission controller to dedicated namespace --- README.md | 36 ++++++++++--------- .../src/utils/validation-server.ts | 8 ----- ...mple.yaml => policy-binding-sample-1.yaml} | 4 +-- ...mple.yaml => policy-binding-sample-2.yaml} | 2 +- examples/policy-binding-sample-3.yaml | 11 ++++++ ...olicy-sample.yaml => policy-sample-1.yaml} | 2 +- k8s/manifests/service-account.yaml | 4 +-- k8s/templates/deployment.yaml.template | 10 +++--- k8s/templates/webhook.yaml.template | 12 +++---- scripts/deploy.sh | 22 ++++++------ scripts/generate-keys.sh | 8 ++--- 11 files changed, 63 insertions(+), 56 deletions(-) rename examples/{policy-binding-sample.yaml => policy-binding-sample-1.yaml} (61%) rename examples/{policy-binding-scoped-sample.yaml => policy-binding-sample-2.yaml} (85%) create mode 100644 examples/policy-binding-sample-3.yaml rename examples/{policy-sample.yaml => policy-sample-1.yaml} (95%) diff --git a/README.md b/README.md index f5dde2a..f1d4bc0 100644 --- a/README.md +++ b/README.md @@ -27,32 +27,33 @@ minikube start --uuid 00000000-0000-0000-0000-000000000001 --extra-config=apiser ./scripts/deploy.sh ``` -Every resource will be deployed to `webhook-demo` namespace, to watch it you can run: +Namespaced resources (webhook server) will be deployed to dedicated `monokle-admission-controller` namespace, to watch it you can run: ```bash -watch kubectl -n webhook-demo get all,CustomResourceDefinition,ValidatingWebhookConfiguration,MutatingWebhookConfiguration +watch kubectl -n monokle-admission-controller get all,CustomResourceDefinition,ValidatingWebhookConfiguration,MutatingWebhookConfiguration ``` After it runs, the result should be something like: ```bash -NAME READY STATUS RESTARTS AGE -pod/webhook-server-677556956c-f7hcq 1/1 Running 0 3m54s +NAME READY STATUS RESTARTS AGE +pod/webhook-server-7cd54c9fcf-wdkdn 0/1 Error 1 (13s ago) 25s NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE -service/webhook-server ClusterIP 10.105.249.5 443/TCP 3m54s +service/webhook-server ClusterIP 10.97.66.194 443/TCP 25s NAME READY UP-TO-DATE AVAILABLE AGE -deployment.apps/webhook-server 1/1 1 1 3m54s +deployment.apps/webhook-server 0/1 1 0 25s NAME DESIRED CURRENT READY AGE -replicaset.apps/webhook-server-677556956c 1 1 1 3m54s +replicaset.apps/webhook-server-7cd54c9fcf 1 1 0 25s -NAME CREATED AT -customresourcedefinition.apiextensions.k8s.io/policies.monokle.com 2023-09-27T08:45:13Z +NAME CREATED AT +customresourcedefinition.apiextensions.k8s.io/policies.monokle.com 2023-10-02T12:17:02Z +customresourcedefinition.apiextensions.k8s.io/policybindings.monokle.com 2023-10-02T12:17:02Z NAME WEBHOOKS AGE -validatingwebhookconfiguration.admissionregistration.k8s.io/demo-webhook 1 3m46s +validatingwebhookconfiguration.admissionregistration.k8s.io/demo-webhook 1 17s ``` For getting info about CRDs: @@ -68,15 +69,16 @@ kubectl describe crd policybindings.monokle.com First you need to create policy resource, for example: ```bash -kubectl apply -f examples/policy-sample.yaml +kubectl apply -f examples/policy-sample-1.yaml kubectl apply -f examples/policy-sample-2.yaml ``` Then it needs to be bind to be used for validation. Either without scope (globally to all, but ignored namespaces) or with `matchResource` field: ```bash -kubectl apply -f examples/policy-binding-sample.yaml -kubectl apply -f examples/policy-binding-scoped-sample.yaml +kubectl apply -f examples/policy-binding-sample-1.yaml +kubectl apply -f examples/policy-binding-sample-2.yaml +kubectl apply -f examples/policy-binding-sample-3.yaml ``` You can inspect deployed policies with: @@ -110,9 +112,11 @@ skaffold dev -f k8s/skaffold.yaml You can also do manual clean-up and re-run `./deploy.sh` script again: ```bash -kubectl delete all -n webhook-demo --all && \ -kubectl delete validatingwebhookconfiguration.admissionregistration.k8s.io/demo-webhook -n webhook-demo && \ -kubectl delete namespace webhook-demo && \ +kubectl delete all -n monokle-admission-controller --all && \ +kubectl delete validatingwebhookconfiguration.admissionregistration.k8s.io/demo-webhook -n monokle-admission-controller && \ +kubectl delete namespace monokle-admission-controller && \ +kubectl delete namespace nstest1 && \ +kubectl delete namespace nstest2 && \ kubectl delete crd policies.monokle.com && \ kubectl delete crd policybindings.monokle.com ``` diff --git a/admission-webhook/src/utils/validation-server.ts b/admission-webhook/src/utils/validation-server.ts index b9bf668..ee6723c 100644 --- a/admission-webhook/src/utils/validation-server.ts +++ b/admission-webhook/src/utils/validation-server.ts @@ -139,14 +139,6 @@ export class ValidationServer { return response; } - // @TODO should not be a part of production code - // Dev workaround - always return true for webhook server to not block hot-reload - if (body.request?.name?.startsWith('webhook-server-')) { - this._logger.debug({msg: 'Allowing webhook server to pass', response}); - - return response; - } - const resourceForValidation = this._createResourceForValidation(body); const validationResponses = await Promise.all(validators.map(async (validator) => { return { diff --git a/examples/policy-binding-sample.yaml b/examples/policy-binding-sample-1.yaml similarity index 61% rename from examples/policy-binding-sample.yaml rename to examples/policy-binding-sample-1.yaml index 681a397..13136e0 100644 --- a/examples/policy-binding-sample.yaml +++ b/examples/policy-binding-sample-1.yaml @@ -1,7 +1,7 @@ apiVersion: monokle.com/v1alpha1 kind: MonoklePolicyBinding metadata: - name: policy-binding-sample + name: policy-binding-sample-1 spec: - policyName: "policy-sample" + policyName: "policy-sample-1" validationActions: [Warn] \ No newline at end of file diff --git a/examples/policy-binding-scoped-sample.yaml b/examples/policy-binding-sample-2.yaml similarity index 85% rename from examples/policy-binding-scoped-sample.yaml rename to examples/policy-binding-sample-2.yaml index f5591fd..9a945ba 100644 --- a/examples/policy-binding-scoped-sample.yaml +++ b/examples/policy-binding-sample-2.yaml @@ -1,7 +1,7 @@ apiVersion: monokle.com/v1alpha1 kind: MonoklePolicyBinding metadata: - name: policy-binding-scoped-sample + name: policy-binding-sample-2 spec: policyName: "policy-sample-2" validationActions: [Warn] diff --git a/examples/policy-binding-sample-3.yaml b/examples/policy-binding-sample-3.yaml new file mode 100644 index 0000000..1f8a5e1 --- /dev/null +++ b/examples/policy-binding-sample-3.yaml @@ -0,0 +1,11 @@ +apiVersion: monokle.com/v1alpha1 +kind: MonoklePolicyBinding +metadata: + name: policy-binding-sample-3 +spec: + policyName: "policy-sample-2" + validationActions: [Warn] + matchResources: + namespaceSelector: + matchLabels: + namespace: nstest1 \ No newline at end of file diff --git a/examples/policy-sample.yaml b/examples/policy-sample-1.yaml similarity index 95% rename from examples/policy-sample.yaml rename to examples/policy-sample-1.yaml index 4fb272a..7030db2 100644 --- a/examples/policy-sample.yaml +++ b/examples/policy-sample-1.yaml @@ -1,7 +1,7 @@ apiVersion: monokle.com/v1alpha1 kind: MonoklePolicy metadata: - name: policy-sample + name: policy-sample-1 spec: plugins: yaml-syntax: true diff --git a/k8s/manifests/service-account.yaml b/k8s/manifests/service-account.yaml index 347a55a..492b1b4 100644 --- a/k8s/manifests/service-account.yaml +++ b/k8s/manifests/service-account.yaml @@ -10,7 +10,7 @@ metadata: rules: - apiGroups: ["monokle.com"] resources: ["policies", "policybindings"] - verbs: ["list", "watch", "get"] + verbs: ["list", "watch"] --- kind: ClusterRoleBinding @@ -20,7 +20,7 @@ metadata: subjects: - kind: ServiceAccount name: monokle-policies - namespace: webhook-demo + namespace: monokle-admission-controller roleRef: kind: ClusterRole name: monokle-policies diff --git a/k8s/templates/deployment.yaml.template b/k8s/templates/deployment.yaml.template index d10c15d..253e8fb 100644 --- a/k8s/templates/deployment.yaml.template +++ b/k8s/templates/deployment.yaml.template @@ -2,7 +2,7 @@ apiVersion: apps/v1 kind: Deployment metadata: name: webhook-server - namespace: webhook-demo + namespace: monokle-admission-controller labels: app: webhook-server spec: @@ -15,9 +15,9 @@ spec: labels: app: webhook-server spec: - #securityContext: + # securityContext: # runAsNonRoot: true - # runAsUser: 1234 + # runAsUser: 1000 containers: - name: server image: admission-webhook @@ -33,7 +33,7 @@ spec: - name: MONOKLE_LOG_LEVEL value: DEBUG - name: MONOKLE_IGNORE_NAMESPACES - value: 'kube-node-lease,kube-public,kube-system' + value: 'kube-node-lease,kube-public,kube-system,monokle-admission-controller' volumes: - name: webhook-tls-certs secret: @@ -44,7 +44,7 @@ apiVersion: v1 kind: Service metadata: name: webhook-server - namespace: webhook-demo + namespace: monokle-admission-controller spec: selector: app: webhook-server diff --git a/k8s/templates/webhook.yaml.template b/k8s/templates/webhook.yaml.template index 23f63e9..0bc1909 100644 --- a/k8s/templates/webhook.yaml.template +++ b/k8s/templates/webhook.yaml.template @@ -3,18 +3,18 @@ kind: ValidatingWebhookConfiguration metadata: name: demo-webhook webhooks: - - name: webhook-server.webhook-demo.svc + - name: webhook-server.monokle-admission-controller.svc sideEffects: None admissionReviewVersions: ["v1", "v1beta1"] clientConfig: service: name: webhook-server - namespace: webhook-demo + namespace: monokle-admission-controller path: "/validate" caBundle: ${CA_PEM_B64} rules: - - operations: [ "CREATE" ] - apiGroups: [""] - apiVersions: ["v1"] - resources: ["pods"] + - operations: ["CREATE", "UPDATE"] + apiGroups: ["*"] + apiVersions: ["*"] + resources: ["*"] scope: "Namespaced" \ No newline at end of file diff --git a/scripts/deploy.sh b/scripts/deploy.sh index 1c60b8d..3dd2316 100755 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -16,7 +16,7 @@ # deploy.sh # -# Sets up the environment for the admission controller webhook demo in the active cluster. +# Sets up the environment for the admission controller webhook in the active cluster. set -euo pipefail @@ -29,13 +29,17 @@ resdir="${basedir}/../k8s/manifests" echo "Generating TLS keys ..." "${basedir}/generate-keys.sh" "$keydir" -# Create the `webhook-demo` namespace. This cannot be part of the YAML file as we first need to create the TLS secret, +# Create the `monokle-admission-controller` namespace. This cannot be part of the YAML file as we first need to create the TLS secret, # which would fail otherwise. echo "Creating Kubernetes objects ..." -kubectl create namespace webhook-demo +kubectl create namespace monokle-admission-controller + +# Create test namespaces +kubectl create namespace nstest1 +kubectl create namespace nstest2 # Create the TLS secret for the generated keys. -kubectl -n webhook-demo create secret tls webhook-server-tls \ +kubectl -n monokle-admission-controller create secret tls webhook-server-tls \ --cert "${keydir}/webhook-server-tls.crt" \ --key "${keydir}/webhook-server-tls.key" @@ -52,14 +56,10 @@ kubectl apply -f "${resdir}/monokle-policy-crd.yaml" kubectl apply -f "${resdir}/monokle-policy-binding-crd.yaml" # Namespaced -kubectl apply -f "${resdir}/service-account.yaml" -n webhook-demo +kubectl apply -f "${resdir}/service-account.yaml" -n monokle-admission-controller -skaffold run -n webhook-demo -f k8s/skaffold.yaml -# kubectl apply -f deployment.yaml -sleep 2 -# kubectl apply -f "${resdir}/monokle-policy-crd.yaml" -# kubectl apply -f "${resdir}/monokle-policy-binding-crd.yaml" -# kubectl apply -f "${resdir}/service-account.yaml" +skaffold run -n monokle-admission-controller -f k8s/skaffold.yaml +sleep 5 kubectl apply -f "${resdir}/webhook.yaml" # Delete the key directory to prevent abuse (DO NOT USE THESE KEYS ANYWHERE ELSE). diff --git a/scripts/generate-keys.sh b/scripts/generate-keys.sh index 7df6494..7ecb23d 100755 --- a/scripts/generate-keys.sh +++ b/scripts/generate-keys.sh @@ -17,7 +17,7 @@ # generate-keys.sh # # Generate a (self-signed) CA certificate and a certificate and private key to be used by the webhook demo server. -# The certificate will be issued for the Common Name (CN) of `webhook-server.webhook-demo.svc`, which is the +# The certificate will be issued for the Common Name (CN) of `webhook-server.monokle-admission-controller.svc`, which is the # cluster-internal DNS name for the service. # # NOTE: THIS SCRIPT EXISTS FOR DEMO PURPOSES ONLY. DO NOT USE IT FOR YOUR PRODUCTION WORKLOADS. @@ -35,14 +35,14 @@ req_extensions = v3_req distinguished_name = req_distinguished_name prompt = no [req_distinguished_name] -CN = webhook-server.webhook-demo.svc +CN = webhook-server.monokle-admission-controller.svc [ v3_req ] basicConstraints = CA:FALSE keyUsage = nonRepudiation, digitalSignature, keyEncipherment extendedKeyUsage = clientAuth, serverAuth subjectAltName = @alt_names [alt_names] -DNS.1 = webhook-server.webhook-demo.svc +DNS.1 = webhook-server.monokle-admission-controller.svc EOF @@ -51,5 +51,5 @@ openssl req -nodes -new -x509 -keyout ca.key -out ca.crt -subj "/CN=Admission Co # Generate the private key for the webhook server openssl genrsa -out webhook-server-tls.key 2048 # Generate a Certificate Signing Request (CSR) for the private key, and sign it with the private key of the CA. -openssl req -new -key webhook-server-tls.key -subj "/CN=webhook-server.webhook-demo.svc" -config server.conf \ +openssl req -new -key webhook-server-tls.key -subj "/CN=webhook-server.monokle-admission-controller.svc" -config server.conf \ | openssl x509 -req -CA ca.crt -CAkey ca.key -CAcreateserial -out webhook-server-tls.crt -extensions v3_req -extfile server.conf From 7da3da525d9bbdad5c7650276b3d75c69bbfe0a1 Mon Sep 17 00:00:00 2001 From: f1ames Date: Mon, 2 Oct 2023 14:36:08 +0200 Subject: [PATCH 18/26] fix: move ignored namespace check to very beginning of webhook handler --- admission-webhook/src/index.ts | 4 ++-- admission-webhook/src/utils/policy-manager.ts | 5 ----- admission-webhook/src/utils/validation-server.ts | 8 +++++--- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/admission-webhook/src/index.ts b/admission-webhook/src/index.ts index 7f8f730..cdad7e7 100644 --- a/admission-webhook/src/index.ts +++ b/admission-webhook/src/index.ts @@ -31,12 +31,12 @@ const logger = pino({ } ); - const policyManager = new PolicyManager(policyInformer, bindingsInformer, IGNORED_NAMESPACES, logger); + const policyManager = new PolicyManager(policyInformer, bindingsInformer, logger); const validatorManager = new ValidatorManager(policyManager, logger); await policyManager.start(); - const server = new ValidationServer(validatorManager, logger); + const server = new ValidationServer(validatorManager, IGNORED_NAMESPACES, logger); await server.start(); })(); diff --git a/admission-webhook/src/utils/policy-manager.ts b/admission-webhook/src/utils/policy-manager.ts index 90e9506..9e3c032 100644 --- a/admission-webhook/src/utils/policy-manager.ts +++ b/admission-webhook/src/utils/policy-manager.ts @@ -35,7 +35,6 @@ export class PolicyManager extends EventEmitter{ constructor( private readonly _policyInformer: InformerWrapper, private readonly _bindingInformer: InformerWrapper, - private readonly _ignoreNamespaces: string[], private readonly _logger: ReturnType, ) { super(); @@ -61,10 +60,6 @@ export class PolicyManager extends EventEmitter{ return []; } - if (this._ignoreNamespaces.includes(namespace)) { - return []; - } - return Array.from(this._bindings.values()) .map((binding) => { const policy = this._policies.get(binding.spec.policyName); diff --git a/admission-webhook/src/utils/validation-server.ts b/admission-webhook/src/utils/validation-server.ts index ee6723c..4c2d33f 100644 --- a/admission-webhook/src/utils/validation-server.ts +++ b/admission-webhook/src/utils/validation-server.ts @@ -48,6 +48,7 @@ export class ValidationServer { constructor( private readonly _validators: ValidatorManager, + private readonly _ignoredNamespaces: string[], private readonly _logger: ReturnType, private readonly _options: ValidationServerOptions = { port: 8443, @@ -102,7 +103,6 @@ export class ValidationServer { private async _initRouting() { this._server.post("/validate", async (req, _res): Promise => { - this._logger.debug({request: req}) this._logger.trace({requestBody: req.body}); const body = req.body as AdmissionRequest; @@ -120,8 +120,8 @@ export class ValidationServer { } } - if (!namespace) { - this._logger.error({msg: 'No namespace found', metadata: body.request}); + if (!namespace || this._ignoredNamespaces.includes(namespace)) { + this._logger.error({msg: 'No namespace found or namespace ignored', namespace}); return response; } @@ -131,6 +131,8 @@ export class ValidationServer { return response; } + this._logger.debug({request: req}); + const validators = this._validators.getMatchingValidators(resource, namespace); this._logger.debug({msg: 'Matching validators', count: validators.length}); From 9d64438e49bb735e720e01f780f6987c1e018008 Mon Sep 17 00:00:00 2001 From: f1ames Date: Mon, 2 Oct 2023 15:58:07 +0200 Subject: [PATCH 19/26] feat: Send k8s warnings on 'Warn' action --- .../src/utils/validation-server.ts | 100 ++++++++++++------ k8s/templates/webhook.yaml.template | 2 +- 2 files changed, 70 insertions(+), 32 deletions(-) diff --git a/admission-webhook/src/utils/validation-server.ts b/admission-webhook/src/utils/validation-server.ts index 4c2d33f..d6cd6b0 100644 --- a/admission-webhook/src/utils/validation-server.ts +++ b/admission-webhook/src/utils/validation-server.ts @@ -2,13 +2,13 @@ import fastify from "fastify"; import pino from 'pino'; import path from "path"; import {readFileSync} from "fs"; -import {Resource} from "@monokle/validation"; +import {Message, Resource, RuleLevel} from "@monokle/validation"; import {V1ObjectMeta} from "@kubernetes/client-node"; import {ValidatorManager} from "./validator-manager"; export type ValidationServerOptions = { - port: number; - host: string; + port: number + host: string }; export type AdmissionRequestObject = { @@ -30,18 +30,29 @@ export type AdmissionRequest = { } }; +// See +// https://pkg.go.dev/k8s.io/api/admission/v1#AdmissionResponse +// https://kubernetes.io/docs/reference/config-api/apiserver-admission.v1/#admission-k8s-io-v1-AdmissionReview export type AdmissionResponse = { - kind: string, - apiVersion: string, + kind: string + apiVersion: string response: { - uid: string, - allowed: boolean, + uid: string + allowed: boolean + warnings?: string[] status: { message: string } } }; +export type Violation = { + ruleId: string + message: Message + level?: RuleLevel + actions: string[] +} + export class ValidationServer { private _server: ReturnType; private _shouldValidate: boolean @@ -150,46 +161,42 @@ export class ValidationServer { } )); - // @TODO each result may have different `validationActions` (defined in bindings) so it should be handled - // it can by be grouping results by action and then performing action for each group - - const warnings = []; - const errors = []; + const violations: Violation[] = []; for (const validationResponse of validationResponses) { + const actions = validationResponse.policy.binding.validationActions; + for (const result of validationResponse.result.runs) { for (const item of result.results) { - if (item.level === "warning") { - warnings.push(item); - } else if (item.level === "error") { - errors.push(item); - } + violations.push({ + ruleId: item.ruleId, + message: item.message, + level: item.level, + actions: actions + }); } } } - if (errors.length > 0 || warnings.length > 0) { - const warningsList = warnings.map((e) => `${e.ruleId}: ${e.message.text}`).join("\n"); - const errorsList = errors.map((e) => `${e.ruleId}: ${e.message.text}`).join("\n"); - const message = []; + const violationsByAction = violations.reduce((acc: Record, violation: Violation) => { + const actions = violation.actions; - if (errors.length > 0) { - message.push(`\n${errors.length} errors found:\n${errorsList}\n`); - } + for (const action of actions) { + if (!acc[action]) { + acc[action] = []; + } - if (warnings.length > 0) { - message.push(`\n${warnings.length} warnings found:\n${warningsList}\n`); + acc[action].push(violation); } - message.push("\nYou can use Monokle (https://monokle.io/) to validate and fix those errors easily!"); + return acc; + }, {}); - response.response.allowed = false; - response.response.status.message = message.join(""); - } + const responseFull = this._handleViolationsByAction(violationsByAction, response); this._logger.debug({response}); this._logger.trace({resourceForValidation, validationResponses}); - return response; + return responseFull; }); } @@ -209,4 +216,35 @@ export class ValidationServer { return resource; } + + private _handleViolationsByAction(violationsByAction: Record, response: AdmissionResponse) { + for (const action of Object.keys(violationsByAction)) { + // 'Warn' action shouldbe mapped to warnings, see: + // - https://kubernetes.io/docs/reference/access-authn-authz/validating-admission-policy/#validation-actions + // - https://kubernetes.io/blog/2020/09/03/warnings/ + if (action.toLowerCase() === 'warn') { + response = this._handleViolationsAsWarn(violationsByAction[action], response); + } + } + + return response; + } + + private _handleViolationsAsWarn(violations: Violation[], response: AdmissionResponse) { + const warnings = violations.filter((v) => v.level === 'warning').map((e) => `${e.ruleId}: ${e.message.text}`); + const errors = violations.filter((v) => v.level === 'error').map((e) => `${e.ruleId}: ${e.message.text}`); + + if (errors.length > 0 || warnings.length > 0) { + response.response.warnings = [ + `Monokle Admission Controller found:`, + `Errors: ${errors.length}`, + ...errors, + `Warnings: ${warnings.length}`, + ...warnings, + "You can use Monokle (https://monokle.io/) to validate and fix those errors easily!" + ]; + } + + return response; + } } diff --git a/k8s/templates/webhook.yaml.template b/k8s/templates/webhook.yaml.template index 0bc1909..b87f27d 100644 --- a/k8s/templates/webhook.yaml.template +++ b/k8s/templates/webhook.yaml.template @@ -16,5 +16,5 @@ webhooks: - operations: ["CREATE", "UPDATE"] apiGroups: ["*"] apiVersions: ["*"] - resources: ["*"] + resources: ["pods"] scope: "Namespaced" \ No newline at end of file From 579a0b3806b2ad13f8eaa127102337076a5b5070 Mon Sep 17 00:00:00 2001 From: f1ames Date: Tue, 3 Oct 2023 09:29:12 +0200 Subject: [PATCH 20/26] fix: make admission controller match all resource kinds --- k8s/templates/webhook.yaml.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/templates/webhook.yaml.template b/k8s/templates/webhook.yaml.template index b87f27d..0bc1909 100644 --- a/k8s/templates/webhook.yaml.template +++ b/k8s/templates/webhook.yaml.template @@ -16,5 +16,5 @@ webhooks: - operations: ["CREATE", "UPDATE"] apiGroups: ["*"] apiVersions: ["*"] - resources: ["pods"] + resources: ["*"] scope: "Namespaced" \ No newline at end of file From db7a2c80f79a3366571c74b40fb598efbb968593 Mon Sep 17 00:00:00 2001 From: f1ames Date: Tue, 3 Oct 2023 09:37:02 +0200 Subject: [PATCH 21/26] fix: mark specific fields as required in Monokle CRDs --- k8s/manifests/monokle-policy-binding-crd.yaml | 5 +++++ k8s/manifests/monokle-policy-crd.yaml | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/k8s/manifests/monokle-policy-binding-crd.yaml b/k8s/manifests/monokle-policy-binding-crd.yaml index d7567d0..138ca6a 100644 --- a/k8s/manifests/monokle-policy-binding-crd.yaml +++ b/k8s/manifests/monokle-policy-binding-crd.yaml @@ -11,9 +11,14 @@ spec: schema: openAPIV3Schema: type: object + required: + - spec properties: spec: type: object + required: + - policyName + - validationActions properties: policyName: type: string diff --git a/k8s/manifests/monokle-policy-crd.yaml b/k8s/manifests/monokle-policy-crd.yaml index 25a5f31..709ddad 100644 --- a/k8s/manifests/monokle-policy-crd.yaml +++ b/k8s/manifests/monokle-policy-crd.yaml @@ -20,9 +20,13 @@ spec: # within an allOf, anyOf, oneOf or not, with the exception of the two pattern for x-kubernetes-int-or-string: true (see below)." openAPIV3Schema: type: object + required: + - spec properties: spec: type: object + required: + - plugins properties: plugins: type: object From 4bd6a0059a294a0bf26fa62abbec283836d385ea Mon Sep 17 00:00:00 2001 From: f1ames Date: Tue, 3 Oct 2023 09:50:28 +0200 Subject: [PATCH 22/26] chore: cleanup outdated comments --- admission-webhook/src/utils/get-informer.ts | 2 +- admission-webhook/src/utils/validation-server.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/admission-webhook/src/utils/get-informer.ts b/admission-webhook/src/utils/get-informer.ts index 9112f12..5fb4142 100644 --- a/admission-webhook/src/utils/get-informer.ts +++ b/admission-webhook/src/utils/get-informer.ts @@ -18,7 +18,7 @@ export async function getInformer( return {informer, start} } -function createInformerStarter(informer: Informer, onError?: k8s.ErrorCallback) { // not sure if can try to start informer multiple times +function createInformerStarter(informer: Informer, onError?: k8s.ErrorCallback) { return async () => { let tries = 0; let started = false; diff --git a/admission-webhook/src/utils/validation-server.ts b/admission-webhook/src/utils/validation-server.ts index d6cd6b0..283aaaa 100644 --- a/admission-webhook/src/utils/validation-server.ts +++ b/admission-webhook/src/utils/validation-server.ts @@ -219,7 +219,7 @@ export class ValidationServer { private _handleViolationsByAction(violationsByAction: Record, response: AdmissionResponse) { for (const action of Object.keys(violationsByAction)) { - // 'Warn' action shouldbe mapped to warnings, see: + // 'Warn' action should be mapped to warnings, see: // - https://kubernetes.io/docs/reference/access-authn-authz/validating-admission-policy/#validation-actions // - https://kubernetes.io/blog/2020/09/03/warnings/ if (action.toLowerCase() === 'warn') { From d9598090324b9f155c2d686267850d668f33e2a0 Mon Sep 17 00:00:00 2001 From: f1ames Date: Tue, 3 Oct 2023 09:55:16 +0200 Subject: [PATCH 23/26] fix: use wildcard for webhook scope by default --- k8s/templates/webhook.yaml.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/templates/webhook.yaml.template b/k8s/templates/webhook.yaml.template index 0bc1909..61157d6 100644 --- a/k8s/templates/webhook.yaml.template +++ b/k8s/templates/webhook.yaml.template @@ -17,4 +17,4 @@ webhooks: apiGroups: ["*"] apiVersions: ["*"] resources: ["*"] - scope: "Namespaced" \ No newline at end of file + scope: "*" \ No newline at end of file From 586aeed14a9366e68d087de3a5be2586f6bd6aaa Mon Sep 17 00:00:00 2001 From: f1ames Date: Tue, 3 Oct 2023 12:21:46 +0200 Subject: [PATCH 24/26] feat: block resource-link plugin --- admission-webhook/src/utils/policy-manager.ts | 39 +++++++++++-------- .../src/utils/policy-postprocessor.ts | 30 ++++++++++++++ 2 files changed, 52 insertions(+), 17 deletions(-) create mode 100644 admission-webhook/src/utils/policy-postprocessor.ts diff --git a/admission-webhook/src/utils/policy-manager.ts b/admission-webhook/src/utils/policy-manager.ts index 9e3c032..612ae25 100644 --- a/admission-webhook/src/utils/policy-manager.ts +++ b/admission-webhook/src/utils/policy-manager.ts @@ -1,9 +1,10 @@ -import {EventEmitter} from "events"; +import {EventEmitter} from 'events'; import pino from 'pino'; import {KubernetesObject} from '@kubernetes/client-node'; import {Config} from '@monokle/validation'; -import {InformerWrapper} from './get-informer'; -import {AdmissionRequestObject} from "./validation-server"; +import {InformerWrapper} from './get-informer.js'; +import {AdmissionRequestObject} from './validation-server.js'; +import {postprocess} from './policy-postprocessor.js'; export type MonoklePolicy = KubernetesObject & { spec: Config @@ -81,36 +82,40 @@ export class PolicyManager extends EventEmitter{ .filter((policy) => policy !== null) as MonokleApplicablePolicy[]; } - private onPolicy(policy: MonoklePolicy) { - this._logger.debug({msg: 'Policy updated', policy}); + private onPolicy(rawPolicy: MonoklePolicy) { + const policy = postprocess(rawPolicy); - this._policies.set(policy.metadata!.name!, policy); + this._logger.debug({msg: 'Policy updated', rawPolicy, policy}); + + this._policies.set(rawPolicy.metadata!.name!, policy); this.emit('policyUpdated', policy); } - private onPolicyRemoval(policy: MonoklePolicy) { - this._logger.debug({msg: 'Policy removed', policy}); + private onPolicyRemoval(rawPolicy: MonoklePolicy) { + const policy = postprocess(rawPolicy); + + this._logger.debug({msg: 'Policy removed', rawPolicy, policy}); - this._policies.delete(policy.metadata!.name!); + this._policies.delete(rawPolicy.metadata!.name!); this.emit('policyRemoved', policy); } - private onBinding(binding: MonoklePolicyBinding) { - this._logger.debug({msg: 'Binding updated', binding}); + private onBinding(rawBinding: MonoklePolicyBinding) { + this._logger.debug({msg: 'Binding updated', rawBinding}); - this._bindings.set(binding.metadata!.name!, binding); + this._bindings.set(rawBinding.metadata!.name!, rawBinding); - this.emit('bindingUpdated', binding); + this.emit('bindingUpdated', rawBinding); } - private onBindingRemoval(binding: MonoklePolicyBinding) { - this._logger.debug({msg: 'Binding removed', binding}); + private onBindingRemoval(rawBinding: MonoklePolicyBinding) { + this._logger.debug({msg: 'Binding removed', rawBinding}); - this._bindings.delete(binding.metadata!.name!); + this._bindings.delete(rawBinding.metadata!.name!); - this.emit('bindingRemoved', binding); + this.emit('bindingRemoved', rawBinding); } private isResourceMatching(binding: MonoklePolicyBinding, resource: AdmissionRequestObject): boolean { diff --git a/admission-webhook/src/utils/policy-postprocessor.ts b/admission-webhook/src/utils/policy-postprocessor.ts new file mode 100644 index 0000000..c4063bc --- /dev/null +++ b/admission-webhook/src/utils/policy-postprocessor.ts @@ -0,0 +1,30 @@ +import {Config} from '@monokle/validation'; +import {MonoklePolicy} from './policy-manager.js'; + +const PLUGIN_BLOCKLIST = [ + 'resource-links', +]; + +export function postprocess(policy: MonoklePolicy) { + const newPolicy = { ...policy }; + newPolicy.spec = blockPlugins(newPolicy.spec); + return newPolicy; +} + +function blockPlugins(policySpec: Config): Config { + if (policySpec.plugins === undefined) { + return policySpec; + } + + const newPlugins = { ...policySpec.plugins }; + for (const blockedPlugin of PLUGIN_BLOCKLIST) { + if (newPlugins[blockedPlugin] === true) { + newPlugins[blockedPlugin] = false; + } + } + + return { + ...policySpec, + plugins: newPlugins, + }; +} From 32378afb53df47fd7fd8abe0459f681135e1f3fb Mon Sep 17 00:00:00 2001 From: f1ames Date: Tue, 3 Oct 2023 12:22:04 +0200 Subject: [PATCH 25/26] chore: fix imports --- admission-webhook/src/utils/validation-server.ts | 12 ++++++------ admission-webhook/src/utils/validator-manager.ts | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/admission-webhook/src/utils/validation-server.ts b/admission-webhook/src/utils/validation-server.ts index 283aaaa..577189f 100644 --- a/admission-webhook/src/utils/validation-server.ts +++ b/admission-webhook/src/utils/validation-server.ts @@ -1,10 +1,10 @@ -import fastify from "fastify"; +import fastify from 'fastify'; import pino from 'pino'; -import path from "path"; -import {readFileSync} from "fs"; -import {Message, Resource, RuleLevel} from "@monokle/validation"; -import {V1ObjectMeta} from "@kubernetes/client-node"; -import {ValidatorManager} from "./validator-manager"; +import path from 'path'; +import {readFileSync} from 'fs'; +import {Message, Resource, RuleLevel} from '@monokle/validation'; +import {V1ObjectMeta} from '@kubernetes/client-node'; +import {ValidatorManager} from './validator-manager.js'; export type ValidationServerOptions = { port: number diff --git a/admission-webhook/src/utils/validator-manager.ts b/admission-webhook/src/utils/validator-manager.ts index aca87fa..5eea015 100644 --- a/admission-webhook/src/utils/validator-manager.ts +++ b/admission-webhook/src/utils/validator-manager.ts @@ -1,7 +1,7 @@ import pino from 'pino'; -import {AnnotationSuppressor, Config, DisabledFixer, FingerprintSuppressor, MonokleValidator, RemotePluginLoader, ResourceParser, SchemaLoader} from "@monokle/validation"; -import {MonokleApplicablePolicy, MonoklePolicy, PolicyManager} from "./policy-manager"; -import {AdmissionRequestObject} from './validation-server'; +import {AnnotationSuppressor, Config, DisabledFixer, FingerprintSuppressor, MonokleValidator, RemotePluginLoader, ResourceParser, SchemaLoader} from '@monokle/validation'; +import {MonokleApplicablePolicy, MonoklePolicy, PolicyManager} from './policy-manager.js'; +import {AdmissionRequestObject} from './validation-server.js'; export type MonokleApplicableValidator = { validator: MonokleValidator, From 7b86f839c97464522ad416c9f2535c30d0a9e2ed Mon Sep 17 00:00:00 2001 From: f1ames Date: Tue, 3 Oct 2023 16:38:30 +0200 Subject: [PATCH 26/26] fix: fix routing initialization --- admission-webhook/src/utils/validation-server.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/admission-webhook/src/utils/validation-server.ts b/admission-webhook/src/utils/validation-server.ts index 577189f..3eb5966 100644 --- a/admission-webhook/src/utils/validation-server.ts +++ b/admission-webhook/src/utils/validation-server.ts @@ -111,7 +111,7 @@ export class ValidationServer { } } - private async _initRouting() { + private _initRouting() { this._server.post("/validate", async (req, _res): Promise => { this._logger.trace({requestBody: req.body});