From e6ce712b22294f5ce950ec6837847d89eab0424d Mon Sep 17 00:00:00 2001 From: f1ames Date: Fri, 29 Sep 2023 13:50:52 +0200 Subject: [PATCH] refactor: add support for policy binding [WIP] --- README.md | 3 +- admission-webhook/src/index.ts | 19 +++--- admission-webhook/src/utils/get-informer.ts | 46 +++++++++----- admission-webhook/src/utils/policy-manager.ts | 60 ++++++++++++------- .../src/utils/validation-server.ts | 28 ++++++--- .../src/utils/validator-manager.ts | 42 +++++++++++-- k8s/manifests/service-account.yaml | 2 +- k8s/templates/deployment.yaml.template | 6 ++ scripts/deploy.sh | 13 +++- 9 files changed, 153 insertions(+), 66 deletions(-) diff --git a/README.md b/README.md index f0696f3..3fe75ae 100644 --- a/README.md +++ b/README.md @@ -97,7 +97,8 @@ You can also do manual clean-up and re-run `./deploy.sh` script again: kubectl delete all -n webhook-demo --all && \ kubectl delete validatingwebhookconfiguration.admissionregistration.k8s.io/demo-webhook -n webhook-demo && \ kubectl delete namespace webhook-demo && \ -kubectl delete crd policies.monokle.com +kubectl delete crd policies.monokle.com && \ +kubectl delete crd policybindings.monokle.com ``` ### Refs diff --git a/admission-webhook/src/index.ts b/admission-webhook/src/index.ts index ff49e3d..66934d3 100644 --- a/admission-webhook/src/index.ts +++ b/admission-webhook/src/index.ts @@ -4,31 +4,34 @@ import {MonoklePolicy, MonoklePolicyBinding, PolicyManager} from './utils/policy import {ValidatorManager} from './utils/validator-manager.js'; import {ValidationServer} from './utils/validation-server.js'; +const LOG_LEVEL = (process.env.MONOKLE_LOG_LEVEL || 'warn').toLowerCase(); +const IGNORED_NAMESPACES = (process.env.MONOKLE_IGNORE_NAMESPACES || '').split(','); + const logger = pino({ name: 'Monokle', - level: 'trace', + level: LOG_LEVEL, }); (async() => { const policyInformer = await getInformer( - 'monokle.io', - 'v1', + 'monokle.com', + 'v1alpha1', 'policies', (err: any) => { - logger.error({msg: 'Informer: Policies: Error', err}); + logger.error({msg: 'Informer: Policies: Error', err: err.message, body: err.body}); } ); const bindingsInformer = await getInformer( - 'monokle.io', - 'v1', + 'monokle.com', + 'v1alpha1', 'policybindings', (err: any) => { - logger.error({msg: 'Informer: Bindings: Error', err}); + logger.error({msg: 'Informer: Bindings: Error', err: err.message, body: err.body}); } ); - const policyManager = new PolicyManager(policyInformer, bindingsInformer, logger); + const policyManager = new PolicyManager(policyInformer, bindingsInformer, IGNORED_NAMESPACES, logger); const validatorManager = new ValidatorManager(policyManager); await policyManager.start(); diff --git a/admission-webhook/src/utils/get-informer.ts b/admission-webhook/src/utils/get-informer.ts index 34d950e..9112f12 100644 --- a/admission-webhook/src/utils/get-informer.ts +++ b/admission-webhook/src/utils/get-informer.ts @@ -2,27 +2,41 @@ import k8s from '@kubernetes/client-node'; export type Informer = k8s.Informer & k8s.ObjectCache; +export type InformerWrapper = { + informer: Informer, + start: () => Promise +} + const ERROR_RESTART_INTERVAL = 500; -export async function getInformer(group: string, version: string, plural: string, onError?: k8s.ErrorCallback) { - let informer: Informer | null = null; - - let tries = 0; - while (!informer) { - try { - tries++; - informer = await createInformer(group, version, plural, onError); - return informer; - } catch (err: any) { - if (onError) { - onError(err); - } +export async function getInformer( + group: string, version: string, plural: string, onError?: k8s.ErrorCallback +): Promise> { + const informer = await createInformer(group, version, plural, onError); + const start = createInformerStarter(informer, onError); + + return {informer, start} +} - await new Promise((resolve) => setTimeout(resolve, ERROR_RESTART_INTERVAL)); +function createInformerStarter(informer: Informer, onError?: k8s.ErrorCallback) { // not sure if can try to start informer multiple times + return async () => { + let tries = 0; + let started = false; + + while (!started) { + try { + tries++; + await informer.start(); + started = true; + } catch (err: any) { + if (onError) { + onError(err); + } + + await new Promise((resolve) => setTimeout(resolve, ERROR_RESTART_INTERVAL)); + } } } - - return informer; } async function createInformer(group: string, version: string, plural: string, onError?: k8s.ErrorCallback) { diff --git a/admission-webhook/src/utils/policy-manager.ts b/admission-webhook/src/utils/policy-manager.ts index 25c5bfa..4f43cc4 100644 --- a/admission-webhook/src/utils/policy-manager.ts +++ b/admission-webhook/src/utils/policy-manager.ts @@ -1,17 +1,24 @@ import pino from 'pino'; import {KubernetesObject} from '@kubernetes/client-node'; import {Config} from '@monokle/validation'; -import {Informer} from './get-informer'; +import {InformerWrapper} from './get-informer'; export type MonoklePolicy = KubernetesObject & { spec: Config } +export type MonoklePolicyBindingConfiguration = { + policyName: string + validationActions: ['Warn'] +} + export type MonoklePolicyBinding = KubernetesObject & { - spec: { - policyName: string - validationActions: 'Warn' - } + spec: MonoklePolicyBindingConfiguration +} + +export type MonokleApplicablePolicy = { + policy: Config, + binding: MonoklePolicyBindingConfiguration } export class PolicyManager { @@ -19,17 +26,18 @@ export class PolicyManager { private _bindings = new Map(); // Map // @TODO use policyName as key instead of bindingName? constructor( - private readonly _policyInformer: Informer, - private readonly _bindingInformer: Informer, + private readonly _policyInformer: InformerWrapper, + private readonly _bindingInformer: InformerWrapper, + private readonly _ignoreNamespaces: string[], private readonly _logger: ReturnType, ) { - this._policyInformer.on('add', this.onPolicy); - this._policyInformer.on('update', this.onPolicy); - this._policyInformer.on('delete', this.onPolicyRemoval); + this._policyInformer.informer.on('add', this.onPolicy.bind(this)); + this._policyInformer.informer.on('update', this.onPolicy.bind(this)); + this._policyInformer.informer.on('delete', this.onPolicyRemoval.bind(this)); - this._bindingInformer.on('add', this.onBinding); - this._bindingInformer.on('update', this.onBinding); - this._bindingInformer.on('delete', this.onBindingRemoval); + this._bindingInformer.informer.on('add', this.onBinding.bind(this)); + this._bindingInformer.informer.on('update', this.onBinding.bind(this)); + this._bindingInformer.informer.on('delete', this.onBindingRemoval.bind(this)); } async start() { @@ -37,20 +45,26 @@ export class PolicyManager { await this._bindingInformer.start(); } - getMatchingPolicies() { // @TODO pass resource data so it can be matched according to matchResources definition (when it's implemented) + getMatchingPolicies(): MonokleApplicablePolicy[] { // @TODO pass resource data so it can be matched according to matchResources definition (when it's implemented) if (this._bindings.size === 0) { return []; } - return Array.from(this._bindings.values()).map((binding) => { - const policy = this._policies.get(binding.spec.policyName); - - if (!policy) { - this._logger.error({msg: 'Binding is pointing to missing policy', binding}); - } - - return policy; - }).filter((policy) => policy !== undefined); + return Array.from(this._bindings.values()) + .map((binding) => { + const policy = this._policies.get(binding.spec.policyName); + + if (!policy) { + this._logger.error({msg: 'Binding is pointing to missing policy', binding}); + return null; + } + + return { + policy: policy.spec, + binding: binding.spec + } + }) + .filter((policy) => policy !== null) as MonokleApplicablePolicy[]; } private onPolicy(policy: MonoklePolicy) { diff --git a/admission-webhook/src/utils/validation-server.ts b/admission-webhook/src/utils/validation-server.ts index 2763ec2..a8e885d 100644 --- a/admission-webhook/src/utils/validation-server.ts +++ b/admission-webhook/src/utils/validation-server.ts @@ -128,17 +128,27 @@ export class ValidationServer { } const resourceForValidation = this._createResourceForValidation(body); - // @TODO iterate over validators and run them all - const validationResponse = await validators[0].validate({ resources: [resourceForValidation] }); + const validationResponses = await Promise.all(validators.map(async (validator) => { + return { + result: await validator.validator.validate({ resources: [resourceForValidation] }), + policy: validator.policy + }; + } + )); + + // @TODO each result may have different `validationActions` (defined in bindings) so it should be handled + // it can by be grouping results by action and then performing action for each group const warnings = []; const errors = []; - for (const result of validationResponse.runs) { - for (const item of result.results) { - if (item.level === "warning") { - warnings.push(item); - } else if (item.level === "error") { - errors.push(item); + for (const validationResponse of validationResponses) { + for (const result of validationResponse.result.runs) { + for (const item of result.results) { + if (item.level === "warning") { + warnings.push(item); + } else if (item.level === "error") { + errors.push(item); + } } } } @@ -163,7 +173,7 @@ export class ValidationServer { } this._logger.debug({response}); - this._logger.trace({resourceForValidation, validationResponse}); + this._logger.trace({resourceForValidation, validationResponses}); return response; }); diff --git a/admission-webhook/src/utils/validator-manager.ts b/admission-webhook/src/utils/validator-manager.ts index 28d7590..ae9c190 100644 --- a/admission-webhook/src/utils/validator-manager.ts +++ b/admission-webhook/src/utils/validator-manager.ts @@ -1,5 +1,10 @@ -import {MonokleValidator} from "@monokle/validation"; -import {PolicyManager} from "./policy-manager"; +import {AnnotationSuppressor, Config, DisabledFixer, FingerprintSuppressor, MonokleValidator, RemotePluginLoader, ResourceParser, SchemaLoader} from "@monokle/validation"; +import {MonokleApplicablePolicy, PolicyManager} from "./policy-manager"; + +export type MonokleApplicableValidator = { + validator: MonokleValidator, + policy: MonokleApplicablePolicy +} export class ValidatorManager { private _validators = new Map(); // Map @@ -12,14 +17,41 @@ export class ValidatorManager { // it would affect performance of the admission webhook response time } - getMatchingValidators(): MonokleValidator[] { + getMatchingValidators(): MonokleApplicableValidator[] { const matchingPolicies = this._policyManager.getMatchingPolicies(); if (matchingPolicies.length === 0) { return []; } - // @TODO - return []; + return matchingPolicies.map((policy) => { + if (!this._validators.has(policy.binding.policyName)) { + this.setupValidator(policy.binding.policyName, policy.policy); + } + + return { + validator: this._validators.get(policy.binding.policyName)!, + policy + } + }); + } + + private async setupValidator(policyName: string, policy: Config) { + if (this._validators.has(policyName)) { + this._validators.get(policyName)!.preload(policy); + } else { + const validator = new MonokleValidator( + { + loader: new RemotePluginLoader(), + parser: new ResourceParser(), + schemaLoader: new SchemaLoader(), + suppressors: [new AnnotationSuppressor(), new FingerprintSuppressor()], + fixer: new DisabledFixer(), + }, + policy + ); + + this._validators.set(policyName, validator); + } } } \ No newline at end of file diff --git a/k8s/manifests/service-account.yaml b/k8s/manifests/service-account.yaml index 2552766..347a55a 100644 --- a/k8s/manifests/service-account.yaml +++ b/k8s/manifests/service-account.yaml @@ -19,7 +19,7 @@ metadata: name: monokle-policies subjects: - kind: ServiceAccount - name: default + name: monokle-policies namespace: webhook-demo roleRef: kind: ClusterRole diff --git a/k8s/templates/deployment.yaml.template b/k8s/templates/deployment.yaml.template index c8587cf..d10c15d 100644 --- a/k8s/templates/deployment.yaml.template +++ b/k8s/templates/deployment.yaml.template @@ -29,10 +29,16 @@ spec: - name: webhook-tls-certs mountPath: /run/secrets/tls readOnly: true + env: + - name: MONOKLE_LOG_LEVEL + value: DEBUG + - name: MONOKLE_IGNORE_NAMESPACES + value: 'kube-node-lease,kube-public,kube-system' volumes: - name: webhook-tls-certs secret: secretName: webhook-server-tls + serviceAccountName: monokle-policies --- apiVersion: v1 kind: Service diff --git a/scripts/deploy.sh b/scripts/deploy.sh index 9493787..1c60b8d 100755 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -47,12 +47,19 @@ ca_pem_b64="$(openssl base64 -A <"${keydir}/ca.crt")" sed -e 's@${CA_PEM_B64}@'"$ca_pem_b64"'@g' <"${templdir}/webhook.yaml.template" > "${resdir}/webhook.yaml" cp "${templdir}/deployment.yaml.template" "${resdir}/deployment.yaml" +# Cluster-wide +kubectl apply -f "${resdir}/monokle-policy-crd.yaml" +kubectl apply -f "${resdir}/monokle-policy-binding-crd.yaml" + +# Namespaced +kubectl apply -f "${resdir}/service-account.yaml" -n webhook-demo + skaffold run -n webhook-demo -f k8s/skaffold.yaml # kubectl apply -f deployment.yaml sleep 2 -kubectl apply -f "${resdir}/monokle-policy-crd.yaml" -kubectl apply -f "${resdir}/monokle-policy-binding-crd.yaml" -kubectl apply -f "${resdir}/service-account.yaml" +# kubectl apply -f "${resdir}/monokle-policy-crd.yaml" +# kubectl apply -f "${resdir}/monokle-policy-binding-crd.yaml" +# kubectl apply -f "${resdir}/service-account.yaml" kubectl apply -f "${resdir}/webhook.yaml" # Delete the key directory to prevent abuse (DO NOT USE THESE KEYS ANYWHERE ELSE).