From 6f8e0ed246506fea13b3ecee50958200b8a129ad Mon Sep 17 00:00:00 2001 From: Teodor Pripoae Date: Fri, 26 Feb 2021 11:29:56 +0200 Subject: [PATCH 1/8] feat: expose service, increase memory limits --- config/default/manager_auth_proxy_patch.yaml | 2 +- config/deploy/operator.yml | 23 +++++++++++++++++--- config/manager/manager.yaml | 21 ++++++++++++++++-- 3 files changed, 40 insertions(+), 6 deletions(-) diff --git a/config/default/manager_auth_proxy_patch.yaml b/config/default/manager_auth_proxy_patch.yaml index 8da60dd..6b7b0a8 100644 --- a/config/default/manager_auth_proxy_patch.yaml +++ b/config/default/manager_auth_proxy_patch.yaml @@ -21,6 +21,6 @@ spec: name: https - name: manager args: - - "--metrics-addr=127.0.0.1:8080" + - "--metrics-addr=0.0.0.0:8080" - "--enable-leader-election" - "--sync-period=20s" \ No newline at end of file diff --git a/config/deploy/operator.yml b/config/deploy/operator.yml index b9116f1..127f2f2 100644 --- a/config/deploy/operator.yml +++ b/config/deploy/operator.yml @@ -351,6 +351,23 @@ subjects: name: template-operator-manager namespace: platform-system --- +apiVersion: v1 +kind: Service +metadata: + annotations: + prometheus.io/scrape: "true" + labels: + control-plane: template-operator + name: template-operator-template-operator + namespace: platform-system +spec: + ports: + - name: prometheus + port: 8080 + protocol: TCP + selector: + control-plane: template-operator +--- apiVersion: apps/v1 kind: Deployment metadata: @@ -380,7 +397,7 @@ spec: - containerPort: 8443 name: https - args: - - --metrics-addr=127.0.0.1:8080 + - --metrics-addr=0.0.0.0:8080 - --enable-leader-election - --sync-period=20s image: flanksource/template-operator:v1 @@ -388,9 +405,9 @@ spec: resources: limits: cpu: 100m - memory: 30Mi + memory: 130Mi requests: cpu: 100m - memory: 20Mi + memory: 120Mi serviceAccount: template-operator-manager terminationGracePeriodSeconds: 10 diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 88b1eac..71eb970 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -30,9 +30,26 @@ spec: resources: limits: cpu: 100m - memory: 30Mi + memory: 130Mi requests: cpu: 100m - memory: 20Mi + memory: 120Mi terminationGracePeriodSeconds: 10 serviceAccount: template-operator-manager +--- +apiVersion: v1 +kind: Service +metadata: + name: template-operator + namespace: platform-system + annotations: + prometheus.io/scrape: "true" + labels: + control-plane: template-operator +spec: + selector: + control-plane: template-operator + ports: + - name: prometheus + protocol: TCP + port: 8080 \ No newline at end of file From 1aba95864b603c2a37bec8c9dca64bdd6114cd83 Mon Sep 17 00:00:00 2001 From: Teodor Pripoae Date: Fri, 26 Feb 2021 11:30:53 +0200 Subject: [PATCH 2/8] fix: bump kommons to v0.3.0 --- go.mod | 3 ++- go.sum | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 65b6362..23d5e28 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.13 require ( github.com/flanksource/commons v1.4.3 - github.com/flanksource/kommons v0.1.9 + github.com/flanksource/kommons v0.3.0 github.com/go-logr/logr v0.3.0 github.com/go-logr/zapr v0.3.0 // indirect github.com/go-openapi/jsonpointer v0.19.3 @@ -13,6 +13,7 @@ require ( github.com/onsi/ginkgo v1.12.1 github.com/onsi/gomega v1.10.1 github.com/pkg/errors v0.9.1 + github.com/prometheus/client_golang v1.7.1 github.com/sirupsen/logrus v1.7.0 github.com/sykesm/zap-logfmt v0.0.4 github.com/tidwall/gjson v1.6.7 diff --git a/go.sum b/go.sum index 55b06dc..1ff79e6 100644 --- a/go.sum +++ b/go.sum @@ -209,6 +209,8 @@ github.com/flanksource/kommons v0.1.8 h1:TMw1eyi7j2xTOFdT1Jmfins70z2aPXJJ0lT1yev github.com/flanksource/kommons v0.1.8/go.mod h1:oaYRidwjSC8KULM861DMPFc8P64JkcA+GNBZkqpSsUY= github.com/flanksource/kommons v0.1.9 h1:o0jYg5M6yjT02NcjoyDBrDnPYhvfkS7IopNjLZUOLIw= github.com/flanksource/kommons v0.1.9/go.mod h1:oaYRidwjSC8KULM861DMPFc8P64JkcA+GNBZkqpSsUY= +github.com/flanksource/kommons v0.3.0 h1:b3IBm1JaAfln8cWJkewk4DfzDIzUG5OotmPnX9Y8IGU= +github.com/flanksource/kommons v0.3.0/go.mod h1:oaYRidwjSC8KULM861DMPFc8P64JkcA+GNBZkqpSsUY= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/fortytw2/leaktest v1.2.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= From b3278adf07d23aee18919a669a7d4d4429d27bb9 Mon Sep 17 00:00:00 2001 From: Teodor Pripoae Date: Fri, 26 Feb 2021 11:31:45 +0200 Subject: [PATCH 3/8] feat: expose total count/failed/success template runs metrics --- controllers/template_controller.go | 51 +++++++++++++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/controllers/template_controller.go b/controllers/template_controller.go index 91ebb3f..d640171 100644 --- a/controllers/template_controller.go +++ b/controllers/template_controller.go @@ -23,17 +23,49 @@ import ( templatev1 "github.com/flanksource/template-operator/api/v1" "github.com/flanksource/template-operator/k8s" "github.com/go-logr/logr" + "github.com/prometheus/client_golang/prometheus" kerrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/tools/record" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/metrics" "sigs.k8s.io/controller-runtime/pkg/reconcile" ) +var ( + templateCount = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "template_operator_template_count", + Help: "Total template runs count", + }, + []string{"template"}, + ) + templateSuccess = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "template_operator_template_success", + Help: "Total successful template runs count", + }, + []string{"template"}, + ) + templateFailed = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "template_operator_template_failed", + Help: "Total failed template runs count", + }, + []string{"template"}, + ) +) + +func init() { + metrics.Registry.MustRegister(templateCount, templateSuccess, templateFailed) +} + // TemplateReconciler reconciles a Template object type TemplateReconciler struct { ControllerClient client.Client Client *kommons.Client + Events record.EventRecorder Log logr.Logger Scheme *runtime.Scheme Cache *k8s.SchemaCache @@ -44,6 +76,7 @@ type TemplateReconciler struct { func (r *TemplateReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { ctx := context.Background() log := r.Log.WithValues("template", req.NamespacedName) + name := req.NamespacedName.String() template := &templatev1.Template{} if err := r.ControllerClient.Get(ctx, req.NamespacedName, template); err != nil { @@ -52,22 +85,38 @@ func (r *TemplateReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { return reconcile.Result{}, nil } log.Error(err, "failed to get template") + incFailed(name) return reconcile.Result{}, err } - tm, err := k8s.NewTemplateManager(r.Client, log, r.Cache) + tm, err := k8s.NewTemplateManager(r.Client, log, r.Cache, r.Events) if err != nil { + incFailed(name) return reconcile.Result{}, err } if err := tm.Run(ctx, template); err != nil { + incFailed(name) return reconcile.Result{}, err } + incSuccess(name) return ctrl.Result{}, nil } func (r *TemplateReconciler) SetupWithManager(mgr ctrl.Manager) error { r.ControllerClient = mgr.GetClient() + r.Events = mgr.GetEventRecorderFor("template-operator") + return ctrl.NewControllerManagedBy(mgr). For(&templatev1.Template{}). Complete(r) } + +func incSuccess(name string) { + templateCount.WithLabelValues(name).Inc() + templateSuccess.WithLabelValues(name).Inc() +} + +func incFailed(name string) { + templateCount.WithLabelValues(name).Inc() + templateFailed.WithLabelValues(name).Inc() +} From 68701478637c1f14df02cf3f32fc0dff7ffd2508 Mon Sep 17 00:00:00 2001 From: Teodor Pripoae Date: Fri, 26 Feb 2021 11:32:20 +0200 Subject: [PATCH 4/8] feat: trigger events on source object if templating fails --- k8s/suite_test.go | 24 ++++++++++++++++++++++++ k8s/template_manager.go | 13 ++++++++++++- k8s/template_manager_test.go | 3 ++- 3 files changed, 38 insertions(+), 2 deletions(-) diff --git a/k8s/suite_test.go b/k8s/suite_test.go index 5407297..10de5ad 100644 --- a/k8s/suite_test.go +++ b/k8s/suite_test.go @@ -1,13 +1,37 @@ package k8s_test import ( + "fmt" "testing" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/runtime" ) func TestK8s(t *testing.T) { RegisterFailHandler(Fail) RunSpecs(t, "K8S Suite") } + +type TestEventRecorder struct{} + +func (r *TestEventRecorder) Event(object runtime.Object, eventtype, reason, message string) { + r.event(object, eventtype, reason, message, map[string]string{}) +} + +// Eventf is just like Event, but with Sprintf for the message field. +func (r *TestEventRecorder) Eventf(object runtime.Object, eventtype, reason, messageFmt string, args ...interface{}) { + msg := fmt.Sprintf(messageFmt, args...) + r.event(object, eventtype, reason, msg, map[string]string{}) +} + +// AnnotatedEventf is just like eventf, but with annotations attached +func (r *TestEventRecorder) AnnotatedEventf(object runtime.Object, annotations map[string]string, eventtype, reason, messageFmt string, args ...interface{}) { + msg := fmt.Sprintf(messageFmt, args...) + r.event(object, eventtype, reason, msg, annotations) +} + +func (r *TestEventRecorder) event(object runtime.Object, eventtype, reason, message string, annotations map[string]string) { + fmt.Printf("Received event type=%s reason=%s message='%s' on object %v\n", eventtype, reason, message, object) +} diff --git a/k8s/template_manager.go b/k8s/template_manager.go index 8d44863..ccb8101 100644 --- a/k8s/template_manager.go +++ b/k8s/template_manager.go @@ -23,6 +23,7 @@ import ( "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/record" "sigs.k8s.io/yaml" ) @@ -38,6 +39,7 @@ type TemplateManager struct { PatchApplier *PatchApplier SchemaManager *SchemaManager FuncMap template.FuncMap + Events record.EventRecorder } type ResourcePatch struct { @@ -59,7 +61,7 @@ type ForEach struct { Map map[string]interface{} } -func NewTemplateManager(c *kommons.Client, log logr.Logger, cache *SchemaCache) (*TemplateManager, error) { +func NewTemplateManager(c *kommons.Client, log logr.Logger, cache *SchemaCache, events record.EventRecorder) (*TemplateManager, error) { clientset, _ := c.GetClientset() restConfig, err := c.GetRESTConfig() @@ -87,6 +89,7 @@ func NewTemplateManager(c *kommons.Client, log logr.Logger, cache *SchemaCache) Client: c, Interface: clientset, Log: log, + Events: events, PatchApplier: patchApplier, SchemaManager: schemaManager, FuncMap: functions.FuncMap(), @@ -156,16 +159,19 @@ func (tm *TemplateManager) Run(ctx context.Context, template *templatev1.Templat for _, source := range sources { target := &source + if !template.Spec.Onceoff || !alreadyApplied(template, *target) { for _, patch := range template.Spec.Patches { target, err = tm.PatchApplier.Apply(target, patch, PatchTypeYaml) if err != nil { + tm.Events.Eventf(&source, v1.EventTypeWarning, "Failed", "Failed to apply patch") return err } } for _, patch := range template.Spec.JsonPatches { target, err = tm.PatchApplier.Apply(target, patch.Patch, PatchTypeJSON) if err != nil { + tm.Events.Eventf(&source, v1.EventTypeWarning, "Failed", "Failed to apply patch") return err } } @@ -173,6 +179,7 @@ func (tm *TemplateManager) Run(ctx context.Context, template *templatev1.Templat target = markApplied(template, target) stripAnnotations(target) if err := tm.Client.ApplyUnstructured(source.GetNamespace(), target); err != nil { + tm.Events.Eventf(&source, v1.EventTypeWarning, "Failed", "Failed to apply object") return err } } @@ -181,6 +188,7 @@ func (tm *TemplateManager) Run(ctx context.Context, template *templatev1.Templat for _, item := range template.Spec.Resources { objs, err := tm.getObjects(item.Raw, target.Object) if err != nil { + tm.Events.Eventf(&source, v1.EventTypeWarning, "Failed", "Failed to get objects") return err } @@ -200,6 +208,7 @@ func (tm *TemplateManager) Run(ctx context.Context, template *templatev1.Templat tm.Log.Info("Applying", "kind", obj.GetKind(), "namespace", obj.GetNamespace(), "name", obj.GetName()) } if err := tm.Client.ApplyUnstructured(obj.GetNamespace(), &obj); err != nil { + tm.Events.Eventf(&source, v1.EventTypeWarning, "Failed", "Failed to apply new resource kind=%s name=%s", obj.GetKind(), obj.GetName()) return err } } @@ -208,6 +217,7 @@ func (tm *TemplateManager) Run(ctx context.Context, template *templatev1.Templat if template.Spec.CopyToNamespaces != nil { namespaces, err := tm.getNamespaces(ctx, *template.Spec.CopyToNamespaces) if err != nil { + tm.Events.Eventf(&source, v1.EventTypeWarning, "Failed", "Failed to get namespaces") return errors.Wrap(err, "failed to get namespaces") } @@ -225,6 +235,7 @@ func (tm *TemplateManager) Run(ctx context.Context, template *templatev1.Templat } if err := tm.Client.ApplyUnstructured(newResource.GetNamespace(), newResource); err != nil { + tm.Events.Eventf(&source, v1.EventTypeWarning, "Failed", "Failed to copy to namespace %s", namespace) return err } } diff --git a/k8s/template_manager_test.go b/k8s/template_manager_test.go index 893f39f..5999e1a 100644 --- a/k8s/template_manager_test.go +++ b/k8s/template_manager_test.go @@ -70,8 +70,9 @@ spec: synchronous_mode: false ` + eventsRecorder := &TestEventRecorder{} cache := k8s.NewSchemaCache(clientset(), 5*time.Minute, testLog) - templateManager, err := k8s.NewTemplateManager(kommonsClient(), testLog, cache) + templateManager, err := k8s.NewTemplateManager(kommonsClient(), testLog, cache, eventsRecorder) Expect(err).ToNot(HaveOccurred()) result, err := templateManager.Template([]byte(templateJSON), db) From 327c6562c7bd236759984306e01f2ce0af744ead Mon Sep 17 00:00:00 2001 From: Teodor Pripoae Date: Sun, 28 Feb 2021 13:37:03 +0200 Subject: [PATCH 5/8] feat: add when conditional and tests --- api/v1/template_types.go | 1 - examples/when.yaml | 71 +++++++++++++++++++ k8s/template_manager.go | 43 ++++++++++++ test/e2e.go | 143 +++++++++++++++++++++++++++++++++++++++ test/e2e.sh | 1 + 5 files changed, 258 insertions(+), 1 deletion(-) create mode 100644 examples/when.yaml diff --git a/api/v1/template_types.go b/api/v1/template_types.go index 8ed3fb8..e04b485 100644 --- a/api/v1/template_types.go +++ b/api/v1/template_types.go @@ -55,7 +55,6 @@ type TemplateStatus struct { } type ResourceSelector struct { - ForEach string `json:"forEach,omitempty"` LabelSelector metav1.LabelSelector `json:"labelSelector,omitempty"` NamespaceSelector metav1.LabelSelector `json:"namespaceSelector,omitempty"` AnnotationSelector map[string]string `json:"annotationSelector,omitempty"` diff --git a/examples/when.yaml b/examples/when.yaml new file mode 100644 index 0000000..8e6e0ae --- /dev/null +++ b/examples/when.yaml @@ -0,0 +1,71 @@ +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: apps.example.flanksource.com +spec: + group: example.flanksource.com + names: + kind: App + listKind: AppList + plural: apps + singular: app + scope: Namespaced + subresources: + status: {} + version: v1 + versions: + - name: v1 + served: true + storage: true + validation: + openAPIV3Schema: + description: Schema validation for the App Crds + type: object + properties: + spec: + type: object +--- +apiVersion: templating.flanksource.com/v1 +kind: Template +metadata: + name: app-example +spec: + source: + apiVersion: apps.example.flanksource.com/v1 + kind: App + resources: + - apiVersion: apps/v1 + kind: Deployment + metadata: + name: "{{.metadata.name}}" + namespace: "{{.metadata.namespace}}" + labels: + app: "{{.metadata.name}}" + spec: + replicas: "{{.spec.replicas | default 1}}" + selector: + matchLabels: + app: "{{.metadata.name}}" + template: + metadata: + labels: + app: "{{.metadata.name}}" + spec: + containers: + - name: web + image: "{{.spec.image}}" + ports: + - containerPort: 80 + - when: "{{.spec.exposeService}}" + apiVersion: v1 + kind: Service + metadata: + name: "{{.metadata.name}}" + namespace: "{{.metadata.namespace}}" + spec: + selector: + app: "{{.metadata.name}}" + ports: + - protocol: TCP + port: 80 + targetPort: 80 \ No newline at end of file diff --git a/k8s/template_manager.go b/k8s/template_manager.go index ccb8101..4f8a30e 100644 --- a/k8s/template_manager.go +++ b/k8s/template_manager.go @@ -50,6 +50,10 @@ type ResourcePatch struct { PatchType PatchType } +type Conditionals struct { + When string `json:"when"` +} + type ForEachResource struct { ForEach string `json:"forEach"` } @@ -305,6 +309,14 @@ func (tm *TemplateManager) getObjects(rawItem []byte, target map[string]interfac return nil, errors.Wrap(err, "failed to unmarshal target copy") } + conditional, err := tm.conditional(rawItem, target) + if err != nil { + return nil, errors.Wrap(err, "failed to get conditional") + } + if !conditional { + return []unstructured.Unstructured{}, nil + } + forEach, err := tm.getForEach(rawItem, target) if err != nil { return nil, errors.Wrap(err, "failed to get forEach") @@ -374,6 +386,19 @@ func (tm *TemplateManager) getForEach(rawItem []byte, target map[string]interfac return tm.JSONPath(target, fer.ForEach) } +func (tm *TemplateManager) conditional(rawItem []byte, target map[string]interface{}) (bool, error) { + conditional := &Conditionals{} + if err := json.Unmarshal(rawItem, conditional); err != nil { + return false, errors.Wrap(err, "failed to unmarshal rawItem into Conditionals") + } + + if conditional.When == "" { + return true, nil + } + + return tm.GetBool(target, conditional.When) +} + func (tm *TemplateManager) getNamespaces(ctx context.Context, copyToNamespaces templatev1.CopyToNamespaces) ([]string, error) { namespaceMap := map[string]bool{} namespaces := []string{} @@ -443,6 +468,24 @@ func (tm *TemplateManager) JSONPath(object interface{}, jsonpath string) (*ForEa return nil, errors.Errorf("field %s is not map or array", jsonpath) } +func (tm *TemplateManager) GetBool(object interface{}, jsonpath string) (bool, error) { + jsonpath = strings.TrimPrefix(jsonpath, "{{") + jsonpath = strings.TrimSuffix(jsonpath, "}}") + jsonpath = strings.TrimPrefix(jsonpath, ".") + jsonObject, err := json.Marshal(object) + if err != nil { + return false, errors.Wrap(err, "failed to marshal json") + } + + value := gjson.Get(string(jsonObject), jsonpath) + + if !value.Exists() { + return false, errors.Wrapf(err, "failed to find path %s", jsonpath) + } + + return value.Bool(), nil +} + func labelSelectorToString(l metav1.LabelSelector) (string, error) { labelMap, err := metav1.LabelSelectorAsMap(&l) if err != nil { diff --git a/test/e2e.go b/test/e2e.go index 78f5428..07e858a 100644 --- a/test/e2e.go +++ b/test/e2e.go @@ -49,6 +49,8 @@ var ( "awx-operator": TestAwxOperator, "for-each-array": TestForEachWithArray, "for-each-map": TestForEachWithMap, + "when-true": TestWhenConditional, + "when-false": TestWhenConditionalFalse, } scheme = runtime.NewScheme() restConfig *rest.Config @@ -506,6 +508,121 @@ func TestForEachWithMap(ctx context.Context, test *console.TestResults) error { return nil } +func TestWhenConditional(ctx context.Context, test *console.TestResults) error { + testName := "TestWhenConditional" + ns := fmt.Sprintf("test-when-e2e-%s", utils.RandomString(6)) + if err := client.CreateOrUpdateNamespace(ns, nil, nil); err != nil { + test.Failf(testName, "failed to create namespace %s: %v", ns, err) + return err + } + + defer func() { + if err := k8s.CoreV1().Namespaces().Delete(context.Background(), ns, metav1.DeleteOptions{}); err != nil { + logger.Errorf("failed to delete namespace %s: %v", ns, err) + } + }() + + appName := fmt.Sprintf("app-test-when") + app := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "example.flanksource.com/v1", + "kind": "App", + "metadata": map[string]interface{}{ + "name": appName, + "namespace": ns, + }, + "spec": map[string]interface{}{ + "image": "nginx", + "exposeService": true, + }, + }, + } + + if err := client.Apply(ns, app); err != nil { + test.Failf(testName, "failed to create test app: %v", err) + return err + } + + defer func() { + if err := client.DeleteUnstructured(ns, app); err != nil { + logger.Errorf("failed to delete app %s: %v", appName, err) + } + }() + + if _, err := waitForDeployment(ctx, appName, ns); err != nil { + test.Failf(testName, "Deployment %s not found: %v", appName, err) + } else { + test.Passf(testName, "Deployment %s found", appName) + } + + if _, err := waitForService(ctx, appName, ns); err != nil { + test.Failf(testName, "Service %s not found: %v", appName, err) + } else { + test.Passf(testName, "Service %s found", appName) + } + + return nil +} + +func TestWhenConditionalFalse(ctx context.Context, test *console.TestResults) error { + testName := "TestWhenConditionalFalse" + ns := fmt.Sprintf("test-when-e2e-%s", utils.RandomString(6)) + if err := client.CreateOrUpdateNamespace(ns, nil, nil); err != nil { + test.Failf(testName, "failed to create namespace %s: %v", ns, err) + return err + } + + defer func() { + if err := k8s.CoreV1().Namespaces().Delete(context.Background(), ns, metav1.DeleteOptions{}); err != nil { + logger.Errorf("failed to delete namespace %s: %v", ns, err) + } + }() + + appName := fmt.Sprintf("app-test-when") + app := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "example.flanksource.com/v1", + "kind": "App", + "metadata": map[string]interface{}{ + "name": appName, + "namespace": ns, + }, + "spec": map[string]interface{}{ + "image": "nginx", + "exposeService": false, + }, + }, + } + + if err := client.Apply(ns, app); err != nil { + test.Failf(testName, "failed to create test app: %v", err) + return err + } + + defer func() { + if err := client.DeleteUnstructured(ns, app); err != nil { + logger.Errorf("failed to delete app %s: %v", appName, err) + } + }() + + if _, err := waitForDeployment(ctx, appName, ns); err != nil { + test.Failf(testName, "Deployment %s not found: %v", appName, err) + } else { + test.Passf(testName, "Deployment %s found", appName) + } + + _, err := k8s.CoreV1().Services(ns).Get(ctx, appName, metav1.GetOptions{}) + if err == nil { + test.Failf(testName, "Service %s was created", appName) + } else if !kerrors.IsNotFound(err) { + test.Failf(testName, "Error getting service %s", appName) + } else { + test.Passf(testName, "Service %s was not created", appName) + } + + return nil +} + func waitForDeploymentChanged(ctx context.Context, deployment *appsv1.Deployment, fn deploymentFn) (*appsv1.Deployment, error) { for { d, err := k8s.AppsV1().Deployments(deployment.Namespace).Get(ctx, deployment.Name, metav1.GetOptions{}) @@ -535,6 +652,32 @@ func waitForSecret(ctx context.Context, name, namespace string) (*v1.Secret, err } } +func waitForDeployment(ctx context.Context, name, namespace string) (*appsv1.Deployment, error) { + for { + deployment, err := k8s.AppsV1().Deployments(namespace).Get(ctx, name, metav1.GetOptions{}) + if err != nil && !kerrors.IsNotFound(err) { + return nil, errors.Wrapf(err, "failed to get deployment %s in namespace %s", name, namespace) + } else if kerrors.IsNotFound(err) { + time.Sleep(2 * time.Second) + continue + } + return deployment, nil + } +} + +func waitForService(ctx context.Context, name, namespace string) (*v1.Service, error) { + for { + svc, err := k8s.CoreV1().Services(namespace).Get(ctx, name, metav1.GetOptions{}) + if err != nil && !kerrors.IsNotFound(err) { + return nil, errors.Wrapf(err, "failed to get service %s in namespace %s", name, namespace) + } else if kerrors.IsNotFound(err) { + time.Sleep(2 * time.Second) + continue + } + return svc, nil + } +} + func waitForPostgresqlDB(ctx context.Context, postgresqlDB *unstructured.Unstructured, test *console.TestResults, testName string) error { name := postgresqlDB.GetName() logger.Debugf("Waiting for PostgresqlDB to be created by Awx Operator") diff --git a/test/e2e.sh b/test/e2e.sh index e73c773..a8d24b3 100755 --- a/test/e2e.sh +++ b/test/e2e.sh @@ -46,6 +46,7 @@ $KARINA deploy postgres-operator kubectl apply -f examples/postgres-operator.yml kubectl apply -f examples/namespacerequest.yml kubectl apply -f examples/for-each.yml +kubectl apply -f examples/when.yml kubectl apply -f test/fixtures/awx-operator.yml export IMG=flanksource/template-operator:v1 From f59fd1c59890abe2adbf0018ac5876a62a889006 Mon Sep 17 00:00:00 2001 From: Teodor Pripoae Date: Sun, 28 Feb 2021 13:46:15 +0200 Subject: [PATCH 6/8] build: fix fixture file name --- test/e2e.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e.sh b/test/e2e.sh index a8d24b3..5e8c69d 100755 --- a/test/e2e.sh +++ b/test/e2e.sh @@ -46,7 +46,7 @@ $KARINA deploy postgres-operator kubectl apply -f examples/postgres-operator.yml kubectl apply -f examples/namespacerequest.yml kubectl apply -f examples/for-each.yml -kubectl apply -f examples/when.yml +kubectl apply -f examples/when.yaml kubectl apply -f test/fixtures/awx-operator.yml export IMG=flanksource/template-operator:v1 From 442f65fe6fcc6986b747daa3a2eb1439fc0f91c7 Mon Sep 17 00:00:00 2001 From: Teodor Pripoae Date: Sun, 28 Feb 2021 14:09:30 +0200 Subject: [PATCH 7/8] fix: remove foreach field from crd definition --- config/crd/bases/templating.flanksource.com_templates.yaml | 4 ---- config/deploy/crd.yml | 4 ---- config/deploy/operator.yml | 4 ---- 3 files changed, 12 deletions(-) diff --git a/config/crd/bases/templating.flanksource.com_templates.yaml b/config/crd/bases/templating.flanksource.com_templates.yaml index cf8b191..23d595f 100644 --- a/config/crd/bases/templating.flanksource.com_templates.yaml +++ b/config/crd/bases/templating.flanksource.com_templates.yaml @@ -131,8 +131,6 @@ spec: type: string fieldSelector: type: string - forEach: - type: string kind: type: string labelSelector: @@ -255,8 +253,6 @@ spec: type: string fieldSelector: type: string - forEach: - type: string kind: type: string labelSelector: diff --git a/config/deploy/crd.yml b/config/deploy/crd.yml index f163aff..00732e8 100644 --- a/config/deploy/crd.yml +++ b/config/deploy/crd.yml @@ -97,8 +97,6 @@ spec: type: string fieldSelector: type: string - forEach: - type: string kind: type: string labelSelector: @@ -183,8 +181,6 @@ spec: type: string fieldSelector: type: string - forEach: - type: string kind: type: string labelSelector: diff --git a/config/deploy/operator.yml b/config/deploy/operator.yml index 127f2f2..d907e39 100644 --- a/config/deploy/operator.yml +++ b/config/deploy/operator.yml @@ -104,8 +104,6 @@ spec: type: string fieldSelector: type: string - forEach: - type: string kind: type: string labelSelector: @@ -190,8 +188,6 @@ spec: type: string fieldSelector: type: string - forEach: - type: string kind: type: string labelSelector: From 1d9f838825897786df943ecb1f517963efca8677 Mon Sep 17 00:00:00 2001 From: Teodor Pripoae Date: Sun, 28 Feb 2021 14:30:46 +0200 Subject: [PATCH 8/8] feat: add config/base deploy file without the CRD --- Makefile | 1 + config/base/deploy.yml | 139 +++++++++++++++++++ config/base/kustomization.yaml | 53 +++++++ config/default/kustomization.yaml | 4 +- config/default/manager_auth_proxy_patch.yaml | 26 ---- config/deploy/operator.yml | 27 ++-- config/manager/manager.yaml | 21 +-- 7 files changed, 218 insertions(+), 53 deletions(-) create mode 100644 config/base/deploy.yml create mode 100644 config/base/kustomization.yaml delete mode 100644 config/default/manager_auth_proxy_patch.yaml diff --git a/Makefile b/Makefile index 2e5873f..d260d0b 100644 --- a/Makefile +++ b/Makefile @@ -52,6 +52,7 @@ static: manifests cd config/manager && kustomize edit set image controller=${IMG} kustomize build config/crd > config/deploy/crd.yml kustomize build config/default > config/deploy/operator.yml + kustomize build config/base > config/base/deploy.yml # Run go fmt against code fmt: diff --git a/config/base/deploy.yml b/config/base/deploy.yml new file mode 100644 index 0000000..1cae8d1 --- /dev/null +++ b/config/base/deploy.yml @@ -0,0 +1,139 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: template-operator-manager + namespace: platform-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: template-operator-leader-election-role + namespace: platform-system +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - "" + resources: + - configmaps/status + verbs: + - get + - update + - patch +- apiGroups: + - "" + resources: + - events + verbs: + - create +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + name: template-operator-manager-role +rules: +- apiGroups: + - '*' + resources: + - '*' + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: template-operator-leader-election-rolebinding + namespace: platform-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: template-operator-leader-election-role +subjects: +- kind: ServiceAccount + name: template-operator-manager + namespace: platform-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: template-operator-manager-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: template-operator-manager-role +subjects: +- kind: ServiceAccount + name: template-operator-manager + namespace: platform-system +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + prometheus.io/scrape: "true" + labels: + control-plane: template-operator + name: template-operator-template-operator + namespace: platform-system +spec: + ports: + - name: prometheus + port: 8080 + protocol: TCP + selector: + control-plane: template-operator +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + control-plane: template-operator + name: template-operator-controller-manager + namespace: platform-system +spec: + replicas: 1 + selector: + matchLabels: + control-plane: template-operator + template: + metadata: + labels: + control-plane: template-operator + spec: + containers: + - args: + - --metrics-addr=0.0.0.0:8080 + - --enable-leader-election + - --sync-period=20s + image: flanksource/template-operator:v1 + name: manager + resources: + limits: + cpu: 100m + memory: 130Mi + requests: + cpu: 100m + memory: 120Mi + - args: + - --secure-listen-address=0.0.0.0:8443 + - --upstream=http://127.0.0.1:8080/ + - --logtostderr=true + - --v=10 + image: gcr.io/kubebuilder/kube-rbac-proxy:v0.5.0 + name: kube-rbac-proxy + ports: + - containerPort: 8443 + name: https + serviceAccount: template-operator-manager + terminationGracePeriodSeconds: 10 diff --git a/config/base/kustomization.yaml b/config/base/kustomization.yaml new file mode 100644 index 0000000..b01683f --- /dev/null +++ b/config/base/kustomization.yaml @@ -0,0 +1,53 @@ +# Adds namespace to all resources. +namespace: platform-system + +namePrefix: template-operator- + +bases: +- ../rbac +- ../manager + +# patchesStrategicMerge: + # Protect the /metrics endpoint by putting it behind auth. + # If you want your controller-manager to expose the /metrics + # endpoint w/o any authn/z, please comment the following line. +# - ../default/manager_auth_proxy_patch.yaml + +# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in +# crd/kustomization.yaml +#- manager_webhook_patch.yaml + +# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. +# Uncomment 'CERTMANAGER' sections in crd/kustomization.yaml to enable the CA injection in the admission webhooks. +# 'CERTMANAGER' needs to be enabled to use ca injection +#- webhookcainjection_patch.yaml + +# the following config is for teaching kustomize how to do var substitution +vars: +# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. +#- name: CERTIFICATE_NAMESPACE # namespace of the certificate CR +# objref: +# kind: Certificate +# group: cert-manager.io +# version: v1alpha2 +# name: serving-cert # this name should match the one in certificate.yaml +# fieldref: +# fieldpath: metadata.namespace +#- name: CERTIFICATE_NAME +# objref: +# kind: Certificate +# group: cert-manager.io +# version: v1alpha2 +# name: serving-cert # this name should match the one in certificate.yaml +#- name: SERVICE_NAMESPACE # namespace of the service +# objref: +# kind: Service +# version: v1 +# name: webhook-service +# fieldref: +# fieldpath: metadata.namespace +#- name: SERVICE_NAME +# objref: +# kind: Service +# version: v1 +# name: webhook-service diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index 8edef7b..5c1e34f 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -24,11 +24,11 @@ bases: # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. #- ../prometheus -patchesStrategicMerge: +# patchesStrategicMerge: # Protect the /metrics endpoint by putting it behind auth. # If you want your controller-manager to expose the /metrics # endpoint w/o any authn/z, please comment the following line. -- manager_auth_proxy_patch.yaml +# - manager_auth_proxy_patch.yaml # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml diff --git a/config/default/manager_auth_proxy_patch.yaml b/config/default/manager_auth_proxy_patch.yaml deleted file mode 100644 index 6b7b0a8..0000000 --- a/config/default/manager_auth_proxy_patch.yaml +++ /dev/null @@ -1,26 +0,0 @@ -# This patch inject a sidecar container which is a HTTP proxy for the -# controller manager, it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews. -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system -spec: - template: - spec: - containers: - - name: kube-rbac-proxy - image: gcr.io/kubebuilder/kube-rbac-proxy:v0.5.0 - args: - - "--secure-listen-address=0.0.0.0:8443" - - "--upstream=http://127.0.0.1:8080/" - - "--logtostderr=true" - - "--v=10" - ports: - - containerPort: 8443 - name: https - - name: manager - args: - - "--metrics-addr=0.0.0.0:8080" - - "--enable-leader-election" - - "--sync-period=20s" \ No newline at end of file diff --git a/config/deploy/operator.yml b/config/deploy/operator.yml index d907e39..805579f 100644 --- a/config/deploy/operator.yml +++ b/config/deploy/operator.yml @@ -1,10 +1,3 @@ -apiVersion: v1 -kind: Namespace -metadata: - labels: - control-plane: template-operator - name: platform-system ---- apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: @@ -382,16 +375,6 @@ spec: control-plane: template-operator spec: containers: - - args: - - --secure-listen-address=0.0.0.0:8443 - - --upstream=http://127.0.0.1:8080/ - - --logtostderr=true - - --v=10 - image: gcr.io/kubebuilder/kube-rbac-proxy:v0.5.0 - name: kube-rbac-proxy - ports: - - containerPort: 8443 - name: https - args: - --metrics-addr=0.0.0.0:8080 - --enable-leader-election @@ -405,5 +388,15 @@ spec: requests: cpu: 100m memory: 120Mi + - args: + - --secure-listen-address=0.0.0.0:8443 + - --upstream=http://127.0.0.1:8080/ + - --logtostderr=true + - --v=10 + image: gcr.io/kubebuilder/kube-rbac-proxy:v0.5.0 + name: kube-rbac-proxy + ports: + - containerPort: 8443 + name: https serviceAccount: template-operator-manager terminationGracePeriodSeconds: 10 diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 71eb970..3436277 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -1,10 +1,3 @@ -apiVersion: v1 -kind: Namespace -metadata: - labels: - control-plane: template-operator - name: system ---- apiVersion: apps/v1 kind: Deployment metadata: @@ -25,7 +18,9 @@ spec: containers: - image: controller:latest args: - - --enable-leader-election + - "--metrics-addr=0.0.0.0:8080" + - "--enable-leader-election" + - "--sync-period=20s" name: manager resources: limits: @@ -34,6 +29,16 @@ spec: requests: cpu: 100m memory: 120Mi + - name: kube-rbac-proxy + image: gcr.io/kubebuilder/kube-rbac-proxy:v0.5.0 + args: + - "--secure-listen-address=0.0.0.0:8443" + - "--upstream=http://127.0.0.1:8080/" + - "--logtostderr=true" + - "--v=10" + ports: + - containerPort: 8443 + name: https terminationGracePeriodSeconds: 10 serviceAccount: template-operator-manager ---