Merge pull request #15 from flanksource/e2e-tests

E2e tests for copy-to-namespace and awx operator

config/default/manager_auth_proxy_patch.yaml

@@ -23,3 +23,4 @@ spec:
         args:
         - "--metrics-addr=127.0.0.1:8080"
         - "--enable-leader-election"
+        - "--sync-period=20s"

controllers/template_controller.go

@@ -36,6 +36,7 @@ type TemplateReconciler struct {
 	Client           *kommons.Client
 	Log              logr.Logger
 	Scheme           *runtime.Scheme
+	Cache            *k8s.SchemaCache
 }
 
 // +kubebuilder:rbac:groups="*",resources="*",verbs="*"
@@ -54,14 +55,13 @@ func (r *TemplateReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
 		return reconcile.Result{}, err
 	}
 
-	tm, err := k8s.NewTemplateManager(r.Client, log)
+	tm, err := k8s.NewTemplateManager(r.Client, log, r.Cache)
 	if err != nil {
 		return reconcile.Result{}, err
 	}
 	if err := tm.Run(ctx, template); err != nil {
 		return reconcile.Result{}, err
 	}
-
 	return ctrl.Result{}, nil
 }
 

examples/namespace-request-a.yml

@@ -0,0 +1,7 @@
+apiVersion: acmp.corp/v1
+kind: NamespaceRequest
+metadata:
+  name: a
+spec:
+  team: blue-team
+  memory: 16

examples/namespacerequest.yml

@@ -16,15 +16,6 @@ spec:
     - name: v1
       served: true
       storage: true
-
----
-apiVersion: acmp.corp/v1
-kind: NamespaceRequest
-metadata:
-  name: a
-spec:
-  team: blue-team
-  memory: 16
 ---
 apiVersion: templating.flanksource.com/v1
 kind: Template

examples/postgres-operator.yml

@@ -150,8 +150,7 @@ spec:
           ttl: 30
         podAnnotations: {}
         postgresql:
-          parameters:
-            log_destination: stderr
+          parameters: "{{ .spec.parameters | default (coll.Dict) | data.ToJSON }}"
           version: "12"
         resources:
           limits:

go.mod

@@ -14,8 +14,10 @@ require (
 	github.com/onsi/gomega v1.10.1
 	github.com/pkg/errors v0.9.1
 	github.com/sirupsen/logrus v1.7.0
+	github.com/sykesm/zap-logfmt v0.0.4
 	github.com/tidwall/gjson v1.6.1
-	golang.org/x/tools v0.0.0-20200616195046-dc31b401abb5 // indirect
+	github.com/zalando/postgres-operator v1.6.0
+	go.uber.org/zap v1.15.0
 	gopkg.in/flanksource/yaml.v3 v3.1.1
 	k8s.io/api v0.19.4
 	k8s.io/apiextensions-apiserver v0.19.4

k8s/patches.go

@@ -225,3 +225,16 @@ func (p *PatchApplier) JSONPath(object interface{}, jsonpath string) string {
 	value := gjson.Get(string(jsonObject), jsonpath)
 	return value.String()
 }
+
+var annotationsBlacklist = []string{
+	"metadata.annotations.serving.knative.dev/creator",
+	"metadata.annotations.serving.knative.dev/lastModifier",
+}
+
+func stripAnnotations(obj *unstructured.Unstructured) {
+	annotations := obj.GetAnnotations()
+	for _, a := range annotationsBlacklist {
+		delete(annotations, a)
+	}
+	obj.SetAnnotations(annotations)
+}

k8s/schema_cache.go

@@ -0,0 +1,95 @@
+package k8s
+
+import (
+	"context"
+	"encoding/json"
+	"sync"
+	"time"
+
+	"github.com/go-logr/logr"
+	"github.com/go-openapi/spec"
+	"github.com/pkg/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+)
+
+type SchemaCache struct {
+	clientset *kubernetes.Clientset
+	expire    time.Duration
+	lock      *sync.Mutex
+
+	resources                []*metav1.APIResourceList
+	resourcesExpireTimestamp time.Time
+
+	schema                *spec.Swagger
+	schemaExpireTimestamp time.Time
+	log                   logr.Logger
+}
+
+func NewSchemaCache(clientset *kubernetes.Clientset, expire time.Duration, log logr.Logger) *SchemaCache {
+	sc := &SchemaCache{
+		clientset: clientset,
+		expire:    expire,
+		lock:      &sync.Mutex{},
+		log:       log,
+
+		resources: nil,
+	}
+	return sc
+}
+
+func (sc *SchemaCache) FetchSchema() (*spec.Swagger, error) {
+	sc.lock.Lock()
+	defer sc.lock.Unlock()
+
+	if sc.resources == nil || time.Now().After(sc.schemaExpireTimestamp) {
+		sc.log.V(2).Info("before fetch schema")
+		if err := sc.fetchAndSetSchema(); err != nil {
+			return nil, errors.Wrap(err, "failed to refetch API schema")
+		}
+		sc.log.V(2).Info("after fetch schema")
+	}
+
+	return sc.schema, nil
+}
+
+func (sc *SchemaCache) FetchResources() ([]*metav1.APIResourceList, error) {
+	sc.lock.Lock()
+	defer sc.lock.Unlock()
+
+	if sc.resources == nil || time.Now().After(sc.resourcesExpireTimestamp) {
+		sc.log.V(2).Info("before fetch resources")
+		if err := sc.fetchAndSetResources(); err != nil {
+			return nil, errors.Wrap(err, "failed to refetch API resources")
+		}
+		sc.log.V(2).Info("after fetch resources")
+	}
+	return sc.resources, nil
+}
+
+func (sc *SchemaCache) fetchAndSetSchema() error {
+	bs, err := sc.clientset.RESTClient().Get().AbsPath("openapi", "v2").DoRaw(context.TODO())
+	if err != nil {
+		return errors.Wrap(err, "failed to fetch schema from server")
+	}
+	s := &spec.Swagger{}
+
+	if err := json.Unmarshal(bs, &s); err != nil {
+		return errors.Wrap(err, "failed to unmarshal openapi")
+	}
+
+	sc.schema = s
+	sc.schemaExpireTimestamp = time.Now().Add(sc.expire)
+
+	return nil
+}
+
+func (sc *SchemaCache) fetchAndSetResources() error {
+	serverResources, err := sc.clientset.ServerResources()
+	if err != nil {
+		return errors.Wrap(err, "failed to list server resources")
+	}
+	sc.resources = serverResources
+	sc.resourcesExpireTimestamp = time.Now().Add(sc.expire)
+	return nil
+}

k8s/schema_manager.go

@@ -34,6 +34,26 @@ type TypedField struct {
 
 type MapInterface map[string]interface{}
 
+func NewSchemaManagerWithCache(clientset *kubernetes.Clientset, crdClient extapi.ApiextensionsV1beta1Interface, cache *SchemaCache) (*SchemaManager, error) {
+	s, err := cache.FetchSchema()
+	if err != nil {
+		return nil, err
+	}
+
+	serverResources, err := cache.FetchResources()
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to list server resources")
+	}
+
+	mgr := &SchemaManager{
+		swagger:         s,
+		serverResources: serverResources,
+		clientset:       clientset,
+		crdClient:       crdClient,
+	}
+	return mgr, nil
+}
+
 func NewSchemaManager(clientset *kubernetes.Clientset, crdClient extapi.ApiextensionsV1beta1Interface) (*SchemaManager, error) {
 	bs, err := clientset.RESTClient().Get().AbsPath("openapi", "v2").DoRaw(context.TODO())
 	if err != nil {

k8s/template_manager.go

@@ -44,7 +44,7 @@ type ResourcePatch struct {
 	PatchType  PatchType
 }
 
-func NewTemplateManager(c *kommons.Client, log logr.Logger) (*TemplateManager, error) {
+func NewTemplateManager(c *kommons.Client, log logr.Logger, cache *SchemaCache) (*TemplateManager, error) {
 	clientset, _ := c.GetClientset()
 
 	restConfig, err := c.GetRESTConfig()
@@ -56,7 +56,7 @@ func NewTemplateManager(c *kommons.Client, log logr.Logger) (*TemplateManager, e
 		return nil, errors.Wrap(err, "failed to create crd client")
 	}
 
-	schemaManager, err := NewSchemaManager(clientset, crdClient)
+	schemaManager, err := NewSchemaManagerWithCache(clientset, crdClient, cache)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to create schema manager")
 	}
@@ -153,6 +153,7 @@ func (tm *TemplateManager) Run(ctx context.Context, template *templatev1.Templat
 			}
 			if len(template.Spec.JsonPatches) > 0 || len(template.Spec.Patches) > 0 {
 				target = markApplied(template, target)
+				stripAnnotations(target)
 				if err := tm.Client.ApplyUnstructured(source.GetNamespace(), target); err != nil {
 					return err
 				}
@@ -177,6 +178,8 @@ func (tm *TemplateManager) Run(ctx context.Context, template *templatev1.Templat
 					crossNamespaceOwner(&obj, source)
 				}
 
+				stripAnnotations(&obj)
+
 				if tm.Log.V(2).Enabled() {
 					tm.Log.V(2).Info("Applying", "kind", obj.GetKind(), "namespace", obj.GetNamespace(), "name", obj.GetName(), "obj", obj)
 				} else {
@@ -192,6 +195,7 @@ func (tm *TemplateManager) Run(ctx context.Context, template *templatev1.Templat
 			for _, namespace := range template.Spec.CopyToNamespaces.Namespaces {
 				newResource := source.DeepCopy()
 				newResource.SetNamespace(namespace)
+				stripAnnotations(newResource)
 
 				crossNamespaceOwner(newResource, source)
 

k8s/template_manager_test.go

@@ -3,6 +3,7 @@ package k8s_test
 import (
 	"fmt"
 	"strings"
+	"time"
 
 	"github.com/flanksource/template-operator/k8s"
 	. "github.com/onsi/ginkgo"
@@ -69,7 +70,8 @@ spec:
   synchronous_mode: false
 `
 
-			templateManager, err := k8s.NewTemplateManager(kommonsClient(), testLog)
+			cache := k8s.NewSchemaCache(clientset(), 5*time.Minute, testLog)
+			templateManager, err := k8s.NewTemplateManager(kommonsClient(), testLog, cache)
 			Expect(err).ToNot(HaveOccurred())
 
 			result, err := templateManager.Template([]byte(templateJSON), db)

main.go

@@ -25,6 +25,10 @@ import (
 	"github.com/flanksource/kommons"
 	templatingflanksourcecomv1 "github.com/flanksource/template-operator/api/v1"
 	"github.com/flanksource/template-operator/controllers"
+	"github.com/flanksource/template-operator/k8s"
+	zaplogfmt "github.com/sykesm/zap-logfmt"
+	uzap "go.uber.org/zap"
+	"go.uber.org/zap/zapcore"
 	"k8s.io/apimachinery/pkg/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
@@ -45,11 +49,23 @@ func init() {
 	// +kubebuilder:scaffold:scheme
 }
 
+func setupLogger(opts zap.Options) {
+	configLog := uzap.NewProductionEncoderConfig()
+	configLog.EncodeTime = func(ts time.Time, encoder zapcore.PrimitiveArrayEncoder) {
+		encoder.AppendString(ts.UTC().Format(time.RFC3339Nano))
+	}
+	logfmtEncoder := zaplogfmt.NewEncoder(configLog)
+
+	logger := zap.New(zap.UseFlagOptions(&opts), zap.Encoder(logfmtEncoder))
+	ctrl.SetLogger(logger)
+}
+
 func main() {
 	var metricsAddr string
 	var enableLeaderElection bool
-	var syncPeriod time.Duration
-	flag.DurationVar(&syncPeriod, "sync-period", 5*time.Minute, "The time duration to run a full reconcole")
+	var syncPeriod, expire time.Duration
+	flag.DurationVar(&syncPeriod, "sync-period", 5*time.Minute, "The time duration to run a full reconcile")
+	flag.DurationVar(&expire, "expire", 15*time.Minute, "The time duration to expire API resources cache")
 	flag.StringVar(&metricsAddr, "metrics-addr", ":8080", "The address the metric endpoint binds to.")
 	flag.BoolVar(&enableLeaderElection, "enable-leader-election", false,
 		"Enable leader election for controller manager. "+
@@ -58,7 +74,7 @@ func main() {
 	opts := zap.Options{}
 	opts.BindFlags(flag.CommandLine)
 	flag.Parse()
-	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
+	setupLogger(opts)
 
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
 		Scheme:             scheme,
@@ -74,9 +90,16 @@ func main() {
 	}
 
 	client := kommons.NewClient(mgr.GetConfig(), logger.StandardLogger())
+	clientset, err := client.GetClientset()
+	if err != nil {
+		setupLog.Error(err, "failed to get clientset")
+		os.Exit(1)
+	}
+	schemaCache := k8s.NewSchemaCache(clientset, expire, ctrl.Log.WithName("schema-cache"))
 
 	if err = (&controllers.TemplateReconciler{
 		Client: client,
+		Cache:  schemaCache,
 		Log:    ctrl.Log.WithName("controllers").WithName("Template"),
 		Scheme: mgr.GetScheme(),
 	}).SetupWithManager(mgr); err != nil {

test/config.yaml

@@ -32,6 +32,11 @@ s3:
   skipTLSVerify: true
   e2e:
     minio: true
+minio:
+  version: RELEASE.2020-09-02T18-19-50Z
+  access_key: minio
+  secret_key: minio123
+  replicas: 1
 ca:
   cert: .certs/root-ca.crt
   privateKey: .certs/root-ca.key
@@ -44,6 +49,7 @@ canaryChecker:
   version: v0.11.2
 postgresOperator:
   version: v1.3.4.flanksource.1
+  backupBucket: e2e-postgres-backups
 test:
   exclude:
     - configmap-reloader