diff --git a/pkg/admission/pathannotation/pathannotation_admission.go b/pkg/admission/pathannotation/pathannotation_admission.go new file mode 100644 index 0000000..1bdef44 --- /dev/null +++ b/pkg/admission/pathannotation/pathannotation_admission.go @@ -0,0 +1,178 @@ +/* +Copyright 2023 The KCP Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package pathannotation + +import ( + "context" + "fmt" + "io" + + kcpinitializers "github.com/kcp-dev/kcp/pkg/admission/initializers" + "github.com/kcp-dev/kcp/sdk/apis/core" + corev1alpha1 "github.com/kcp-dev/kcp/sdk/apis/core/v1alpha1" + kcpinformers "github.com/kcp-dev/kcp/sdk/client/informers/externalversions" + corev1alpha1listers "github.com/kcp-dev/kcp/sdk/client/listers/core/v1alpha1" + "github.com/kcp-dev/logicalcluster/v3" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apiserver/pkg/admission" + genericapirequest "k8s.io/apiserver/pkg/endpoints/request" + + schedulingv1alpha1 "github.com/kcp-dev/contrib-tmc/apis/scheduling/v1alpha1" +) + +const ( + PluginName = "kcp.io/TMCPathAnnotation" +) + +func Register(plugins *admission.Plugins) { + plugins.Register(PluginName, + func(_ io.Reader) (admission.Interface, error) { + return &pathAnnotationPlugin{ + Handler: admission.NewHandler(admission.Create, admission.Update), + }, nil + }) +} + +// Validate checks the value of the logical cluster path annotation to match the +// canonical path in the context. + +// Admit sets the value of the logical cluster path annotation for some resources +// to match the canonical path in the context. + +type pathAnnotationPlugin struct { + *admission.Handler + + logicalClusterLister corev1alpha1listers.LogicalClusterClusterLister + + // getLogicalCluster is a convenience function for easier unit testing, + // it reads a LogicalCluster resource with the given name and from the given cluster. + getLogicalCluster func(clusterName logicalcluster.Name, name string) (*corev1alpha1.LogicalCluster, error) +} + +var pathAnnotationResources = sets.New[string]( + schedulingv1alpha1.Resource("locations").String(), +) + +// Ensure that the required admission interfaces are implemented. +var _ = admission.ValidationInterface(&pathAnnotationPlugin{}) +var _ = admission.MutationInterface(&pathAnnotationPlugin{}) +var _ = admission.InitializationValidator(&pathAnnotationPlugin{}) +var _ = kcpinitializers.WantsKcpInformers(&pathAnnotationPlugin{}) + +func (p *pathAnnotationPlugin) Admit(ctx context.Context, a admission.Attributes, _ admission.ObjectInterfaces) error { + clusterName, err := genericapirequest.ClusterNameFrom(ctx) + if err != nil { + return apierrors.NewInternalError(err) + } + + if a.GetOperation() != admission.Create && a.GetOperation() != admission.Update { + return nil + } + + if a.GetResource().GroupResource() == corev1alpha1.Resource("logicalclusters") { + return nil + } + + u, ok := a.GetObject().(metav1.Object) + if !ok { + return fmt.Errorf("unexpected type %T", a.GetObject()) + } + + annotations := u.GetAnnotations() + value, found := annotations[core.LogicalClusterPathAnnotationKey] + if !found && !pathAnnotationResources.Has(a.GetResource().GroupResource().String()) { + return nil + } + + logicalCluster, err := p.getLogicalCluster(clusterName, corev1alpha1.LogicalClusterName) + if err != nil { + return admission.NewForbidden(a, fmt.Errorf("cannot get this workspace: %w", err)) + } + thisPath := logicalCluster.Annotations[core.LogicalClusterPathAnnotationKey] + if thisPath == "" { + thisPath = logicalcluster.From(logicalCluster).Path().String() + } + + if thisPath != "" && value != thisPath { + if annotations == nil { + annotations = map[string]string{} + } + annotations[core.LogicalClusterPathAnnotationKey] = thisPath + u.SetAnnotations(annotations) + } + + return nil +} + +func (p *pathAnnotationPlugin) Validate(ctx context.Context, a admission.Attributes, _ admission.ObjectInterfaces) error { + clusterName, err := genericapirequest.ClusterNameFrom(ctx) + if err != nil { + return apierrors.NewInternalError(err) + } + + if a.GetOperation() != admission.Create && a.GetOperation() != admission.Update { + return nil + } + + if a.GetResource().GroupResource() == corev1alpha1.Resource("logicalclusters") { + return nil + } + + u, ok := a.GetObject().(metav1.Object) + if !ok { + return fmt.Errorf("unexpected type %T", a.GetObject()) + } + + value, found := u.GetAnnotations()[core.LogicalClusterPathAnnotationKey] + if pathAnnotationResources.Has(a.GetResource().GroupResource().String()) || found { + logicalCluster, err := p.getLogicalCluster(clusterName, corev1alpha1.LogicalClusterName) + if err != nil { + return admission.NewForbidden(a, fmt.Errorf("cannot get this workspace: %w", err)) + } + thisPath := logicalCluster.Annotations[core.LogicalClusterPathAnnotationKey] + if thisPath == "" { + thisPath = logicalcluster.From(logicalCluster).Path().String() + } + + if value != thisPath { + return admission.NewForbidden(a, fmt.Errorf("annotation %q must match canonical path %q", core.LogicalClusterPathAnnotationKey, thisPath)) + } + } + + return nil +} + +func (p *pathAnnotationPlugin) ValidateInitialization() error { + if p.logicalClusterLister == nil { + return fmt.Errorf(PluginName + " plugin needs an LogicalCluster lister") + } + return nil +} + +func (p *pathAnnotationPlugin) SetKcpInformers(local, global kcpinformers.SharedInformerFactory) { + logicalClusterReady := local.Core().V1alpha1().LogicalClusters().Informer().HasSynced + p.SetReadyFunc(func() bool { + return logicalClusterReady() + }) + p.logicalClusterLister = local.Core().V1alpha1().LogicalClusters().Lister() + p.getLogicalCluster = func(clusterName logicalcluster.Name, name string) (*corev1alpha1.LogicalCluster, error) { + return p.logicalClusterLister.Cluster(clusterName).Get(name) + } +} diff --git a/pkg/admission/pathannotation/pathannotation_admission_test.go b/pkg/admission/pathannotation/pathannotation_admission_test.go new file mode 100644 index 0000000..529f4a2 --- /dev/null +++ b/pkg/admission/pathannotation/pathannotation_admission_test.go @@ -0,0 +1,299 @@ +/* +Copyright 2023 The KCP Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package pathannotation + +import ( + "context" + "fmt" + "testing" + + "github.com/kcp-dev/kcp/sdk/apis/core" + corev1alpha1 "github.com/kcp-dev/kcp/sdk/apis/core/v1alpha1" + "github.com/kcp-dev/logicalcluster/v3" + + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apiserver/pkg/admission" + "k8s.io/apiserver/pkg/endpoints/request" + + schedulingv1alpha1 "github.com/kcp-dev/contrib-tmc/apis/scheduling/v1alpha1" +) + +func TestPathAnnotationAdmit(t *testing.T) { + scenarios := []struct { + name string + admissionObject runtime.Object + admissionResource schema.GroupVersionResource + admissionVerb admission.Operation + admissionOptions runtime.Object + admissionContext context.Context //nolint:containedctx + getLogicalCluster func(clusterName logicalcluster.Name, name string) (*corev1alpha1.LogicalCluster, error) + + expectError bool + validateAdmissionObject func(t *testing.T, obj runtime.Object) + }{ + { + name: "error when no cluster in the context", + admissionContext: context.TODO(), + expectError: true, + }, + { + name: "admission is not applied to logicalclusters", + admissionContext: admissionContextFor("foo"), + admissionResource: corev1alpha1.SchemeGroupVersion.WithResource("logicalclusters"), + admissionObject: &corev1alpha1.LogicalCluster{}, + validateAdmissionObject: objectWithoutPathAnnotation, + }, + { + name: "admission is not applied to a resource that undergoes a deletion", + admissionContext: admissionContextFor("foo"), + admissionResource: schedulingv1alpha1.SchemeGroupVersion.WithResource("locations"), + admissionVerb: admission.Delete, + admissionObject: &schedulingv1alpha1.Location{}, + validateAdmissionObject: objectWithoutPathAnnotation, + }, + { + name: "admission is not applied to an unsupported resource", + admissionContext: admissionContextFor("foo"), + admissionResource: schedulingv1alpha1.SchemeGroupVersion.WithResource("placements"), + admissionVerb: admission.Create, + admissionObject: &schedulingv1alpha1.Placement{}, + getLogicalCluster: getCluster("foo"), + validateAdmissionObject: func(t *testing.T, obj runtime.Object) { + t.Helper() + objMeta, err := meta.Accessor(obj) + if err != nil { + t.Fatal(err) + } + if _, has := objMeta.GetAnnotations()[core.LogicalClusterPathAnnotationKey]; has { + t.Fatalf("the %q annotation cannot be automatically set on a Placement resource", core.LogicalClusterPathAnnotationKey) + } + }, + }, + { + name: "admission is applied to an unsupported resource if it has the path annotation present", + admissionContext: admissionContextFor("foo"), + admissionResource: schedulingv1alpha1.SchemeGroupVersion.WithResource("placements"), + admissionVerb: admission.Create, + admissionObject: &schedulingv1alpha1.Placement{ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{core.LogicalClusterPathAnnotationKey: ""}}}, + getLogicalCluster: getCluster("foo"), + validateAdmissionObject: objectHasPathAnnotation("root:foo"), + }, + { + name: "a path is derived from the LogicalCluster object if it doesn't have the path annotation", + admissionContext: admissionContextFor("foo"), + admissionVerb: admission.Create, + admissionResource: schedulingv1alpha1.SchemeGroupVersion.WithResource("locations"), + admissionObject: &schedulingv1alpha1.Location{}, + getLogicalCluster: getCluster("foo"), + validateAdmissionObject: objectHasPathAnnotation("root:foo"), + }, + { + name: "a path is updated when is different from the one applied to the LogicalCluster resource", + admissionVerb: admission.Create, + admissionResource: schedulingv1alpha1.SchemeGroupVersion.WithResource("locations"), + admissionObject: &schedulingv1alpha1.Location{ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{core.LogicalClusterPathAnnotationKey: "bar:foo"}}}, + admissionContext: admissionContextFor("foo"), + getLogicalCluster: getCluster("foo"), + validateAdmissionObject: objectHasPathAnnotation("root:foo"), + }, + { + name: "happy path: a Location is annotated with a path", + admissionVerb: admission.Create, + admissionResource: schedulingv1alpha1.SchemeGroupVersion.WithResource("locations"), + admissionObject: &schedulingv1alpha1.Location{}, + admissionContext: admissionContextFor("foo"), + getLogicalCluster: getCluster("foo"), + validateAdmissionObject: objectHasPathAnnotation("root:foo"), + }, + } + + for _, scenario := range scenarios { + t.Run(scenario.name, func(t *testing.T) { + target := &pathAnnotationPlugin{getLogicalCluster: scenario.getLogicalCluster} + attr := admission.NewAttributesRecord( + scenario.admissionObject, + nil, + schema.GroupVersionKind{}, + "", + "", + scenario.admissionResource, + "", + scenario.admissionVerb, + scenario.admissionOptions, + false, + nil, + ) + + err := target.Admit(scenario.admissionContext, attr, nil) + + if scenario.expectError && err == nil { + t.Errorf("expected to get an error") + } + if !scenario.expectError && err != nil { + t.Errorf("unexpected error: %v", err) + } + if scenario.validateAdmissionObject != nil { + scenario.validateAdmissionObject(t, scenario.admissionObject) + } + }) + } +} + +func TestPathAnnotationValidate(t *testing.T) { + scenarios := []struct { + name string + admissionObject runtime.Object + admissionResource schema.GroupVersionResource + admissionVerb admission.Operation + admissionOptions runtime.Object + admissionContext context.Context //nolint:containedctx + getLogicalCluster func(clusterName logicalcluster.Name, name string) (*corev1alpha1.LogicalCluster, error) + + expectError bool + }{ + { + name: "error when no cluster in the context", + admissionContext: context.TODO(), + expectError: true, + }, + { + name: "admission is not applied to logicalclusters", + admissionContext: admissionContextFor("foo"), + admissionResource: corev1alpha1.SchemeGroupVersion.WithResource("logicalclusters"), + }, + { + name: "admission is not applied to a resource that undergoes a deletion", + admissionContext: admissionContextFor("foo"), + admissionResource: schedulingv1alpha1.SchemeGroupVersion.WithResource("locations"), + admissionVerb: admission.Delete, + }, + { + name: "admission is not applied to an unsupported resource", + admissionContext: admissionContextFor("foo"), + admissionResource: schedulingv1alpha1.SchemeGroupVersion.WithResource("placements"), + admissionVerb: admission.Create, + admissionObject: &schedulingv1alpha1.Placement{}, + getLogicalCluster: getCluster("foo"), + }, + { + name: "a Location with incorrect path annotation is NOT admitted", + admissionVerb: admission.Create, + admissionResource: schedulingv1alpha1.SchemeGroupVersion.WithResource("locations"), + admissionObject: &schedulingv1alpha1.Location{ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{core.LogicalClusterPathAnnotationKey: "universe:milky-way"}}}, + admissionContext: admissionContextFor("foo"), + getLogicalCluster: getCluster("foo"), + expectError: true, + }, + { + name: "a Location without the path annotation is NOT admitted", + admissionVerb: admission.Create, + admissionResource: schedulingv1alpha1.SchemeGroupVersion.WithResource("locations"), + admissionObject: &schedulingv1alpha1.Location{}, + admissionContext: admissionContextFor("foo"), + getLogicalCluster: getCluster("foo"), + expectError: true, + }, + { + name: "happy path: a Location with the path annotation is admitted", + admissionVerb: admission.Create, + admissionResource: schedulingv1alpha1.SchemeGroupVersion.WithResource("locations"), + admissionObject: &schedulingv1alpha1.Location{ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{core.LogicalClusterPathAnnotationKey: "root:foo"}}}, + admissionContext: admissionContextFor("foo"), + getLogicalCluster: getCluster("foo"), + }, + } + + for _, scenario := range scenarios { + t.Run(scenario.name, func(t *testing.T) { + target := &pathAnnotationPlugin{getLogicalCluster: scenario.getLogicalCluster} + attr := admission.NewAttributesRecord( + scenario.admissionObject, + nil, + schema.GroupVersionKind{}, + "", + "", + scenario.admissionResource, + "", + scenario.admissionVerb, + scenario.admissionOptions, + false, + nil, + ) + + err := target.Validate(scenario.admissionContext, attr, nil) + + if scenario.expectError && err == nil { + t.Errorf("expected to get an error") + } + if !scenario.expectError && err != nil { + t.Errorf("unexpected error: %v", err) + } + }) + } +} + +func getCluster(expectedClusterName string) func(clusterName logicalcluster.Name, name string) (*corev1alpha1.LogicalCluster, error) { + return func(clusterName logicalcluster.Name, name string) (*corev1alpha1.LogicalCluster, error) { + if clusterName.String() != expectedClusterName { + return nil, fmt.Errorf("unexpected clusterName = %q, expected = %q", clusterName, expectedClusterName) + } + if name != corev1alpha1.LogicalClusterName { + return nil, fmt.Errorf("unexpected name = %q, expected = %q", clusterName, corev1alpha1.LogicalClusterName) + } + return &corev1alpha1.LogicalCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: corev1alpha1.LogicalClusterName, + Annotations: map[string]string{ + core.LogicalClusterPathAnnotationKey: "root:foo", + }, + }, + }, nil + } +} + +func objectHasPathAnnotation(expectedPathAnnotation string) func(t *testing.T, obj runtime.Object) { + return func(t *testing.T, obj runtime.Object) { + t.Helper() + objMeta, err := meta.Accessor(obj) + if err != nil { + t.Fatal(err) + } + pathAnnotation := objMeta.GetAnnotations()[core.LogicalClusterPathAnnotationKey] + if pathAnnotation == "" || pathAnnotation != expectedPathAnnotation { + t.Fatalf("unexpected value = %q, in the %q annotation, expected = %q", pathAnnotation, core.LogicalClusterPathAnnotationKey, expectedPathAnnotation) + } + } +} + +func objectWithoutPathAnnotation(t *testing.T, obj runtime.Object) { + t.Helper() + objMeta, err := meta.Accessor(obj) + if err != nil { + t.Fatal(err) + } + _, has := objMeta.GetAnnotations()[core.LogicalClusterPathAnnotationKey] + if has { + t.Fatalf("object = %v should not have %q annotation set", objMeta.GetName(), core.LogicalClusterPathAnnotationKey) + } +} + +func admissionContextFor(clusterName string) context.Context { + return request.WithCluster(context.Background(), request.Cluster{Name: logicalcluster.Name(clusterName)}) +} diff --git a/pkg/admission/plugins.go b/pkg/admission/plugins.go new file mode 100644 index 0000000..41a0e2d --- /dev/null +++ b/pkg/admission/plugins.go @@ -0,0 +1,50 @@ +/* +Copyright 2022 The KCP Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package admission + +import ( + "k8s.io/apiserver/pkg/admission" + mutatingwebhook "k8s.io/apiserver/pkg/admission/plugin/webhook/mutating" + + "github.com/kcp-dev/contrib-tmc/pkg/admission/pathannotation" +) + +// tmcOrderedPlugins is the list of TMC plugins in order. +var tmcOrderedPlugins = []string{ + pathannotation.PluginName, +} + +func beforeWebhooks(currents []string, plugins []string) []string { + ret := make([]string, 0, len(currents)+len(plugins)) + for _, plugin := range currents { + if plugin == mutatingwebhook.PluginName { + ret = append(ret, plugins...) + } + ret = append(ret, plugin) + } + return ret +} + +func AddTMCOrderedPlugins(currents []string) []string { + return beforeWebhooks(currents, tmcOrderedPlugins) +} + +// RegisterAllTMCAdmissionPlugins registers all admission plugins. +// The order of registration is irrelevant, see AllOrderedPlugins for execution order. +func RegisterAllTMCAdmissionPlugins(plugins *admission.Plugins) { + pathannotation.Register(plugins) +} diff --git a/pkg/reconciler/scheduling/placement/placement_reconcile_scheduling.go b/pkg/reconciler/scheduling/placement/placement_reconcile_scheduling.go index acd0e0b..aaaaff7 100644 --- a/pkg/reconciler/scheduling/placement/placement_reconcile_scheduling.go +++ b/pkg/reconciler/scheduling/placement/placement_reconcile_scheduling.go @@ -40,14 +40,11 @@ type placementReconciler struct { func (r *placementReconciler) reconcile(ctx context.Context, placement *schedulingv1alpha1.Placement) (reconcileStatus, *schedulingv1alpha1.Placement, error) { // get location workspace at first var locationWorkspace logicalcluster.Path - // https://github.com/kcp-dev/contrib-tmc/issues/4 - // TODO(MJ): currently this disables the cross workspace placements. This is due to - // fact indexers are build ontop of logicalcluster paths and we use here the 'readable' paths - // if len(placement.Spec.LocationWorkspace) > 0 { - // locationWorkspace = logicalcluster.NewPath(placement.Spec.LocationWorkspace) - // } else { - locationWorkspace = logicalcluster.From(placement).Path() - // } + if len(placement.Spec.LocationWorkspace) > 0 { + locationWorkspace = logicalcluster.NewPath(placement.Spec.LocationWorkspace) + } else { + locationWorkspace = logicalcluster.From(placement).Path() + } locationWorkspace, validLocationNames, err := r.validLocationNames(placement, locationWorkspace) if err != nil { diff --git a/tmc/server/options/options.go b/tmc/server/options/options.go index 1e69d54..22bb26c 100644 --- a/tmc/server/options/options.go +++ b/tmc/server/options/options.go @@ -21,6 +21,7 @@ import ( cliflag "k8s.io/component-base/cli/flag" + tmcadmission "github.com/kcp-dev/contrib-tmc/pkg/admission" tmcvirtualoptions "github.com/kcp-dev/contrib-tmc/tmc/virtual/options" ) @@ -56,6 +57,10 @@ func NewOptions(rootDir string) *Options { Extra: ExtraOptions{}, } + // add TMC admission plugins + tmcadmission.RegisterAllTMCAdmissionPlugins(o.Core.GenericControlPlane.Admission.Plugins) + orderedPlugins := tmcadmission.AddTMCOrderedPlugins(o.Core.GenericControlPlane.Admission.RecommendedPluginOrder) + o.Core.GenericControlPlane.Admission.RecommendedPluginOrder = orderedPlugins return o }