Make Provisioning Work

Simply a case of taking our scheduled application order and constructing
a provisioner out of it.

charts/application/crds/application.unikorn-cloud.org_applicationsets.yaml

@@ -29,8 +29,8 @@ spec:
         description: |-
           ApplicationSet defines a set of applications.
           It works like a normal package manager, installing a package will automatically
-          install any dependencies and recommended packages.  Removeing a package will also
-          remove any dependencies and recoomended packages unless they are kept alive by
+          install any dependencies and recommended packages.  Removing a package will also
+          remove any dependencies and recommended packages unless they are kept alive by
           another package in the set.
         properties:
           apiVersion:
@@ -126,4 +126,5 @@ spec:
         type: object
     served: true
     storage: true
-    subresources: {}
+    subresources:
+      status: {}

charts/application/templates/application-controller/clusterrole.yaml

@@ -7,19 +7,30 @@ metadata:
 rules:
 # Manage applicationsets (my job).
 - apiGroups:
-  - unikorn-cloud.org
+  - application.unikorn-cloud.org
   resources:
   - applicationsets
   verbs:
   - list
   - watch
+  - update
 # Manage applicationsets (my job).
 - apiGroups:
-  - unikorn-cloud.org
+  - application.unikorn-cloud.org
   resources:
   - applicationsets/status
   verbs:
   - update
+# Get Kubernetes clusters so we can provision applications on it.
+- apiGroups:
+  - unikorn-cloud.org
+  resources:
+  - helmapplications
+  - kubernetesclusters
+  - clustermanagers
+  verbs:
+  - list
+  - watch
 # ArgoCD integration.
 - apiGroups:
   - argoproj.io
@@ -40,6 +51,3 @@ rules:
   verbs:
   - list
   - watch
-  - create
-  - patch
-  - delete

go.mod

@@ -11,7 +11,7 @@ require (
 	github.com/spf13/pflag v1.0.5
 	github.com/spjmurray/go-sat v0.3.1
 	github.com/stretchr/testify v1.9.0
-	github.com/unikorn-cloud/core v0.1.80
+	github.com/unikorn-cloud/core v0.1.81
 	github.com/unikorn-cloud/identity v0.2.44
 	github.com/unikorn-cloud/kubernetes v0.2.47
 	go.opentelemetry.io/otel/sdk v1.31.0
@@ -86,7 +86,6 @@ require (
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53 // indirect
 	google.golang.org/grpc v1.67.1 // indirect
 	google.golang.org/protobuf v1.35.1 // indirect
-	gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect

go.sum

@@ -135,8 +135,8 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE=
 github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
-github.com/unikorn-cloud/core v0.1.80 h1:Xkk8QV+8a4kOfJge9teCzwZjGtAz7pE+cJvCXULtEVw=
-github.com/unikorn-cloud/core v0.1.80/go.mod h1:wEKzCwAnIyTbo27l++Wl+gK95TAxMsFS3y3jbFB03aw=
+github.com/unikorn-cloud/core v0.1.81 h1:7x0yyspkuDtjTTXl3+P8uORcVffeUZhvNzdoEDktSjQ=
+github.com/unikorn-cloud/core v0.1.81/go.mod h1:wEKzCwAnIyTbo27l++Wl+gK95TAxMsFS3y3jbFB03aw=
 github.com/unikorn-cloud/identity v0.2.44 h1:tXV/qsJ77Dkx8ba8gnBFXHWUgBNsJ2oo/5TjnyhkH7U=
 github.com/unikorn-cloud/identity v0.2.44/go.mod h1:JMbS6iTYzt0OVt5AkqZys3WVnpLabGvUl8kGWcxzFZI=
 github.com/unikorn-cloud/kubernetes v0.2.47 h1:dc2V0RWabhZ6hUwrRkdWVMI48eNq5oZbgAGS3RC1r+I=

pkg/apis/unikorn/v1alpha1/helpers.go

@@ -73,7 +73,7 @@ func (c *ApplicationSet) ResourceLabels() (labels.Set, error) {
 	}
 
 	labels := labels.Set{
-		constants.KindLabel:              "unikorn-cloud.org/applicationset",
+		constants.KindLabel:              "applicationset",
 		constants.OrganizationLabel:      organization,
 		constants.ProjectLabel:           project,
 		constants.KubernetesClusterLabel: cluster,

pkg/apis/unikorn/v1alpha1/types.go

@@ -32,12 +32,13 @@ type ApplicationSetList struct {
 
 // ApplicationSet defines a set of applications.
 // It works like a normal package manager, installing a package will automatically
-// install any dependencies and recommended packages.  Removeing a package will also
-// remove any dependencies and recoomended packages unless they are kept alive by
+// install any dependencies and recommended packages.  Removing a package will also
+// remove any dependencies and recommended packages unless they are kept alive by
 // another package in the set.
 // +genclient
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
 // +kubebuilder:resource:scope=Namespaced,categories=unikorn
+// +kubebuilder:subresource:status
 // +kubebuilder:printcolumn:name="display name",type="string",JSONPath=".metadata.labels['unikorn-cloud\\.org/name']"
 // +kubebuilder:printcolumn:name="age",type="date",JSONPath=".metadata.creationTimestamp"
 type ApplicationSet struct {

pkg/managers/application/manager.go

@@ -23,6 +23,7 @@ import (
 	coreclient "github.com/unikorn-cloud/core/pkg/client"
 	coremanager "github.com/unikorn-cloud/core/pkg/manager"
 	"github.com/unikorn-cloud/core/pkg/manager/options"
+	kubernetesunikornv1 "github.com/unikorn-cloud/kubernetes/pkg/apis/unikorn/v1alpha1"
 
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
@@ -75,5 +76,6 @@ func (*Factory) Upgrade(_ client.Client) error {
 func (*Factory) Schemes() []coreclient.SchemeAdder {
 	return []coreclient.SchemeAdder{
 		unikornv1.AddToScheme,
+		kubernetesunikornv1.AddToScheme,
 	}
 }

pkg/provisioners/managers/application/provisioner.go

@@ -19,20 +19,23 @@ package application
 import (
 	"context"
 	"errors"
+	"fmt"
 
 	"github.com/spf13/pflag"
 	sat "github.com/spjmurray/go-sat"
 
 	unikornv1 "github.com/unikorn-cloud/application/pkg/apis/unikorn/v1alpha1"
-	"github.com/unikorn-cloud/application/pkg/constants"
 	"github.com/unikorn-cloud/application/pkg/solver"
 	unikornv1core "github.com/unikorn-cloud/core/pkg/apis/unikorn/v1alpha1"
 	coreclient "github.com/unikorn-cloud/core/pkg/client"
 	"github.com/unikorn-cloud/core/pkg/manager"
 	"github.com/unikorn-cloud/core/pkg/provisioners"
-	identityclient "github.com/unikorn-cloud/identity/pkg/client"
-	kubernetesclient "github.com/unikorn-cloud/kubernetes/pkg/client"
-	kubernetesapi "github.com/unikorn-cloud/kubernetes/pkg/openapi"
+	"github.com/unikorn-cloud/core/pkg/provisioners/application"
+	"github.com/unikorn-cloud/core/pkg/provisioners/remotecluster"
+	"github.com/unikorn-cloud/core/pkg/provisioners/serial"
+	unikornv1kubernetes "github.com/unikorn-cloud/kubernetes/pkg/apis/unikorn/v1alpha1"
+	"github.com/unikorn-cloud/kubernetes/pkg/provisioners/helmapplications/clusteropenstack"
+	"github.com/unikorn-cloud/kubernetes/pkg/provisioners/helmapplications/vcluster"
 
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
@@ -45,26 +48,15 @@ var (
 	ErrGraph = errors.New("graph error")
 
 	ErrConstraint = errors.New("constraint error")
+
+	ErrClusterManager = errors.New("cluster manager lookup failed")
 )
 
 // Options allows access to CLI options in the provisioner.
 type Options struct {
-	// kubernetesOptions allows the kubernetes host and CA to be set.
-	kubernetesOptions *kubernetesclient.Options
-	// clientOptions give access to client certificate information as
-	// we need to talk to kubernetes to get a token, and then to kubernetes
-	// to ensure cloud identities and networks are provisioned, as well
-	// as deptovisioning them.
-	clientOptions coreclient.HTTPClientOptions
 }
 
 func (o *Options) AddFlags(f *pflag.FlagSet) {
-	if o.kubernetesOptions == nil {
-		o.kubernetesOptions = kubernetesclient.NewOptions()
-	}
-
-	o.kubernetesOptions.AddFlags(f)
-	o.clientOptions.AddFlags(f)
 }
 
 // Provisioner encapsulates control plane provisioning.
@@ -94,35 +86,8 @@ func (p *Provisioner) Object() unikornv1core.ManagableResourceInterface {
 	return &p.applicationset
 }
 
-// getKubernetesClient returns an authenticated context with a client credentials access token
-// and a client.  The context must be used by subseqent API calls in order to extract
-// the access token.
-//
-//nolint:unparam
-func (p *Provisioner) getKubernetesClient(ctx context.Context, traceName string) (context.Context, kubernetesapi.ClientWithResponsesInterface, error) {
-	cli, err := coreclient.ProvisionerClientFromContext(ctx)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	tokenIssuer := identityclient.NewTokenIssuer(cli, p.options.kubernetesOptions, &p.options.clientOptions, constants.Application, constants.Version)
-
-	token, err := tokenIssuer.Issue(ctx, traceName)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	getter := kubernetesclient.New(cli, p.options.kubernetesOptions, &p.options.clientOptions)
-
-	client, err := getter.Client(ctx, token)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	return ctx, client, nil
-}
-
-func getApplications(ctx context.Context, cli client.Client, namespace string) (solver.ApplicationIndex, error) {
+// getApplicationIndex loads up all applications we have defined.
+func getApplicationIndex(ctx context.Context, cli client.Client, namespace string) (solver.ApplicationIndex, error) {
 	var applications unikornv1core.HelmApplicationList
 
 	options := &client.ListOptions{
@@ -142,6 +107,9 @@ func getApplications(ctx context.Context, cli client.Client, namespace string) (
 	return solver.NewApplicationIndex(l...)
 }
 
+// schedulerVistor is used to walk the graph of application versions from the SAT solution.
+// It builds up an order in which to install applications so that all dependencies are
+// satisfied before an application is installed.
 type schedulerVistor struct {
 	applications solver.ApplicationIndex
 	appVersions  map[string]solver.AppVersion
@@ -187,12 +155,8 @@ func (v *schedulerVistor) Visit(av solver.AppVersion, enqueue func(solver.AppVer
 	return nil
 }
 
-func Schedule(ctx context.Context, client client.Client, namespace string, solution sat.Set[solver.AppVersion]) ([]solver.AppVersion, error) {
-	applications, err := getApplications(ctx, client, namespace)
-	if err != nil {
-		return nil, err
-	}
-
+// Schedule takes a SAT solution and creates an installation order for the applications.
+func Schedule(ctx context.Context, applications solver.ApplicationIndex, solution sat.Set[solver.AppVersion]) ([]solver.AppVersion, error) {
 	// Okay, we need to build up a reverse map of dependencies and also
 	// record the packages with no dependencies as those will be installed
 	// first.
@@ -244,28 +208,105 @@ func Schedule(ctx context.Context, client client.Client, namespace string, solut
 	return visitor.order, nil
 }
 
+// getClusterManager gets the control plane object that owns this cluster.
+func (p *Provisioner) getClusterManager(ctx context.Context, cluster *unikornv1kubernetes.KubernetesCluster) (*unikornv1kubernetes.ClusterManager, error) {
+	cli, err := coreclient.ProvisionerClientFromContext(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	key := client.ObjectKey{
+		Namespace: cluster.Namespace,
+		Name:      cluster.Spec.ClusterManagerID,
+	}
+
+	var clusterManager unikornv1kubernetes.ClusterManager
+
+	if err := cli.Get(ctx, key, &clusterManager); err != nil {
+		return nil, fmt.Errorf("%w: %s", ErrClusterManager, err.Error())
+	}
+
+	return &clusterManager, nil
+}
+
+// getProvisioner creates the provisioner.
+func (p *Provisioner) getProvisioner(ctx context.Context) (provisioners.Provisioner, error) {
+	cli, err := coreclient.ProvisionerClientFromContext(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	namespace, err := coreclient.NamespaceFromContext(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	index, err := getApplicationIndex(ctx, cli, namespace)
+	if err != nil {
+		return nil, err
+	}
+
+	solution, err := solver.SolveApplicationSet(ctx, index, &p.applicationset)
+	if err != nil {
+		return nil, err
+	}
+
+	order, err := Schedule(ctx, index, solution)
+	if err != nil {
+		return nil, err
+	}
+
+	apps := make([]provisioners.Provisioner, len(order))
+
+	for i, av := range order {
+		applicationGetter := func(context.Context) (*unikornv1core.HelmApplication, *unikornv1core.SemanticVersion, error) {
+			return index[av.Name], &av.Version, nil
+		}
+
+		apps[i] = application.New(applicationGetter).InNamespace("managed-applications")
+	}
+
+	cluster := &unikornv1kubernetes.KubernetesCluster{}
+
+	if err := cli.Get(ctx, client.ObjectKey{Namespace: p.applicationset.Namespace, Name: p.applicationset.Name}, cluster); err != nil {
+		return nil, err
+	}
+
+	clusterManager, err := p.getClusterManager(ctx, cluster)
+	if err != nil {
+		return nil, err
+	}
+
+	remoteClusterManager := remotecluster.New(vcluster.NewRemoteCluster(cluster.Namespace, clusterManager.Name, clusterManager), false)
+
+	remoteCluster := remotecluster.New(clusteropenstack.NewRemoteCluster(cluster), false)
+
+	provisioner := remoteClusterManager.ProvisionOn(
+		remoteCluster.ProvisionOn(
+			serial.New("application-set", apps...),
+			remotecluster.BackgroundDeletion,
+		),
+	)
+
+	return provisioner, nil
+}
+
 // Provision implements the Provision interface.
 func (p *Provisioner) Provision(ctx context.Context) error {
-	clientContext, client, err := p.getKubernetesClient(ctx, "provision")
+	provisioner, err := p.getProvisioner(ctx)
 	if err != nil {
 		return err
 	}
 
-	// TODO: do something!
-	_, _ = clientContext, client
-
-	return nil
+	return provisioner.Provision(ctx)
 }
 
 // Deprovision implements the Provision interface.
 func (p *Provisioner) Deprovision(ctx context.Context) error {
-	clientContext, client, err := p.getKubernetesClient(ctx, "deprovision")
+	provisioner, err := p.getProvisioner(ctx)
 	if err != nil {
 		return err
 	}
 
-	// TODO: do something!
-	_, _ = clientContext, client
-
-	return nil
+	return provisioner.Deprovision(ctx)
 }