From 460b41a693ece056faa265233ae37b8ee853df2e Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Tue, 2 Jul 2024 12:07:15 +0300 Subject: [PATCH] device-injector: add support for CDI injection. Add support for injecting annotated CDI devices using the new native NRI CDI injection API. Signed-off-by: Krisztian Litkey --- plugins/device-injector/README.md | 31 +++++++- plugins/device-injector/device-injector.go | 66 +++++++++++++++-- .../device-injector/device-injector_test.go | 70 +++++++++++++++++++ plugins/device-injector/go.mod | 4 ++ plugins/device-injector/go.sum | 4 ++ 5 files changed, 169 insertions(+), 6 deletions(-) create mode 100644 plugins/device-injector/device-injector_test.go diff --git a/plugins/device-injector/README.md b/plugins/device-injector/README.md index 5c594d66..4fd2e4c0 100644 --- a/plugins/device-injector/README.md +++ b/plugins/device-injector/README.md @@ -1,7 +1,7 @@ ## Device Injector Plugin -This sample plugin can inject devices and mounts into containers using -pod annotations. +This sample plugin can inject Linux device nodes, CDI devices, and mounts into +containers using pod annotations. ### Device Annotations @@ -26,6 +26,33 @@ The annotation syntax for device injection is `file_mode`, `uid` and `gid` can be omitted, the rest are mandatory. +### CDI Device Annotations + +Devices are annotated using the `cdi-devices.nri.io` annotation key prefix. +The key `cdi-devices.nri.io/container.$CONTAINER_NAME` annotates CDI devices +to be injected into `$CONTAINER_NAME`. The keys `cdi-devices.nri.io` and +`cdi-devices.nri.io/pod` annotate CDI devices to be injected into all +containers of the pod. + +The annotation value syntax is an array of CDI device names to inject. For +instance, the following annotation + +``` +metadata: + name: bash + annotations: + cdi-devices.nri.io/container.c0: | + - vendor0.com/device=null + cdi-devices.nri.io/container.c1: | + - vendor0.com/device=zero + cdi-devices.nri.io/container.mgmt: | + - vendor0.com/device=all +``` + +requests the injection of the vendor0.com/device=null, vendor0.com/device=zero, +and vendor0.com/device=all CDI devices, to the c0, c1, and mgmt containers of +the pod. + ### Mount Annotations Mounts are annotated in a similar manner to devices, but using the diff --git a/plugins/device-injector/device-injector.go b/plugins/device-injector/device-injector.go index 3c19cd2b..69539031 100644 --- a/plugins/device-injector/device-injector.go +++ b/plugins/device-injector/device-injector.go @@ -35,6 +35,8 @@ const ( deviceKey = "devices.nri.io" // Prefix of the key used for mount annotations. mountKey = "mounts.nri.io" + // Prefix of the key used for CDI device annotations. + cdiDeviceKey = "cdi-devices.nri.io" ) var ( @@ -69,10 +71,11 @@ type plugin struct { // CreateContainer handles container creation requests. func (p *plugin) CreateContainer(_ context.Context, pod *api.PodSandbox, container *api.Container) (*api.ContainerAdjustment, []*api.ContainerUpdate, error) { var ( - ctrName string - devices []device - mounts []mount - err error + ctrName string + devices []device + cdiDevices []string + mounts []mount + err error ) ctrName = containerName(pod, container) @@ -104,6 +107,31 @@ func (p *plugin) CreateContainer(_ context.Context, pod *api.PodSandbox, contain } } + // inject CDI devices to container + cdiDevices, err = parseCDIDevices(container.Name, pod.Annotations) + if err != nil { + return nil, nil, err + } + + if len(cdiDevices) == 0 { + log.Infof("%s: no CDI devices annotated...", ctrName) + } else { + if verbose { + dump(ctrName, "annotated CDI devices", devices) + } + + for _, name := range cdiDevices { + adjust.AddCDIDevice( + &api.CDIDevice{ + Name: name, + }, + ) + if !verbose { + log.Infof("%s: injected CDI device %q...", ctrName, name) + } + } + } + // inject mounts to container mounts, err = parseMounts(container.Name, pod.Annotations) if err != nil { @@ -162,6 +190,36 @@ func parseDevices(ctr string, annotations map[string]string) ([]device, error) { return devices, nil } +func parseCDIDevices(ctr string, annotations map[string]string) ([]string, error) { + var ( + key string + annotation []byte + cdiDevices []string + ) + + // look up effective device annotation and unmarshal devices + for _, key = range []string{ + cdiDeviceKey + "/container." + ctr, + cdiDeviceKey + "/pod", + cdiDeviceKey, + } { + if value, ok := annotations[key]; ok { + annotation = []byte(value) + break + } + } + + if annotation == nil { + return nil, nil + } + + if err := yaml.Unmarshal(annotation, &cdiDevices); err != nil { + return nil, fmt.Errorf("invalid CDI device annotation %q: %w", key, err) + } + + return cdiDevices, nil +} + func parseMounts(ctr string, annotations map[string]string) ([]mount, error) { var ( key string diff --git a/plugins/device-injector/device-injector_test.go b/plugins/device-injector/device-injector_test.go new file mode 100644 index 00000000..a0d54c72 --- /dev/null +++ b/plugins/device-injector/device-injector_test.go @@ -0,0 +1,70 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package main + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestParseCDIDevices(t *testing.T) { + type testCase struct { + name string + annotations map[string]string + result []string + } + + for _, tc := range []*testCase{ + { + name: "no annotated CDI devices", + annotations: map[string]string{ + "foo": "bar", + }, + }, + { + name: "a single annotated CDI device", + annotations: map[string]string{ + "cdi-devices.nri.io/container.ctr0": ` +- vendor0.com/device=null +`, + }, + result: []string{ + "vendor0.com/device=null", + }, + }, + { + name: "multiple annotated CDI devices", + annotations: map[string]string{ + "cdi-devices.nri.io/container.ctr0": ` +- vendor0.com/device=null +- vendor0.com/device=zero +`, + }, + result: []string{ + "vendor0.com/device=null", + "vendor0.com/device=zero", + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + devices, err := parseCDIDevices("ctr0", tc.annotations) + require.Nil(t, err, "CDI device parsing error") + require.Equal(t, tc.result, devices, "parsed CDI devices") + }) + } +} diff --git a/plugins/device-injector/go.mod b/plugins/device-injector/go.mod index e651c9d4..4952b4aa 100644 --- a/plugins/device-injector/go.mod +++ b/plugins/device-injector/go.mod @@ -5,14 +5,17 @@ go 1.20 require ( github.com/containerd/nri v0.2.0 github.com/sirupsen/logrus v1.9.0 + github.com/stretchr/testify v1.8.0 sigs.k8s.io/yaml v1.3.0 ) require ( github.com/containerd/ttrpc v1.2.3 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/opencontainers/runtime-spec v1.0.3-0.20220825212826-86290f6a00fb // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect golang.org/x/net v0.17.0 // indirect golang.org/x/sys v0.13.0 // indirect golang.org/x/text v0.13.0 // indirect @@ -20,6 +23,7 @@ require ( google.golang.org/grpc v1.57.1 // indirect google.golang.org/protobuf v1.31.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/cri-api v0.25.3 // indirect ) diff --git a/plugins/device-injector/go.sum b/plugins/device-injector/go.sum index 238d00b7..01491189 100644 --- a/plugins/device-injector/go.sum +++ b/plugins/device-injector/go.sum @@ -25,8 +25,11 @@ github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3x github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -75,6 +78,7 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= k8s.io/cri-api v0.25.3 h1:YaiQ05CM4+5L2DAz0KoSa4sv4/VlQvLbf3WHKICPSXs= k8s.io/cri-api v0.25.3/go.mod h1:riC/P0yOGUf2K1735wW+CXs1aY2ctBgePtnnoFLd0dU= sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=