Skip to content

Commit

Permalink
device-injector: add support for CDI injection.
Browse files Browse the repository at this point in the history
Add support for injecting annotated CDI devices using the
new native NRI CDI injection API.

Signed-off-by: Krisztian Litkey <[email protected]>
  • Loading branch information
klihub committed Aug 12, 2024
1 parent 646502f commit 485146e
Show file tree
Hide file tree
Showing 5 changed files with 179 additions and 7 deletions.
31 changes: 29 additions & 2 deletions plugins/device-injector/README.md
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
## Device Injector Plugin

This sample plugin can inject devices and mounts into containers using
pod annotations.
This sample plugin can inject Linux device nodes, CDI devices, and mounts into
containers using pod annotations.

### Device Annotations

Expand All @@ -26,6 +26,33 @@ The annotation syntax for device injection is

`file_mode`, `uid` and `gid` can be omitted, the rest are mandatory.

### CDI Device Annotations

Devices are annotated using the `cdi-devices.nri.io` annotation key prefix.
The key `cdi-devices.nri.io/container.$CONTAINER_NAME` annotates CDI devices
to be injected into `$CONTAINER_NAME`. The keys `cdi-devices.nri.io` and
`cdi-devices.nri.io/pod` annotate CDI devices to be injected into all
containers of the pod.

The annotation value syntax is an array of CDI device names to inject. For
instance, the following annotation

```
metadata:
name: bash
annotations:
cdi-devices.nri.io/container.c0: |
- vendor0.com/device=null
cdi-devices.nri.io/container.c1: |
- vendor0.com/device=zero
cdi-devices.nri.io/container.mgmt: |
- vendor0.com/device=all
```

requests the injection of the vendor0.com/device=null, vendor0.com/device=zero,
and vendor0.com/device=all CDI devices, to the c0, c1, and mgmt containers of
the pod.

### Mount Annotations

Mounts are annotated in a similar manner to devices, but using the
Expand Down
68 changes: 63 additions & 5 deletions plugins/device-injector/device-injector.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,8 @@ const (
deviceKey = "devices.nri.io"
// Prefix of the key used for mount annotations.
mountKey = "mounts.nri.io"
// Prefix of the key used for CDI device annotations.
cdiDeviceKey = "cdi-devices.nri.io"
)

var (
Expand Down Expand Up @@ -69,10 +71,11 @@ type plugin struct {
// CreateContainer handles container creation requests.
func (p *plugin) CreateContainer(_ context.Context, pod *api.PodSandbox, container *api.Container) (*api.ContainerAdjustment, []*api.ContainerUpdate, error) {
var (
ctrName string
devices []device
mounts []mount
err error
ctrName string
devices []device
cdiDevices []string
mounts []mount
err error
)

ctrName = containerName(pod, container)
Expand All @@ -90,7 +93,7 @@ func (p *plugin) CreateContainer(_ context.Context, pod *api.PodSandbox, contain
}

if len(devices) == 0 {
log.Infof("%s: no devices annotated...", ctrName)
log.Debugf("%s: no devices annotated...", ctrName)
} else {
if verbose {
dump(ctrName, "annotated devices", devices)
Expand All @@ -104,6 +107,31 @@ func (p *plugin) CreateContainer(_ context.Context, pod *api.PodSandbox, contain
}
}

// inject CDI devices to container
cdiDevices, err = parseCDIDevices(container.Name, pod.Annotations)
if err != nil {
return nil, nil, err
}

if len(cdiDevices) == 0 {
log.Debugf("%s: no CDI devices annotated...", ctrName)
} else {
if verbose {
dump(ctrName, "annotated CDI devices", devices)
}

for _, name := range cdiDevices {
adjust.AddCDIDevice(
&api.CDIDevice{
Name: name,
},
)
if !verbose {
log.Infof("%s: injected CDI device %q...", ctrName, name)
}
}
}

// inject mounts to container
mounts, err = parseMounts(container.Name, pod.Annotations)
if err != nil {
Expand Down Expand Up @@ -162,6 +190,36 @@ func parseDevices(ctr string, annotations map[string]string) ([]device, error) {
return devices, nil
}

func parseCDIDevices(ctr string, annotations map[string]string) ([]string, error) {
var (
key string
annotation []byte
cdiDevices []string
)

// look up effective device annotation and unmarshal devices
for _, key = range []string{
cdiDeviceKey + "/container." + ctr,
cdiDeviceKey + "/pod",
cdiDeviceKey,
} {
if value, ok := annotations[key]; ok {
annotation = []byte(value)
break
}
}

if annotation == nil {
return nil, nil
}

if err := yaml.Unmarshal(annotation, &cdiDevices); err != nil {
return nil, fmt.Errorf("invalid CDI device annotation %q: %w", key, err)
}

return cdiDevices, nil
}

func parseMounts(ctr string, annotations map[string]string) ([]mount, error) {
var (
key string
Expand Down
79 changes: 79 additions & 0 deletions plugins/device-injector/device-injector_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
"testing"

"github.com/stretchr/testify/require"
)

func TestParseCDIDevices(t *testing.T) {
type testCase struct {
name string
annotations map[string]string
result []string
}

for _, tc := range []*testCase{
{
name: "no annotated CDI devices",
annotations: map[string]string{
"foo": "bar",
},
},
{
name: "a single annotated CDI device",
annotations: map[string]string{
"cdi-devices.nri.io/container.ctr0": `
- vendor0.com/device=null
`,
},
result: []string{
"vendor0.com/device=null",
},
},
{
name: "multiple annotated CDI devices",
annotations: map[string]string{
"cdi-devices.nri.io/container.ctr0": `
- vendor0.com/device=null
- vendor0.com/device=zero
`,
},
result: []string{
"vendor0.com/device=null",
"vendor0.com/device=zero",
},
},
{
name: "annotated CDI device for non-matching container name",
annotations: map[string]string{
"cdi-devices.nri.io/container.ctr1": `
- vendor0.com/device=null
- vendor0.com/device=zero
`,
},
},
} {
t.Run(tc.name, func(t *testing.T) {
devices, err := parseCDIDevices("ctr0", tc.annotations)
require.Nil(t, err, "CDI device parsing error")
require.Equal(t, tc.result, devices, "parsed CDI devices")
})
}
}
4 changes: 4 additions & 0 deletions plugins/device-injector/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -5,21 +5,25 @@ go 1.20
require (
github.com/containerd/nri v0.2.0
github.com/sirupsen/logrus v1.9.0
github.com/stretchr/testify v1.8.0
sigs.k8s.io/yaml v1.3.0
)

require (
github.com/containerd/ttrpc v1.2.3 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/opencontainers/runtime-spec v1.0.3-0.20220825212826-86290f6a00fb // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
golang.org/x/net v0.17.0 // indirect
golang.org/x/sys v0.13.0 // indirect
golang.org/x/text v0.13.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20230731190214-cbb8c96f2d6d // indirect
google.golang.org/grpc v1.57.1 // indirect
google.golang.org/protobuf v1.31.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
k8s.io/cri-api v0.25.3 // indirect
)

Expand Down
4 changes: 4 additions & 0 deletions plugins/device-injector/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,11 @@ github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3x
github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
Expand Down Expand Up @@ -75,6 +78,7 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
k8s.io/cri-api v0.25.3 h1:YaiQ05CM4+5L2DAz0KoSa4sv4/VlQvLbf3WHKICPSXs=
k8s.io/cri-api v0.25.3/go.mod h1:riC/P0yOGUf2K1735wW+CXs1aY2ctBgePtnnoFLd0dU=
sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=
Expand Down

0 comments on commit 485146e

Please sign in to comment.