diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9b540fc0a..763701eaa 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -34,5 +34,3 @@ jobs: - run: make ubuntu20.04 - run: TAG_VERSION="${BRANCH_NAME}" make push-tag - run: make push-latest - - run: make vgpu - - run: TAG_VERSION="${BRANCH_NAME}" make push-vgpu-tag diff --git a/Makefile b/Makefile index 1ec9bbeea..a7571f3f1 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -20,8 +20,7 @@ DOCKER ?= docker REGISTRY ?= volcanosh -VERSION ?= latest -TAG_VERSION ?= 1.0.0 +VERSION ?= 1.0.0 ##### Public rules ##### @@ -39,24 +38,11 @@ push-latest: $(DOCKER) tag "$(REGISTRY)/volcano-device-plugin:$(VERSION)-ubuntu20.04" "$(REGISTRY)/volcano-device-plugin:latest" $(DOCKER) push "$(REGISTRY)/volcano-device-plugin:latest" -push-tag: - $(DOCKER) tag "$(REGISTRY)/volcano-device-plugin:$(VERSION)-ubuntu20.04" "$(REGISTRY)/volcano-device-plugin:$(TAG_VERSION)" - $(DOCKER) push "$(REGISTRY)/volcano-device-plugin:$(TAG_VERSION)" - -push-vgpu-tag: - $(DOCKER) tag "$(REGISTRY)/volcano-vgpu-device-plugin:$(VERSION)-ubuntu20.04" "$(REGISTRY)/volcano-vgpu-device-plugin:$(TAG_VERSION)" - $(DOCKER) push "$(REGISTRY)/volcano-vgpu-device-plugin:$(TAG_VERSION)" - ubuntu20.04: - $(DOCKER) build --pull \ + $(DOCKER) build --network=host --pull \ --tag $(REGISTRY)/volcano-device-plugin:$(VERSION)-ubuntu20.04 \ --file docker/amd64/Dockerfile.ubuntu20.04 . -vgpu: - $(DOCKER) build --pull \ - --tag $(REGISTRY)/volcano-vgpu-device-plugin:$(VERSION)-ubuntu20.04 \ - --file docker/amd64/Dockerfile.vgpu-ubuntu20.04 . - centos7: $(DOCKER) build --pull \ --tag $(REGISTRY)/volcano-device-plugin:$(VERSION)-centos7 \ diff --git a/README.md b/README.md index bc449df45..ec1a6405c 100644 --- a/README.md +++ b/README.md @@ -73,41 +73,13 @@ We will be editing the docker daemon config file which is usually present at `/e Once you have enabled this option on *all* the GPU nodes you wish to use, you can then enable GPU support in your cluster by deploying the following Daemonset: -VGPU: -``` -$ kubectl create -f volcano-vgpu-device-plugin.yml -``` - -GPU-SHARE (**Will be deprecated in volcano v1.9**): ```shell $ kubectl create -f volcano-device-plugin.yml ``` **Note** that volcano device plugin can be configured. For example, it can specify gpu strategy by adding in the yaml file ''args: ["--gpu-strategy=number"]'' under ''image: volcanosh/volcano-device-plugin''. More configuration can be found at [volcano device plugin configuration](https://github.com/volcano-sh/devices/blob/master/doc/config.md). -### Running VGPU Jobs - -VGPU can be requested by both set "volcano.sh/vgpu-number" and "volcano.sh/vgpu-memory" in resource.limit - -```shell script -$ cat < **WARNING:** *if you don't request GPUs when using the device plugin with NVIDIA images all > the GPUs on the machine will be exposed inside your container.* -### Running GPU Number Jobs (**Will be deprecated in volcano v1.9**) +### Running GPU Number Jobs (Without number isolation) NVIDIA GPUs can now be requested via container level resource requirements using the resource name volcano.sh/gpu-number: @@ -170,7 +142,7 @@ EOF Please note that: - the device plugin feature is beta as of Kubernetes v1.11. -- the gpu-share device plugin is alpha and is missing the following features, and will be deprecated in volcano v1.9 +- the Volcano device plugin is alpha and is missing - More comprehensive GPU health checking features - GPU cleanup features - GPU hard isolation @@ -180,16 +152,19 @@ The next sections are focused on building the device plugin and running it. ### With Docker -#### Deploy as DaemonSet: +#### Build +```shell +$ make ubuntu20.04. +``` -GPU-SHARE: +#### Run locally ```shell -$ kubectl create -f nvidia-device-plugin.yml +$ docker run --security-opt=no-new-privileges --cap-drop=ALL --network=none -it -v /var/lib/kubelet/device-plugins:/var/lib/kubelet/device-plugins nvidia/k8s-device-plugin:{version} ``` -VGPU: +#### Deploy as DaemonSet: ```shell -$ kubectl create -f nvidia-vgpu-device-plugin.yml +$ kubectl create -f nvidia-device-plugin.yml ``` # Issues and Contributing diff --git a/cmd/vgpu/main.go b/cmd/vgpu/main.go deleted file mode 100644 index ea6432a89..000000000 --- a/cmd/vgpu/main.go +++ /dev/null @@ -1,191 +0,0 @@ -/* -Copyright 2023 The Volcano Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ -package main - -import ( - "fmt" - "syscall" - - "github.com/NVIDIA/gpu-monitoring-tools/bindings/go/nvml" - "github.com/fsnotify/fsnotify" - "github.com/spf13/cobra" - "github.com/spf13/viper" - "k8s.io/klog/v2" - pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1" - nvidiadevice "volcano.sh/k8s-device-plugin/pkg/plugin/vgpu" - "volcano.sh/k8s-device-plugin/pkg/plugin/vgpu/config" - "volcano.sh/k8s-device-plugin/pkg/plugin/vgpu/util" -) - -var ( - failOnInitErrorFlag bool - migStrategyFlag string - - rootCmd = &cobra.Command{ - Use: "device-plugin", - Short: "kubernetes vgpu device-plugin", - Run: func(cmd *cobra.Command, args []string) { - if err := start(); err != nil { - klog.Fatal(err) - } - }, - } -) - -type devicePluginConfigs struct { - Nodeconfig []struct { - Name string `json:"name"` - Devicememoryscaling float64 `json:"devicememoryscaling"` - Devicesplitcount int `json:"devicesplitcount"` - Migstrategy string `json:"migstrategy"` - } `json:"nodeconfig"` -} - -func init() { - // https://github.com/spf13/viper/issues/461 - viper.BindEnv("node-name", "NODE_NAME") - - rootCmd.Flags().SortFlags = false - rootCmd.PersistentFlags().SortFlags = false - - rootCmd.Flags().StringVar(&migStrategyFlag, "mig-strategy", "none", "the desired strategy for exposing MIG devices on GPUs that support it:\n\t\t[none | single | mixed]") - rootCmd.Flags().BoolVar(&failOnInitErrorFlag, "fail-on-init-error", true, "fail the plugin if an error is encountered during initialization, otherwise block indefinitely") - rootCmd.Flags().UintVar(&config.DeviceSplitCount, "device-split-count", 2, "the number for NVIDIA device split") - rootCmd.Flags().Float64Var(&config.DeviceCoresScaling, "device-cores-scaling", 1.0, "the ratio for NVIDIA device cores scaling") - rootCmd.Flags().StringVar(&config.NodeName, "node-name", viper.GetString("node-name"), "node name") - - rootCmd.PersistentFlags().AddGoFlagSet(util.GlobalFlagSet()) - rootCmd.AddCommand(config.VersionCmd) -} - -func start() error { - klog.Info("Loading NVML") - if err := nvml.Init(); err != nil { - klog.Infof("Failed to initialize NVML: %v.", err) - klog.Infof("If this is a GPU node, did you set the docker default runtime to `nvidia`?") - klog.Infof("You can check the prerequisites at: https://github.com/NVIDIA/k8s-device-plugin#prerequisites") - klog.Infof("You can learn how to set the runtime at: https://github.com/NVIDIA/k8s-device-plugin#quick-start") - klog.Infof("If this is not a GPU node, you should set up a toleration or nodeSelector to only deploy this plugin on GPU nodes") - if failOnInitErrorFlag { - return fmt.Errorf("failed to initialize NVML: %v", err) - } - select {} - } - defer func() { klog.Info("Shutdown of NVML returned:", nvml.Shutdown()) }() - - klog.Info("Starting FS watcher.") - watcher, err := NewFSWatcher(pluginapi.DevicePluginPath) - if err != nil { - return fmt.Errorf("failed to create FS watcher: %v", err) - } - defer watcher.Close() - - klog.Info("Starting OS watcher.") - sigs := NewOSWatcher(syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) - - cache := nvidiadevice.NewDeviceCache() - cache.Start() - defer cache.Stop() - - register := nvidiadevice.NewDeviceRegister(cache) - register.Start() - defer register.Stop() - - var plugins []*nvidiadevice.NvidiaDevicePlugin -restart: - // If we are restarting, idempotently stop any running plugins before - // recreating them below. - for _, p := range plugins { - p.Stop() - } - klog.Info("Retreiving plugins.") - migStrategy, err := nvidiadevice.NewMigStrategy(migStrategyFlag) - if err != nil { - return fmt.Errorf("error creating MIG strategy: %v", err) - } - plugins = migStrategy.GetPlugins(cache) - - started := 0 - pluginStartError := make(chan struct{}) - for _, p := range plugins { - // Just continue if there are no devices to serve for plugin p. - if len(p.Devices()) == 0 { - continue - } - - // Start the gRPC server for plugin p and connect it with the kubelet. - if err := p.Start(); err != nil { - //klog.SetOutput(os.Stderr) - klog.Info("Could not contact Kubelet, retrying. Did you enable the device plugin feature gate?") - klog.Info("You can check the prerequisites at: https://github.com/NVIDIA/k8s-device-plugin#prerequisites") - klog.Info("You can learn how to set the runtime at: https://github.com/NVIDIA/k8s-device-plugin#quick-start") - close(pluginStartError) - goto events - } - started++ - } - - if started == 0 { - klog.Info("No devices found. Waiting indefinitely.") - } - -events: - // Start an infinite loop, waiting for several indicators to either log - // some messages, trigger a restart of the plugins, or exit the program. - for { - select { - // If there was an error starting any plugins, restart them all. - case <-pluginStartError: - goto restart - - // Detect a kubelet restart by watching for a newly created - // 'pluginapi.KubeletSocket' file. When this occurs, restart this loop, - // restarting all of the plugins in the process. - case event := <-watcher.Events: - if event.Name == pluginapi.KubeletSocket && event.Op&fsnotify.Create == fsnotify.Create { - klog.Infof("inotify: %s created, restarting.", pluginapi.KubeletSocket) - goto restart - } - - // Watch for any other fs errors and log them. - case err := <-watcher.Errors: - klog.Infof("inotify: %s", err) - - // Watch for any signals from the OS. On SIGHUP, restart this loop, - // restarting all of the plugins in the process. On all other - // signals, exit the loop and exit the program. - case s := <-sigs: - switch s { - case syscall.SIGHUP: - klog.Info("Received SIGHUP, restarting.") - goto restart - default: - klog.Infof("Received signal %v, shutting down.", s) - for _, p := range plugins { - p.Stop() - } - break events - } - } - } - return nil -} - -func main() { - if err := rootCmd.Execute(); err != nil { - klog.Fatal(err) - } -} diff --git a/cmd/vgpu/watchers.go b/cmd/vgpu/watchers.go deleted file mode 100644 index 120cf1c42..000000000 --- a/cmd/vgpu/watchers.go +++ /dev/null @@ -1,48 +0,0 @@ -/* -Copyright 2023 The Volcano Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "os" - "os/signal" - - "github.com/fsnotify/fsnotify" -) - -func NewFSWatcher(files ...string) (*fsnotify.Watcher, error) { - watcher, err := fsnotify.NewWatcher() - if err != nil { - return nil, err - } - - for _, f := range files { - err = watcher.Add(f) - if err != nil { - watcher.Close() - return nil, err - } - } - - return watcher, nil -} - -func NewOSWatcher(sigs ...os.Signal) chan os.Signal { - sigChan := make(chan os.Signal, 1) - signal.Notify(sigChan, sigs...) - - return sigChan -} diff --git a/docker/amd64/Dockerfile.ubuntu20.04 b/docker/amd64/Dockerfile.ubuntu20.04 index c05629a43..c9c15e985 100644 --- a/docker/amd64/Dockerfile.ubuntu20.04 +++ b/docker/amd64/Dockerfile.ubuntu20.04 @@ -40,4 +40,4 @@ ENV NVIDIA_DRIVER_CAPABILITIES=utility COPY --from=build /go/src/volcano.sh/devices/volcano-device-plugin /usr/bin/volcano-device-plugin -ENTRYPOINT ["volcano-device-plugin"] +ENTRYPOINT ["volcano-device-plugin"] \ No newline at end of file diff --git a/docker/amd64/Dockerfile.vgpu-ubuntu20.04 b/docker/amd64/Dockerfile.vgpu-ubuntu20.04 deleted file mode 100644 index 4d3e216e4..000000000 --- a/docker/amd64/Dockerfile.vgpu-ubuntu20.04 +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -FROM ubuntu:20.04 as build - -RUN apt-get update && apt-get install -y --no-install-recommends \ - g++ \ - ca-certificates \ - wget && \ - rm -rf /var/lib/apt/lists/* - -ENV GOLANG_VERSION 1.19.3 -RUN wget -nv -O - https://storage.googleapis.com/golang/go${GOLANG_VERSION}.linux-amd64.tar.gz \ - | tar -C /usr/local -xz -ENV GOPATH /go -ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH - -WORKDIR /go/src/volcano.sh/devices -COPY . . - -RUN export CGO_LDFLAGS_ALLOW='-Wl,--unresolved-symbols=ignore-in-object-files' && \ - go build -ldflags="-s -w" -o volcano-vgpu-device-plugin ./cmd/vgpu - -FROM debian:stretch-slim - -ENV NVIDIA_VISIBLE_DEVICES=all -ENV NVIDIA_DRIVER_CAPABILITIES=utility - -COPY --from=build /go/src/volcano.sh/devices/volcano-vgpu-device-plugin /usr/bin/volcano-vgpu-device-plugin -RUN mkdir -p /k8s-vgpu/lib/nvidia - -ENTRYPOINT ["volcano-vgpu-device-plugin"] diff --git a/examples/gpu-share.yml b/example/gpu-share.yml similarity index 100% rename from examples/gpu-share.yml rename to example/gpu-share.yml diff --git a/examples/vgpu-case01.yml b/examples/vgpu-case01.yml deleted file mode 100644 index fc350e904..000000000 --- a/examples/vgpu-case01.yml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: pod1 -spec: - restartPolicy: OnFailure - schedulerName: volcano - containers: - - image: nvidia/cuda:10.1-base-ubuntu18.04 - name: pod1-ctr - command: ["sleep"] - args: ["100000"] - resources: - limits: - volcano.sh/vgpu-memory: 1024 - #volcano.sh/vgpu-number: 1 diff --git a/examples/vgpu-case02.yml b/examples/vgpu-case02.yml deleted file mode 100644 index d55586d0d..000000000 --- a/examples/vgpu-case02.yml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: pod1 -spec: - restartPolicy: OnFailure - schedulerName: volcano - containers: - - image: nvidia/cuda:10.1-base-ubuntu18.04 - name: pod1-ctr - command: ["sleep"] - args: ["100000"] - resources: - limits: - volcano.sh/vgpu-number: 1 - volcano.sh/vgpu-memory: 1024 diff --git a/examples/vgpu-case03.yml b/examples/vgpu-case03.yml deleted file mode 100644 index cf79aaf87..000000000 --- a/examples/vgpu-case03.yml +++ /dev/null @@ -1,26 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: gpu-pod12 -spec: - schedulerName: volcano - containers: - - name: ubuntu-container - image: ubuntu:18.04 - command: ["bash", "-c", "sleep 86400"] - resources: - limits: - volcano.sh/vgpu-number: 2 # requesting 2 vGPUs - volcano.sh/vgpu-memory: 2000 - #volcano.sh/vgpu-memory-percentage: 50 #Each vGPU containers 50% device memory of that GPU. Can not be used with nvidia.com/gpumem - - name: ubuntu-container0 - image: ubuntu:18.04 - command: ["bash", "-c", "sleep 86400"] - - name: ubuntu-container1 - image: ubuntu:18.04 - command: ["bash", "-c", "sleep 86400"] - resources: - limits: - volcano.sh/vgpu-number: 2 # requesting 2 vGPUs - volcano.sh/vgpu-memory: 3000 - diff --git a/go.mod b/go.mod index 654ddae99..4a3a7d389 100644 --- a/go.mod +++ b/go.mod @@ -28,15 +28,13 @@ replace ( ) require ( - github.com/NVIDIA/go-gpuallocator v0.2.3 - github.com/NVIDIA/gpu-monitoring-tools v0.0.0-20201109160820-d08ea3cdcce4 + github.com/NVIDIA/gpu-monitoring-tools v0.0.0-20200421213100-de959f43b55a github.com/fsnotify/fsnotify v1.4.9 + github.com/mitchellh/gox v1.0.1 // indirect github.com/prometheus/common v0.4.1 - github.com/spf13/cobra v0.0.5 - github.com/spf13/viper v1.3.2 github.com/stretchr/testify v1.5.1 github.com/urfave/cli/v2 v2.4.0 - golang.org/x/net v0.0.0-20200421231249-e086a090c8fd + golang.org/x/net v0.0.0-20200421231249-e086a090c8fd // indirect google.golang.org/grpc v1.29.0 k8s.io/api v0.18.2 k8s.io/apimachinery v0.18.2 diff --git a/go.sum b/go.sum index 53fd49c75..27b5a88d8 100644 --- a/go.sum +++ b/go.sum @@ -13,7 +13,6 @@ github.com/Azure/go-autorest/autorest/to v0.2.0/go.mod h1:GunWKJp1AEqgMaGLV+iocm github.com/Azure/go-autorest/autorest/validation v0.1.0/go.mod h1:Ha3z/SqBeaalWQvokg3NZAlQTalVMtOIAs1aGK7G6u8= github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/GoogleCloudPlatform/k8s-cloud-provider v0.0.0-20190822182118-27a4ced34534/go.mod h1:iroGtC8B3tQiqtds1l+mgk/BBOrxbqjH+eUfFQYRc14= @@ -21,10 +20,8 @@ github.com/JeffAshton/win_pdh v0.0.0-20161109143554-76bb4ee9f0ab/go.mod h1:3VYc5 github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd/go.mod h1:64YHyfSL2R96J44Nlwm39UHepQbyR5q10x7iYa1ks2E= github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= github.com/Microsoft/hcsshim v0.0.0-20190417211021-672e52e9209d/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg= -github.com/NVIDIA/go-gpuallocator v0.2.3 h1:YTXxNpHo71u16DPBWSsHpxV/Eac76ElF5B/rDOM9zqc= -github.com/NVIDIA/go-gpuallocator v0.2.3/go.mod h1:cNlWZtJeN15qXGoOzZnOA9yY3CiJrUtUsfGJHFefiDA= -github.com/NVIDIA/gpu-monitoring-tools v0.0.0-20201109160820-d08ea3cdcce4 h1:6KSetbMgb2MieLm34BNJKiEuiP5Tj9Tr94wTipnlYDA= -github.com/NVIDIA/gpu-monitoring-tools v0.0.0-20201109160820-d08ea3cdcce4/go.mod h1:l0Cq257MSJMvg9URCXUjc8pgKY2SK1oSvIx6qG0bzzc= +github.com/NVIDIA/gpu-monitoring-tools v0.0.0-20200421213100-de959f43b55a h1:DploSoAcQ8tcaEjGnaPGt0I33v/dsc53Xam+OsOx3X8= +github.com/NVIDIA/gpu-monitoring-tools v0.0.0-20200421213100-de959f43b55a/go.mod h1:l0Cq257MSJMvg9URCXUjc8pgKY2SK1oSvIx6qG0bzzc= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/OpenPeeDeeP/depguard v1.0.0/go.mod h1:7/4sitnI9YlQgTLLk734QlzXT8DuHVnAyztLplQjk+o= github.com/OpenPeeDeeP/depguard v1.0.1/go.mod h1:xsIw86fROiiwelg+jB2uM9PiKihMMmUx/1V+TNhjQvM= @@ -257,7 +254,6 @@ github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXi github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= @@ -277,11 +273,12 @@ github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-version v1.0.0 h1:21MVWPKDphxa7ineQQTrCU5brh7OuVVAzGOCnnCPtE8= +github.com/hashicorp/go-version v1.0.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru v0.0.0-20180201235237-0fb14efe8c47/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/hcl v0.0.0-20180404174102-ef8a98b0bbce/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w= -github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/heketi/heketi v9.0.1-0.20190917153846-c2e2a4ab7ab9+incompatible/go.mod h1:bB9ly3RchcQqsQ9CpyaQwvva7RS5ytVoSoholZQON6o= github.com/heketi/tests v0.0.0-20151005000721-f3775cbcefd6/go.mod h1:xGMAM8JLi7UkZt1i4FQeQy0R2T8GLUwQhOP5M1gBhy4= @@ -289,7 +286,6 @@ github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU= github.com/jimstudt/http-authentication v0.0.0-20140401203705-3eca13d6893a/go.mod h1:wK6yTYYcgjHE1Z1QtXACPDjcFJyBskHEdagmnq3vsP8= @@ -334,7 +330,6 @@ github.com/lucas-clemente/quic-go v0.10.2/go.mod h1:hvaRS9IHjFLMq76puFJeWNfmn+H7 github.com/lucas-clemente/quic-go-certificates v0.0.0-20160823095156-d2f86524cced/go.mod h1:NCcRLrOTZbzhZvixZLlERbJtDtYsmMw8Jc4vS8Z0g58= github.com/magiconair/properties v1.7.6/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -361,8 +356,11 @@ github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrk github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936/go.mod h1:r1VsdOzOPt1ZSrGZWFoNhsAedKnEd6r9Np1+5blZCWk= github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= +github.com/mitchellh/gox v1.0.1 h1:x0jD3dcHk9a9xPSDN6YEL4xL6Qz0dvNYm8yZqui5chI= +github.com/mitchellh/gox v1.0.1/go.mod h1:ED6BioOGXMswlXa2zxfh/xdd5QhwYliBFn9V18Ap4z4= +github.com/mitchellh/iochan v1.0.0 h1:C+X3KsSTLFVBr/tK1eYN/vs4rJcvsiLU338UhYPJWeY= +github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20180220230111-00c29f56e238/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= @@ -402,7 +400,6 @@ github.com/opencontainers/runtime-spec v1.0.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/ github.com/opencontainers/selinux v1.3.1-0.20190929122143-5215b1806f52/go.mod h1:+BLncwf63G4dgOzykXAxcmnFlUaOlkDdmw/CqsW6pjs= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.1.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -456,18 +453,14 @@ github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4k github.com/sourcegraph/go-diff v0.5.1/go.mod h1:j2dHj3m8aZgQO8lMTcTnBcXkRRRqi34cd2MNlA9u1mE= github.com/spf13/afero v1.1.0/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/cast v1.2.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg= -github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.2/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/jwalterweatherman v0.0.0-20180109140146-7c0cea34c8ec/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= @@ -475,7 +468,6 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.0.2/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM= -github.com/spf13/viper v1.3.2 h1:VUFqw5KcqRf7i70GOzW7N+Q7+gxVBkSSqiXB12+JQ4M= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/storageos/go-api v0.0.0-20180912212459-343b3eff91fc/go.mod h1:ZrLn+e0ZuF3Y65PNF6dIwbJPZqfmtCXxFm9ckv0agOY= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= diff --git a/pkg/filewatcher/filewatcher.go b/pkg/filewatcher/filewatcher.go index f39800319..ab6c7a9cc 100644 --- a/pkg/filewatcher/filewatcher.go +++ b/pkg/filewatcher/filewatcher.go @@ -1,5 +1,5 @@ /* -Copyright 2023 The Volcano Authors. +Copyright 2020 The Volcano Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/plugin/interface.go b/pkg/plugin/interface.go index bf10a8431..8fbb93f0c 100644 --- a/pkg/plugin/interface.go +++ b/pkg/plugin/interface.go @@ -1,5 +1,5 @@ /* -Copyright 2023 The Volcano Authors. +Copyright 2020 The Volcano Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/plugin/nvidia/const.go b/pkg/plugin/nvidia/const.go index 092272f6d..ca1cc83d8 100644 --- a/pkg/plugin/nvidia/const.go +++ b/pkg/plugin/nvidia/const.go @@ -1,5 +1,5 @@ /* -Copyright 2023 The Volcano Authors. +Copyright 2020 The Volcano Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -38,6 +38,4 @@ const ( TotalGPUMemory = "VOLCANO_GPU_MEMORY_TOTAL" // NVIDIA visible devices VisibleDevice = "NVIDIA_VISIBLE_DEVICES" - - deviceName = "gpu" ) diff --git a/pkg/plugin/nvidia/kube_interactor.go b/pkg/plugin/nvidia/kube_interactor.go index b66ae0c3a..62107ca6a 100644 --- a/pkg/plugin/nvidia/kube_interactor.go +++ b/pkg/plugin/nvidia/kube_interactor.go @@ -1,5 +1,5 @@ /* -Copyright 2023 The Volcano Authors. +Copyright 2020 The Volcano Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -87,7 +87,9 @@ func (ki *KubeInteractor) GetPendingPodsOnNode() ([]v1.Pod, error) { return nil, fmt.Errorf("kube interactor timedout: %v", err) } - res = append(res, pl.Items...) + for _, pod := range pl.Items { + res = append(res, pod) + } return res, nil } diff --git a/pkg/plugin/nvidia/nvidia.go b/pkg/plugin/nvidia/nvidia.go index 98a0d7c46..30ad92d86 100644 --- a/pkg/plugin/nvidia/nvidia.go +++ b/pkg/plugin/nvidia/nvidia.go @@ -1,18 +1,18 @@ /* -Copyright 2023 The Volcano Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ + * Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package nvidia @@ -93,7 +93,7 @@ func buildDevice(d *nvml.Device, devIndex uint) *Device { if d.CPUAffinity != nil { dev.Topology = &pluginapi.TopologyInfo{ Nodes: []*pluginapi.NUMANode{ - { + &pluginapi.NUMANode{ ID: int64(*(d.CPUAffinity)), }, }, diff --git a/pkg/plugin/nvidia/server.go b/pkg/plugin/nvidia/server.go index a3f5ffaef..cf312a89a 100644 --- a/pkg/plugin/nvidia/server.go +++ b/pkg/plugin/nvidia/server.go @@ -1,5 +1,5 @@ /* -Copyright 2023 The Volcano Authors. +Copyright 2020 The Volcano Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -34,7 +34,7 @@ import ( "google.golang.org/grpc" pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1" apis "volcano.sh/k8s-device-plugin/pkg/apis" - "volcano.sh/k8s-device-plugin/pkg/lock" + "volcano.sh/k8s-device-plugin/pkg/util" ) // NvidiaDevicePlugin implements the Kubernetes device plugin API @@ -136,6 +136,7 @@ func (m *NvidiaDevicePlugin) Name() string { func (m *NvidiaDevicePlugin) Start() error { m.initialize() // must be called after initialize + //gpustrategy=share patch number if m.resourceName == VolcanoGPUMemory { if err := m.kubeInteractor.PatchGPUResourceOnNode(len(m.physicalDevices)); err != nil { log.Printf("failed to patch gpu resource: %v", err) @@ -366,7 +367,7 @@ func (m *NvidiaDevicePlugin) Allocate(ctx context.Context, reqs *pluginapi.Alloc } if candidatePod == nil { - lock.ReleaseNodeLock(m.kubeInteractor.nodeName, deviceName) + util.ReleaseNodeLock(m.kubeInteractor.nodeName, "gpu") return nil, fmt.Errorf("failed to find candidate pod") } @@ -374,13 +375,13 @@ Allocate: ids := GetGPUIDsFromPodAnnotation(candidatePod) if ids == nil { klog.Warningf("Failed to get the gpu ids for pod %s/%s", candidatePod.Namespace, candidatePod.Name) - lock.ReleaseNodeLock(m.kubeInteractor.nodeName, deviceName) + util.ReleaseNodeLock(m.kubeInteractor.nodeName, "gpu") return nil, fmt.Errorf("failed to find gpu ids") } for _, id := range ids { _, exist := m.GetDeviceNameByIndex(uint(id)) if !exist { - lock.ReleaseNodeLock(m.kubeInteractor.nodeName, deviceName) + util.ReleaseNodeLock(m.kubeInteractor.nodeName, "gpu") klog.Warningf("Failed to find the dev for pod %s/%s because it's not able to find dev with index %d", candidatePod.Namespace, candidatePod.Name, id) return nil, fmt.Errorf("failed to find gpu device") @@ -401,13 +402,13 @@ Allocate: err = UpdatePodAnnotations(m.kubeInteractor.clientset, candidatePod) if err != nil { - lock.ReleaseNodeLock(m.kubeInteractor.nodeName, deviceName) + util.ReleaseNodeLock(m.kubeInteractor.nodeName, "gpu") return nil, fmt.Errorf("failed to update pod annotation %v", err) } - lock.UseClient(m.kubeInteractor.clientset) + util.UseClient(m.kubeInteractor.clientset) klog.V(3).Infoln("Releasing lock: nodeName=", m.kubeInteractor.nodeName) - err = lock.ReleaseNodeLock(m.kubeInteractor.nodeName, deviceName) + err = util.ReleaseNodeLock(m.kubeInteractor.nodeName, "gpu") if err != nil { klog.Errorf("failed to release lock %s", err.Error()) } diff --git a/pkg/plugin/nvidia/utils.go b/pkg/plugin/nvidia/utils.go index 330223fc6..af0cc5a93 100644 --- a/pkg/plugin/nvidia/utils.go +++ b/pkg/plugin/nvidia/utils.go @@ -1,5 +1,5 @@ /* -Copyright 2023 The Volcano Authors. +Copyright 2020 The Volcano Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/plugin/vgpu/cache.go b/pkg/plugin/vgpu/cache.go deleted file mode 100644 index 7077e4561..000000000 --- a/pkg/plugin/vgpu/cache.go +++ /dev/null @@ -1,84 +0,0 @@ -/* -Copyright 2023 The Volcano Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package vgpu - -import ( - "sync" - - pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1" -) - -type DeviceCache struct { - GpuDeviceManager - - cache []*Device - stopCh chan interface{} - unhealthy chan *Device - notifyCh map[string]chan *Device - mutex sync.Mutex -} - -func NewDeviceCache() *DeviceCache { - return &DeviceCache{ - GpuDeviceManager: GpuDeviceManager{true}, - stopCh: make(chan interface{}), - unhealthy: make(chan *Device), - notifyCh: make(map[string]chan *Device), - } -} - -func (d *DeviceCache) AddNotifyChannel(name string, ch chan *Device) { - d.mutex.Lock() - defer d.mutex.Unlock() - d.notifyCh[name] = ch -} - -func (d *DeviceCache) RemoveNotifyChannel(name string) { - d.mutex.Lock() - defer d.mutex.Unlock() - delete(d.notifyCh, name) -} - -func (d *DeviceCache) Start() { - d.cache = d.Devices() - go d.CheckHealth(d.stopCh, d.cache, d.unhealthy) - go d.notify() -} - -func (d *DeviceCache) Stop() { - close(d.stopCh) -} - -func (d *DeviceCache) GetCache() []*Device { - return d.cache -} - -func (d *DeviceCache) notify() { - for { - select { - case <-d.stopCh: - return - case dev := <-d.unhealthy: - dev.Health = pluginapi.Unhealthy - d.mutex.Lock() - for _, ch := range d.notifyCh { - ch <- dev - } - d.mutex.Unlock() - } - } -} diff --git a/pkg/plugin/vgpu/config/config.go b/pkg/plugin/vgpu/config/config.go deleted file mode 100644 index d463d612c..000000000 --- a/pkg/plugin/vgpu/config/config.go +++ /dev/null @@ -1,25 +0,0 @@ -/* -Copyright 2023 The Volcano Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package config - -var ( - DeviceSplitCount uint - DeviceCoresScaling float64 - NodeName string - RuntimeSocketFlag string - DisableCoreLimit bool -) diff --git a/pkg/plugin/vgpu/config/version.go b/pkg/plugin/vgpu/config/version.go deleted file mode 100644 index 5977aed2a..000000000 --- a/pkg/plugin/vgpu/config/version.go +++ /dev/null @@ -1,38 +0,0 @@ -/* -Copyright 2023 The Volcano Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package config - -import ( - "fmt" - - "github.com/spf13/cobra" -) - -var ( - version string - VersionCmd = &cobra.Command{ - Use: "version", - Short: "print version", - Run: func(cmd *cobra.Command, args []string) { - fmt.Println(Version()) - }, - } -) - -func Version() string { - return version -} diff --git a/pkg/plugin/vgpu/mig-strategy.go b/pkg/plugin/vgpu/mig-strategy.go deleted file mode 100644 index 016e29b1c..000000000 --- a/pkg/plugin/vgpu/mig-strategy.go +++ /dev/null @@ -1,183 +0,0 @@ -/* -Copyright 2023 The Volcano Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package vgpu - -import ( - "fmt" - "log" - - "github.com/NVIDIA/go-gpuallocator/gpuallocator" - "github.com/NVIDIA/gpu-monitoring-tools/bindings/go/nvml" - pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1" - "volcano.sh/k8s-device-plugin/pkg/plugin/vgpu/util" -) - -// Constants representing the various MIG strategies -const ( - MigStrategyNone = "none" - MigStrategySingle = "single" - MigStrategyMixed = "mixed" -) - -// MigStrategyResourceSet holds a set of resource names for a given MIG strategy -type MigStrategyResourceSet map[string]struct{} - -// MigStrategy provides an interface for building the set of plugins required to implement a given MIG strategy -type MigStrategy interface { - GetPlugins(cache *DeviceCache) []*NvidiaDevicePlugin - MatchesResource(mig *nvml.Device, resource string) bool -} - -// NewMigStrategy returns a reference to a given MigStrategy based on the 'strategy' passed in -func NewMigStrategy(strategy string) (MigStrategy, error) { - switch strategy { - case MigStrategyNone: - return &migStrategyNone{}, nil - case MigStrategySingle: - return &migStrategySingle{}, nil - case MigStrategyMixed: - return &migStrategyMixed{}, nil - } - return nil, fmt.Errorf("unknown strategy: %v", strategy) -} - -type migStrategyNone struct{} -type migStrategySingle struct{} -type migStrategyMixed struct{} - -// migStrategyNone -func (s *migStrategyNone) GetPlugins(cache *DeviceCache) []*NvidiaDevicePlugin { - return []*NvidiaDevicePlugin{ - NewNvidiaDevicePlugin( - //"nvidia.com/gpu", - util.ResourceName, - cache, - gpuallocator.NewBestEffortPolicy(), - pluginapi.DevicePluginPath+"nvidia-gpu.sock"), - } -} - -func (s *migStrategyNone) MatchesResource(mig *nvml.Device, resource string) bool { - panic("Should never be called") -} - -// migStrategySingle -func (s *migStrategySingle) GetPlugins(cache *DeviceCache) []*NvidiaDevicePlugin { - panic("single mode in MIG currently not supported") -} - -func (s *migStrategySingle) validMigDevice(mig *nvml.Device) bool { - attr, err := mig.GetAttributes() - check(err) - - return attr.GpuInstanceSliceCount == attr.ComputeInstanceSliceCount -} - -func (s *migStrategySingle) getResourceName(mig *nvml.Device) string { - attr, err := mig.GetAttributes() - check(err) - - g := attr.GpuInstanceSliceCount - c := attr.ComputeInstanceSliceCount - gb := ((attr.MemorySizeMB + 1024 - 1) / 1024) - - var r string - if g == c { - r = fmt.Sprintf("mig-%dg.%dgb", g, gb) - } else { - r = fmt.Sprintf("mig-%dc.%dg.%dgb", c, g, gb) - } - - return r -} - -func (s *migStrategySingle) MatchesResource(mig *nvml.Device, resource string) bool { - return true -} - -// migStrategyMixed -func (s *migStrategyMixed) GetPlugins(cache *DeviceCache) []*NvidiaDevicePlugin { - devices := NewMIGCapableDevices() - - if err := devices.AssertAllMigEnabledDevicesAreValid(); err != nil { - panic(fmt.Errorf("at least one device with migEnabled=true was not configured correctly: %v", err)) - } - - resources := make(MigStrategyResourceSet) - migs, err := devices.GetAllMigDevices() - if err != nil { - panic(fmt.Errorf("unable to retrieve list of MIG devices: %v", err)) - } - for _, mig := range migs { - r := s.getResourceName(mig) - if !s.validMigDevice(mig) { - log.Printf("Skipping unsupported MIG device: %v", r) - continue - } - resources[r] = struct{}{} - } - - plugins := []*NvidiaDevicePlugin{ - NewNvidiaDevicePlugin( - //"nvidia.com/gpu", - util.ResourceName, - cache, - gpuallocator.NewBestEffortPolicy(), - pluginapi.DevicePluginPath+"nvidia-gpu.sock"), - } - - for resource := range resources { - plugin := NewMIGNvidiaDevicePlugin( - "nvidia.com/"+resource, - NewMigDeviceManager(s, resource), - "NVIDIA_VISIBLE_DEVICES", - gpuallocator.Policy(nil), - pluginapi.DevicePluginPath+"nvidia-"+resource+".sock") - plugins = append(plugins, plugin) - } - - return plugins -} - -func (s *migStrategyMixed) validMigDevice(mig *nvml.Device) bool { - attr, err := mig.GetAttributes() - check(err) - - return attr.GpuInstanceSliceCount == attr.ComputeInstanceSliceCount -} - -func (s *migStrategyMixed) getResourceName(mig *nvml.Device) string { - attr, err := mig.GetAttributes() - check(err) - - g := attr.GpuInstanceSliceCount - c := attr.ComputeInstanceSliceCount - gb := ((attr.MemorySizeMB + 1024 - 1) / 1024) - - var r string - if g == c { - r = fmt.Sprintf("mig-%dg.%dgb", g, gb) - } else { - r = fmt.Sprintf("mig-%dc.%dg.%dgb", c, g, gb) - } - - return r -} - -func (s *migStrategyMixed) MatchesResource(mig *nvml.Device, resource string) bool { - return s.getResourceName(mig) == resource -} diff --git a/pkg/plugin/vgpu/mig.go b/pkg/plugin/vgpu/mig.go deleted file mode 100644 index addb4ea81..000000000 --- a/pkg/plugin/vgpu/mig.go +++ /dev/null @@ -1,240 +0,0 @@ -/* -Copyright 2023 The Volcano Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package vgpu - -import ( - "bufio" - "fmt" - "log" - "os" - - "github.com/NVIDIA/gpu-monitoring-tools/bindings/go/nvml" -) - -const ( - nvidiaProcDriverPath = "/proc/driver/nvidia" - nvidiaCapabilitiesPath = nvidiaProcDriverPath + "/capabilities" - - nvcapsProcDriverPath = "/proc/driver/nvidia-caps" - nvcapsMigMinorsPath = nvcapsProcDriverPath + "/mig-minors" - nvcapsDevicePath = "/dev/nvidia-caps" -) - -// MIGCapableDevices stores information about all devices on the node -type MIGCapableDevices struct { - // devicesMap holds a list of devices, separated by whether they have MigEnabled or not - devicesMap map[bool][]*nvml.Device -} - -// NewMIGCapableDevices creates a new MIGCapableDevices struct and returns a pointer to it. -func NewMIGCapableDevices() *MIGCapableDevices { - return &MIGCapableDevices{ - devicesMap: nil, // Is initialized on first use - } -} - -func (devices *MIGCapableDevices) getDevicesMap() (map[bool][]*nvml.Device, error) { - if devices.devicesMap == nil { - n, err := nvml.GetDeviceCount() - if err != nil { - return nil, err - } - - migEnabledDevicesMap := make(map[bool][]*nvml.Device) - for i := uint(0); i < n; i++ { - d, err := nvml.NewDeviceLite(i) - if err != nil { - return nil, err - } - - isMigEnabled, err := d.IsMigEnabled() - if err != nil { - return nil, err - } - - migEnabledDevicesMap[isMigEnabled] = append(migEnabledDevicesMap[isMigEnabled], d) - } - - devices.devicesMap = migEnabledDevicesMap - } - return devices.devicesMap, nil -} - -// GetDevicesWithMigEnabled returns a list of devices with migEnabled=true -func (devices *MIGCapableDevices) GetDevicesWithMigEnabled() ([]*nvml.Device, error) { - devicesMap, err := devices.getDevicesMap() - if err != nil { - return nil, err - } - return devicesMap[true], nil -} - -// GetDevicesWithMigDisabled returns a list of devices with migEnabled=false -func (devices *MIGCapableDevices) GetDevicesWithMigDisabled() ([]*nvml.Device, error) { - devicesMap, err := devices.getDevicesMap() - if err != nil { - return nil, err - } - return devicesMap[false], nil -} - -// AssertAllMigEnabledDevicesAreValid ensures that all devices with migEnabled=true are valid. This means: -// * The have at least 1 mig devices associated with them -// Returns nill if the device is valid, or an error if these are not valid -func (devices *MIGCapableDevices) AssertAllMigEnabledDevicesAreValid() error { - devicesMap, err := devices.getDevicesMap() - if err != nil { - return err - } - - for _, d := range devicesMap[true] { - migs, err := d.GetMigDevices() - if err != nil { - return err - } - if len(migs) == 0 { - return fmt.Errorf("no MIG devices associated with %v: %v", d.Path, d.UUID) - } - } - return nil -} - -// GetAllMigDevices returns a list of all MIG devices. -func (devices *MIGCapableDevices) GetAllMigDevices() ([]*nvml.Device, error) { - devicesMap, err := devices.getDevicesMap() - if err != nil { - return nil, err - } - - var migs []*nvml.Device - for _, d := range devicesMap[true] { - devs, err := d.GetMigDevices() - if err != nil { - return nil, err - } - migs = append(migs, devs...) - } - return migs, nil -} - -// GetMigCapabilityDevicePaths returns a mapping of MIG capability path to device node path -func GetMigCapabilityDevicePaths() (map[string]string, error) { - // Open nvcapsMigMinorsPath for walking. - // If the nvcapsMigMinorsPath does not exist, then we are not on a MIG - // capable machine, so there is nothing to do. - // The format of this file is discussed in: - // https://docs.nvidia.com/datacenter/tesla/mig-user-guide/index.html#unique_1576522674 - minorsFile, err := os.Open(nvcapsMigMinorsPath) - if os.IsNotExist(err) { - return nil, nil - } - if err != nil { - return nil, fmt.Errorf("error opening MIG minors file: %v", err) - } - defer minorsFile.Close() - - // Define a function to process each each line of nvcapsMigMinorsPath - processLine := func(line string) (string, int, error) { - var gpu, gi, ci, migMinor int - - // Look for a CI access file - n, _ := fmt.Sscanf(line, "gpu%d/gi%d/ci%d/access %d", &gpu, &gi, &ci, &migMinor) - if n == 4 { - capPath := fmt.Sprintf(nvidiaCapabilitiesPath+"/gpu%d/mig/gi%d/ci%d/access", gpu, gi, ci) - return capPath, migMinor, nil - } - - // Look for a GI access file - n, _ = fmt.Sscanf(line, "gpu%d/gi%d/access %d", &gpu, &gi, &migMinor) - if n == 3 { - capPath := fmt.Sprintf(nvidiaCapabilitiesPath+"/gpu%d/mig/gi%d/access", gpu, gi) - return capPath, migMinor, nil - } - - // Look for the MIG config file - n, _ = fmt.Sscanf(line, "config %d", &migMinor) - if n == 1 { - capPath := fmt.Sprintf(nvidiaCapabilitiesPath + "/mig/config") - return capPath, migMinor, nil - } - - // Look for the MIG monitor file - n, _ = fmt.Sscanf(line, "monitor %d", &migMinor) - if n == 1 { - capPath := fmt.Sprintf(nvidiaCapabilitiesPath + "/mig/monitor") - return capPath, migMinor, nil - } - - return "", 0, fmt.Errorf("unparsable line: %v", line) - } - - // Walk each line of nvcapsMigMinorsPath and construct a mapping of nvidia - // capabilities path to device minor for that capability - capsDevicePaths := make(map[string]string) - scanner := bufio.NewScanner(minorsFile) - for scanner.Scan() { - capPath, migMinor, err := processLine(scanner.Text()) - if err != nil { - log.Printf("Skipping line in MIG minors file: %v", err) - continue - } - capsDevicePaths[capPath] = fmt.Sprintf(nvcapsDevicePath+"/nvidia-cap%d", migMinor) - } - return capsDevicePaths, nil -} - -// GetMigDeviceNodePaths returns a list of device node paths associated with a MIG device -func GetMigDeviceNodePaths(parent *nvml.Device, mig *nvml.Device) ([]string, error) { - capDevicePaths, err := GetMigCapabilityDevicePaths() - if err != nil { - return nil, fmt.Errorf("error getting MIG capability device paths: %v", err) - } - - var gpu int - _, err = fmt.Sscanf(parent.Path, "/dev/nvidia%d", &gpu) - if err != nil { - return nil, fmt.Errorf("error getting GPU minor: %v", err) - } - - gi, err := mig.GetGPUInstanceId() - if err != nil { - return nil, fmt.Errorf("error getting MIG GPU instance ID: %v", err) - } - - ci, err := mig.GetComputeInstanceId() - if err != nil { - return nil, fmt.Errorf("error getting MIG compute instance ID: %v", err) - } - - giCapPath := fmt.Sprintf(nvidiaCapabilitiesPath+"/gpu%d/mig/gi%d/access", gpu, gi) - if _, exists := capDevicePaths[giCapPath]; !exists { - return nil, fmt.Errorf("missing MIG GPU instance capability path: %v", giCapPath) - } - - ciCapPath := fmt.Sprintf(nvidiaCapabilitiesPath+"/gpu%d/mig/gi%d/ci%d/access", gpu, gi, ci) - if _, exists := capDevicePaths[ciCapPath]; !exists { - return nil, fmt.Errorf("missing MIG GPU instance capability path: %v", giCapPath) - } - - devicePaths := []string{ - parent.Path, - capDevicePaths[giCapPath], - capDevicePaths[ciCapPath], - } - - return devicePaths, nil -} diff --git a/pkg/plugin/vgpu/nvidia.go b/pkg/plugin/vgpu/nvidia.go deleted file mode 100644 index c35682a34..000000000 --- a/pkg/plugin/vgpu/nvidia.go +++ /dev/null @@ -1,269 +0,0 @@ -/* -Copyright 2023 The Volcano Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package vgpu - -import ( - "fmt" - "log" - "os" - "strconv" - "strings" - - "github.com/NVIDIA/gpu-monitoring-tools/bindings/go/nvml" - "volcano.sh/k8s-device-plugin/pkg/plugin/vgpu/util" - - pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1" -) - -const ( - envDisableHealthChecks = "DP_DISABLE_HEALTHCHECKS" - allHealthChecks = "xids" -) - -// Device couples an underlying pluginapi.Device type with its device node paths -type Device struct { - pluginapi.Device - Paths []string - Index string - Memory uint64 -} - -// ResourceManager provides an interface for listing a set of Devices and checking health on them -type ResourceManager interface { - Devices() []*Device - CheckHealth(stop <-chan interface{}, devices []*Device, unhealthy chan<- *Device) -} - -// GpuDeviceManager implements the ResourceManager interface for full GPU devices -type GpuDeviceManager struct { - skipMigEnabledGPUs bool -} - -// MigDeviceManager implements the ResourceManager interface for MIG devices -type MigDeviceManager struct { - strategy MigStrategy - resource string -} - -func check(err error) { - if err != nil { - log.Panicln("Fatal:", err) - } -} - -// NewGpuDeviceManager returns a reference to a new GpuDeviceManager -func NewGpuDeviceManager(skipMigEnabledGPUs bool) *GpuDeviceManager { - return &GpuDeviceManager{ - skipMigEnabledGPUs: skipMigEnabledGPUs, - } -} - -// NewMigDeviceManager returns a reference to a new MigDeviceManager -func NewMigDeviceManager(strategy MigStrategy, resource string) *MigDeviceManager { - return &MigDeviceManager{ - strategy: strategy, - resource: resource, - } -} - -// Devices returns a list of devices from the GpuDeviceManager -func (g *GpuDeviceManager) Devices() []*Device { - n, err := nvml.GetDeviceCount() - check(err) - if n > util.DeviceLimit { - n = util.DeviceLimit - } - - var devs []*Device - for i := uint(0); i < n; i++ { - d, err := nvml.NewDevice(i) - check(err) - - migEnabled, err := d.IsMigEnabled() - check(err) - - if migEnabled && g.skipMigEnabledGPUs { - continue - } - - devs = append(devs, buildDevice(d, []string{d.Path}, fmt.Sprintf("%v", i))) - } - - return devs -} - -// Devices returns a list of devices from the MigDeviceManager -func (m *MigDeviceManager) Devices() []*Device { - n, err := nvml.GetDeviceCount() - check(err) - - var devs []*Device - for i := uint(0); i < n; i++ { - d, err := nvml.NewDeviceLite(i) - check(err) - - migEnabled, err := d.IsMigEnabled() - check(err) - - if !migEnabled { - continue - } - - migs, err := d.GetMigDevices() - check(err) - - for j, mig := range migs { - if !m.strategy.MatchesResource(mig, m.resource) { - continue - } - - paths, err := GetMigDeviceNodePaths(d, mig) - check(err) - - devs = append(devs, buildDevice(mig, paths, fmt.Sprintf("%v:%v", i, j))) - } - } - - return devs -} - -// CheckHealth performs health checks on a set of devices, writing to the 'unhealthy' channel with any unhealthy devices -func (g *GpuDeviceManager) CheckHealth(stop <-chan interface{}, devices []*Device, unhealthy chan<- *Device) { - checkHealth(stop, devices, unhealthy) -} - -// CheckHealth performs health checks on a set of devices, writing to the 'unhealthy' channel with any unhealthy devices -func (m *MigDeviceManager) CheckHealth(stop <-chan interface{}, devices []*Device, unhealthy chan<- *Device) { - checkHealth(stop, devices, unhealthy) -} - -func buildDevice(d *nvml.Device, paths []string, index string) *Device { - dev := Device{} - dev.ID = d.UUID - dev.Health = pluginapi.Healthy - dev.Paths = paths - dev.Index = index - dev.Memory = *d.Memory - if d.CPUAffinity != nil { - dev.Topology = &pluginapi.TopologyInfo{ - Nodes: []*pluginapi.NUMANode{ - { - ID: int64(*(d.CPUAffinity)), - }, - }, - } - } - return &dev -} - -func checkHealth(stop <-chan interface{}, devices []*Device, unhealthy chan<- *Device) { - disableHealthChecks := strings.ToLower(os.Getenv(envDisableHealthChecks)) - if disableHealthChecks == "all" { - disableHealthChecks = allHealthChecks - } - if strings.Contains(disableHealthChecks, "xids") { - return - } - - eventSet := nvml.NewEventSet() - defer nvml.DeleteEventSet(eventSet) - - for _, d := range devices { - gpu, _, _, err := nvml.ParseMigDeviceUUID(d.ID) - if err != nil { - gpu = d.ID - } - - err = nvml.RegisterEventForDevice(eventSet, nvml.XidCriticalError, gpu) - if err != nil && strings.HasSuffix(err.Error(), "Not Supported") { - log.Printf("Warning: %s is too old to support healthchecking: %s. Marking it unhealthy.", d.ID, err) - unhealthy <- d - continue - } - check(err) - } - - for { - select { - case <-stop: - return - default: - } - - e, err := nvml.WaitForEvent(eventSet, 5000) - if err != nil && e.Etype != nvml.XidCriticalError { - continue - } - - // FIXME: formalize the full list and document it. - // http://docs.nvidia.com/deploy/xid-errors/index.html#topic_4 - // Application errors: the GPU should still be healthy - if e.Edata == 31 || e.Edata == 43 || e.Edata == 45 { - continue - } - - if e.UUID == nil || len(*e.UUID) == 0 { - // All devices are unhealthy - log.Printf("XidCriticalError: Xid=%d, All devices will go unhealthy.", e.Edata) - for _, d := range devices { - unhealthy <- d - } - continue - } - - for _, d := range devices { - // Please see https://github.com/NVIDIA/gpu-monitoring-tools/blob/148415f505c96052cb3b7fdf443b34ac853139ec/bindings/go/nvml/nvml.h#L1424 - // for the rationale why gi and ci can be set as such when the UUID is a full GPU UUID and not a MIG device UUID. - gpu, gi, ci, err := nvml.ParseMigDeviceUUID(d.ID) - if err != nil { - gpu = d.ID - gi = 0xFFFFFFFF - ci = 0xFFFFFFFF - } - - if gpu == *e.UUID && gi == *e.GpuInstanceId && ci == *e.ComputeInstanceId { - log.Printf("XidCriticalError: Xid=%d on Device=%s, the device will go unhealthy.", e.Edata, d.ID) - unhealthy <- d - } - } - } -} - -// getAdditionalXids returns a list of additional Xids to skip from the specified string. -// The input is treaded as a comma-separated string and all valid uint64 values are considered as Xid values. Invalid values -// are ignored. -func getAdditionalXids(input string) []uint64 { - if input == "" { - return nil - } - - var additionalXids []uint64 - for _, additionalXid := range strings.Split(input, ",") { - trimmed := strings.TrimSpace(additionalXid) - if trimmed == "" { - continue - } - xid, err := strconv.ParseUint(trimmed, 10, 64) - if err != nil { - log.Printf("Ignoring malformed Xid value %v: %v", trimmed, err) - continue - } - additionalXids = append(additionalXids, xid) - } - - return additionalXids -} diff --git a/pkg/plugin/vgpu/plugin.go b/pkg/plugin/vgpu/plugin.go deleted file mode 100644 index dbe3b6ab5..000000000 --- a/pkg/plugin/vgpu/plugin.go +++ /dev/null @@ -1,422 +0,0 @@ -/* -Copyright 2023 The Volcano Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package vgpu - -import ( - "errors" - "fmt" - "log" - "net" - "os" - "path" - "strings" - "time" - - "k8s.io/klog/v2" - "volcano.sh/k8s-device-plugin/pkg/lock" - "volcano.sh/k8s-device-plugin/pkg/plugin/vgpu/config" - "volcano.sh/k8s-device-plugin/pkg/plugin/vgpu/util" - - "github.com/NVIDIA/go-gpuallocator/gpuallocator" - "golang.org/x/net/context" - "google.golang.org/grpc" - pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1" -) - -// Constants to represent the various device list strategies -const ( - DeviceListStrategyEnvvar = "envvar" - DeviceListStrategyVolumeMounts = "volume-mounts" -) - -// Constants to represent the various device id strategies -const ( - DeviceIDStrategyUUID = "uuid" - DeviceIDStrategyIndex = "index" -) - -// Constants for use by the 'volume-mounts' device list strategy -const ( - deviceListAsVolumeMountsHostPath = "/dev/null" - deviceListAsVolumeMountsContainerPathRoot = "/var/run/nvidia-container-devices" -) - -// NvidiaDevicePlugin implements the Kubernetes device plugin API -type NvidiaDevicePlugin struct { - ResourceManager - deviceCache *DeviceCache - resourceName string - deviceListEnvvar string - allocatePolicy gpuallocator.Policy - socket string - - server *grpc.Server - cachedDevices []*Device - health chan *Device - stop chan interface{} - changed chan struct{} - migStrategy string -} - -// NewNvidiaDevicePlugin returns an initialized NvidiaDevicePlugin -func NewNvidiaDevicePlugin(resourceName string, deviceCache *DeviceCache, allocatePolicy gpuallocator.Policy, socket string) *NvidiaDevicePlugin { - return &NvidiaDevicePlugin{ - deviceCache: deviceCache, - resourceName: resourceName, - allocatePolicy: allocatePolicy, - socket: socket, - migStrategy: "none", - - // These will be reinitialized every - // time the plugin server is restarted. - server: nil, - health: nil, - stop: nil, - } -} - -// NewNvidiaDevicePlugin returns an initialized NvidiaDevicePlugin -func NewMIGNvidiaDevicePlugin(resourceName string, resourceManager ResourceManager, deviceListEnvvar string, allocatePolicy gpuallocator.Policy, socket string) *NvidiaDevicePlugin { - return &NvidiaDevicePlugin{ - ResourceManager: resourceManager, - resourceName: resourceName, - deviceListEnvvar: deviceListEnvvar, - allocatePolicy: allocatePolicy, - socket: socket, - - // These will be reinitialized every - // time the plugin server is restarted. - cachedDevices: nil, - server: nil, - health: nil, - stop: nil, - migStrategy: "mixed", - } -} - -func (m *NvidiaDevicePlugin) initialize() { - var err error - if strings.Compare(m.migStrategy, "mixed") == 0 { - m.cachedDevices = m.ResourceManager.Devices() - } - m.server = grpc.NewServer([]grpc.ServerOption{}...) - m.health = make(chan *Device) - m.stop = make(chan interface{}) - check(err) -} - -func (m *NvidiaDevicePlugin) cleanup() { - close(m.stop) - m.server = nil - m.health = nil - m.stop = nil -} - -// Start starts the gRPC server, registers the device plugin with the Kubelet, -// and starts the device healthchecks. -func (m *NvidiaDevicePlugin) Start() error { - m.initialize() - - err := m.Serve() - if err != nil { - log.Printf("Could not start device plugin for '%s': %s", m.resourceName, err) - m.cleanup() - return err - } - log.Printf("Starting to serve '%s' on %s", m.resourceName, m.socket) - - err = m.Register() - if err != nil { - log.Printf("Could not register device plugin: %s", err) - m.Stop() - return err - } - log.Printf("Registered device plugin for '%s' with Kubelet", m.resourceName) - - if strings.Compare(m.migStrategy, "none") == 0 { - m.deviceCache.AddNotifyChannel("plugin", m.health) - } else if strings.Compare(m.migStrategy, "mixed") == 0 { - go m.CheckHealth(m.stop, m.cachedDevices, m.health) - } else { - log.Panicln("migstrategy not recognized", m.migStrategy) - } - return nil -} - -// Stop stops the gRPC server. -func (m *NvidiaDevicePlugin) Stop() error { - if m == nil || m.server == nil { - return nil - } - log.Printf("Stopping to serve '%s' on %s", m.resourceName, m.socket) - m.deviceCache.RemoveNotifyChannel("plugin") - m.server.Stop() - if err := os.Remove(m.socket); err != nil && !os.IsNotExist(err) { - return err - } - m.cleanup() - return nil -} - -// Serve starts the gRPC server of the device plugin. -func (m *NvidiaDevicePlugin) Serve() error { - os.Remove(m.socket) - sock, err := net.Listen("unix", m.socket) - if err != nil { - return err - } - - pluginapi.RegisterDevicePluginServer(m.server, m) - - go func() { - lastCrashTime := time.Now() - restartCount := 0 - for { - log.Printf("Starting GRPC server for '%s'", m.resourceName) - err := m.server.Serve(sock) - if err == nil { - break - } - - log.Printf("GRPC server for '%s' crashed with error: %v", m.resourceName, err) - - // restart if it has not been too often - // i.e. if server has crashed more than 5 times and it didn't last more than one hour each time - if restartCount > 5 { - // quit - log.Fatalf("GRPC server for '%s' has repeatedly crashed recently. Quitting", m.resourceName) - } - timeSinceLastCrash := time.Since(lastCrashTime).Seconds() - lastCrashTime = time.Now() - if timeSinceLastCrash > 3600 { - // it has been one hour since the last crash.. reset the count - // to reflect on the frequency - restartCount = 1 - } else { - restartCount++ - } - } - }() - - // Wait for server to start by launching a blocking connexion - conn, err := m.dial(m.socket, 5*time.Second) - if err != nil { - return err - } - conn.Close() - - return nil -} - -// Register registers the device plugin for the given resourceName with Kubelet. -func (m *NvidiaDevicePlugin) Register() error { - conn, err := m.dial(pluginapi.KubeletSocket, 5*time.Second) - if err != nil { - return err - } - defer conn.Close() - - client := pluginapi.NewRegistrationClient(conn) - reqt := &pluginapi.RegisterRequest{ - Version: pluginapi.Version, - Endpoint: path.Base(m.socket), - ResourceName: m.resourceName, - Options: &pluginapi.DevicePluginOptions{}, - } - - _, err = client.Register(context.Background(), reqt) - if err != nil { - return err - } - return nil -} - -// GetDevicePluginOptions returns the values of the optional settings for this plugin -func (m *NvidiaDevicePlugin) GetDevicePluginOptions(context.Context, *pluginapi.Empty) (*pluginapi.DevicePluginOptions, error) { - options := &pluginapi.DevicePluginOptions{} - return options, nil -} - -// ListAndWatch lists devices and update that list according to the health status -func (m *NvidiaDevicePlugin) ListAndWatch(e *pluginapi.Empty, s pluginapi.DevicePlugin_ListAndWatchServer) error { - _ = s.Send(&pluginapi.ListAndWatchResponse{Devices: m.apiDevices()}) - for { - select { - case <-m.stop: - return nil - case d := <-m.health: - // FIXME: there is no way to recover from the Unhealthy state. - //d.Health = pluginapi.Unhealthy - log.Printf("'%s' device marked unhealthy: %s", m.resourceName, d.ID) - _ = s.Send(&pluginapi.ListAndWatchResponse{Devices: m.apiDevices()}) - } - } -} - -func (m *NvidiaDevicePlugin) MIGAllocate(ctx context.Context, reqs *pluginapi.AllocateRequest) (*pluginapi.AllocateResponse, error) { - responses := pluginapi.AllocateResponse{} - for _, req := range reqs.ContainerRequests { - for _, id := range req.DevicesIDs { - if !m.deviceExists(id) { - return nil, fmt.Errorf("invalid allocation request for '%s': unknown device: %s", m.resourceName, id) - } - } - - response := pluginapi.ContainerAllocateResponse{} - - uuids := req.DevicesIDs - deviceIDs := m.deviceIDsFromUUIDs(uuids) - - response.Envs = m.apiEnvs(m.deviceListEnvvar, deviceIDs) - - klog.Infof("response=", response.Envs) - responses.ContainerResponses = append(responses.ContainerResponses, &response) - } - - return &responses, nil -} - -// Allocate which return list of devices. -func (m *NvidiaDevicePlugin) Allocate(ctx context.Context, reqs *pluginapi.AllocateRequest) (*pluginapi.AllocateResponse, error) { - klog.Infoln("Allocate", reqs.ContainerRequests) - if len(reqs.ContainerRequests) > 1 { - return &pluginapi.AllocateResponse{}, errors.New("multiple Container Requests not supported") - } - if strings.Compare(m.migStrategy, "mixed") == 0 { - return m.MIGAllocate(ctx, reqs) - } - responses := pluginapi.AllocateResponse{} - nodename := os.Getenv("NODE_NAME") - - current, err := util.GetPendingPod(nodename) - if err != nil { - lock.ReleaseNodeLock(nodename, util.VGPUDeviceName) - return &pluginapi.AllocateResponse{}, err - } - - for idx := range reqs.ContainerRequests { - _, devreq, err := util.GetNextDeviceRequest(util.NvidiaGPUDevice, *current) - klog.Infoln("deviceAllocateFromAnnotation=", devreq) - if err != nil { - klog.Errorln("get device from annotation failed", err.Error()) - util.PodAllocationFailed(nodename, current) - return &pluginapi.AllocateResponse{}, err - } - if len(devreq) != len(reqs.ContainerRequests[idx].DevicesIDs) { - klog.Errorln("device number not matched", devreq, reqs.ContainerRequests[idx].DevicesIDs) - util.PodAllocationFailed(nodename, current) - return &pluginapi.AllocateResponse{}, errors.New("device number not matched") - } - - err = util.EraseNextDeviceTypeFromAnnotation(util.NvidiaGPUDevice, *current) - if err != nil { - klog.Errorln("Erase annotation failed", err.Error()) - util.PodAllocationFailed(nodename, current) - return &pluginapi.AllocateResponse{}, err - } - - response := pluginapi.ContainerAllocateResponse{} - response.Envs = make(map[string]string) - for i, dev := range devreq { - limitKey := fmt.Sprintf("CUDA_DEVICE_MEMORY_LIMIT_%v", i) - response.Envs[limitKey] = fmt.Sprintf("%vm", dev.Usedmem) - tmp := response.Envs["NVIDIA_VISIBLE_DEVICES"] - if i > 0 { - response.Envs["NVIDIA_VISIBLE_DEVICES"] = fmt.Sprintf("%v,%v", tmp, dev.UUID) - } else { - response.Envs["NVIDIA_VISIBLE_DEVICES"] = dev.UUID - } - } - } - klog.Infoln("Allocate Response", responses.ContainerResponses) - util.PodAllocationTrySuccess(nodename, current) - return &responses, nil -} - -// PreStartContainer is unimplemented for this plugin -func (m *NvidiaDevicePlugin) PreStartContainer(context.Context, *pluginapi.PreStartContainerRequest) (*pluginapi.PreStartContainerResponse, error) { - return &pluginapi.PreStartContainerResponse{}, nil -} - -// dial establishes the gRPC communication with the registered device plugin. -func (m *NvidiaDevicePlugin) dial(unixSocketPath string, timeout time.Duration) (*grpc.ClientConn, error) { - c, err := grpc.Dial(unixSocketPath, grpc.WithInsecure(), grpc.WithBlock(), - grpc.WithTimeout(timeout), - grpc.WithDialer(func(addr string, timeout time.Duration) (net.Conn, error) { - return net.DialTimeout("unix", addr, timeout) - }), - ) - - if err != nil { - return nil, err - } - - return c, nil -} - -func (m *NvidiaDevicePlugin) Devices() []*Device { - if strings.Compare(m.migStrategy, "none") == 0 { - return m.deviceCache.GetCache() - } - if strings.Compare(m.migStrategy, "mixed") == 0 { - return m.ResourceManager.Devices() - } - log.Panic("migStrategy not recognized,exiting...") - return []*Device{} -} - -func (m *NvidiaDevicePlugin) deviceExists(id string) bool { - for _, d := range m.cachedDevices { - if d.ID == id { - return true - } - } - return false -} - -func (m *NvidiaDevicePlugin) deviceIDsFromUUIDs(uuids []string) []string { - return uuids -} - -func (m *NvidiaDevicePlugin) apiDevices() []*pluginapi.Device { - if strings.Compare(m.migStrategy, "mixed") == 0 { - var pdevs []*pluginapi.Device - for _, d := range m.cachedDevices { - pdevs = append(pdevs, &d.Device) - } - return pdevs - } - devices := m.Devices() - var res []*pluginapi.Device - for _, dev := range devices { - for i := uint(0); i < config.DeviceSplitCount; i++ { - id := fmt.Sprintf("%v-%v", dev.ID, i) - res = append(res, &pluginapi.Device{ - ID: id, - Health: dev.Health, - Topology: nil, - }) - } - } - return res -} - -func (m *NvidiaDevicePlugin) apiEnvs(envvar string, deviceIDs []string) map[string]string { - return map[string]string{ - envvar: strings.Join(deviceIDs, ","), - } -} diff --git a/pkg/plugin/vgpu/register.go b/pkg/plugin/vgpu/register.go deleted file mode 100644 index 626bcbd38..000000000 --- a/pkg/plugin/vgpu/register.go +++ /dev/null @@ -1,108 +0,0 @@ -/* -Copyright 2023 The Volcano Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package vgpu - -import ( - "fmt" - "time" - - "github.com/NVIDIA/gpu-monitoring-tools/bindings/go/nvml" - "k8s.io/klog/v2" - "volcano.sh/k8s-device-plugin/pkg/plugin/vgpu/config" - "volcano.sh/k8s-device-plugin/pkg/plugin/vgpu/util" -) - -type DevListFunc func() []*Device - -type DeviceRegister struct { - deviceCache *DeviceCache - unhealthy chan *Device - stopCh chan struct{} -} - -func NewDeviceRegister(deviceCache *DeviceCache) *DeviceRegister { - return &DeviceRegister{ - deviceCache: deviceCache, - unhealthy: make(chan *Device), - stopCh: make(chan struct{}), - } -} - -func (r *DeviceRegister) Start() { - r.deviceCache.AddNotifyChannel("register", r.unhealthy) - go r.WatchAndRegister() -} - -func (r *DeviceRegister) Stop() { - close(r.stopCh) -} - -func (r *DeviceRegister) apiDevices() *[]*util.DeviceInfo { - devs := r.deviceCache.GetCache() - res := make([]*util.DeviceInfo, 0, len(devs)) - for _, dev := range devs { - ndev, err := nvml.NewDeviceByUUID(dev.ID) - if err != nil { - fmt.Println("nvml new device by uuid error id=", dev.ID) - panic(0) - } else { - klog.V(3).Infoln("nvml registered device id=", dev.ID, "memory=", *ndev.Memory, "type=", *ndev.Model) - } - registeredmem := int32(*ndev.Memory) - res = append(res, &util.DeviceInfo{ - Id: dev.ID, - Count: int32(config.DeviceSplitCount), - Devmem: registeredmem, - Type: fmt.Sprintf("%v-%v", "NVIDIA", *ndev.Model), - Health: dev.Health == "healthy", - }) - } - return &res -} - -func (r *DeviceRegister) RegistrInAnnotation() error { - devices := r.apiDevices() - annos := make(map[string]string) - node, err := util.GetNode(config.NodeName) - if err != nil { - klog.Errorln("get node error", err.Error()) - return err - } - encodeddevices := util.EncodeNodeDevices(*devices) - annos[util.NodeHandshake] = "Reported " + time.Now().String() - annos[util.NodeNvidiaDeviceRegistered] = encodeddevices - klog.Infoln("Reporting devices", encodeddevices, "in", time.Now().String()) - err = util.PatchNodeAnnotations(node, annos) - - if err != nil { - klog.Errorln("patch node error", err.Error()) - } - return err -} - -func (r *DeviceRegister) WatchAndRegister() { - klog.Infof("into WatchAndRegister") - for { - err := r.RegistrInAnnotation() - if err != nil { - klog.Errorf("register error, %v", err) - time.Sleep(time.Second * 5) - } else { - time.Sleep(time.Second * 30) - } - } -} diff --git a/pkg/plugin/vgpu/util/types.go b/pkg/plugin/vgpu/util/types.go deleted file mode 100644 index 0d7aa8750..000000000 --- a/pkg/plugin/vgpu/util/types.go +++ /dev/null @@ -1,97 +0,0 @@ -/* -Copyright 2023 The Volcano Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package util - -const ( - AssignedTimeAnnotations = "volcano.sh/vgpu-time" - AssignedIDsAnnotations = "volcano.sh/vgpu-ids-new" - AssignedIDsToAllocateAnnotations = "volcano.sh/devices-to-allocate" - AssignedNodeAnnotations = "volcano.sh/vgpu-node" - BindTimeAnnotations = "volcano.sh/bind-time" - DeviceBindPhase = "volcano.sh/bind-phase" - - GPUInUse = "nvidia.com/use-gputype" - GPUNoUse = "nvidia.com/nouse-gputype" - - DeviceBindAllocating = "allocating" - DeviceBindFailed = "failed" - DeviceBindSuccess = "success" - - DeviceLimit = 100 - - BestEffort string = "best-effort" - Restricted string = "restricted" - Guaranteed string = "guaranteed" - - NvidiaGPUDevice = "NVIDIA" - NvidiaGPUCommonWord = "GPU" - - NodeLockTime = "volcano.sh/mutex.lock" - MaxLockRetry = 5 - - NodeHandshake = "volcano.sh/node-vgpu-handshake" - NodeNvidiaDeviceRegistered = "volcano.sh/node-vgpu-register" - - // DeviceName used to indicate this device - VGPUDeviceName = "vgpu" -) - -var ( - ResourceName string - ResourceMem string - ResourceCores string - ResourceMemPercentage string - ResourcePriority string - DebugMode bool - - MLUResourceCount string - MLUResourceMemory string - - KnownDevice = map[string]string{ - NodeHandshake: NodeNvidiaDeviceRegistered, - } -) - -type ContainerDevice struct { - UUID string - Type string - Usedmem int32 - Usedcores int32 -} - -type ContainerDeviceRequest struct { - Nums int32 - Type string - Memreq int32 - MemPercentagereq int32 - Coresreq int32 -} - -type ContainerDevices []ContainerDevice - -type PodDevices []ContainerDevices - -type DeviceInfo struct { - Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` - Count int32 `protobuf:"varint,2,opt,name=count,proto3" json:"count,omitempty"` - Devmem int32 `protobuf:"varint,3,opt,name=devmem,proto3" json:"devmem,omitempty"` - Type string `protobuf:"bytes,4,opt,name=type,proto3" json:"type,omitempty"` - Health bool `protobuf:"varint,5,opt,name=health,proto3" json:"health,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} diff --git a/pkg/plugin/vgpu/util/util.go b/pkg/plugin/vgpu/util/util.go deleted file mode 100644 index 63cd2c362..000000000 --- a/pkg/plugin/vgpu/util/util.go +++ /dev/null @@ -1,308 +0,0 @@ -/* -Copyright 2023 The Volcano Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package util - -import ( - "context" - "encoding/json" - "errors" - "flag" - "fmt" - "os" - "strconv" - "strings" - - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - k8stypes "k8s.io/apimachinery/pkg/types" - "k8s.io/klog/v2" - "volcano.sh/k8s-device-plugin/pkg/lock" -) - -var DevicesToHandle []string - -func init() { - client, _ := lock.NewClient() - lock.UseClient(client) - DevicesToHandle = []string{} - DevicesToHandle = append(DevicesToHandle, NvidiaGPUCommonWord) -} - -func GlobalFlagSet() *flag.FlagSet { - fs := flag.NewFlagSet(os.Args[0], flag.ExitOnError) - fs.StringVar(&ResourceName, "resource-name", "volcano.sh/vgpu-number", "resource name") - fs.BoolVar(&DebugMode, "debug", false, "debug mode") - klog.InitFlags(fs) - return fs -} - -func GetNode(nodename string) (*v1.Node, error) { - n, err := lock.GetClient().CoreV1().Nodes().Get(context.Background(), nodename, metav1.GetOptions{}) - return n, err -} - -func GetPendingPod(node string) (*v1.Pod, error) { - podlist, err := lock.GetClient().CoreV1().Pods("").List(context.Background(), metav1.ListOptions{}) - if err != nil { - return nil, err - } - for _, p := range podlist.Items { - if _, ok := p.Annotations[BindTimeAnnotations]; !ok { - continue - } - if phase, ok := p.Annotations[DeviceBindPhase]; !ok { - continue - } else { - if strings.Compare(phase, DeviceBindAllocating) != 0 { - continue - } - } - if n, ok := p.Annotations[AssignedNodeAnnotations]; !ok { - continue - } else { - if strings.Compare(n, node) == 0 { - return &p, nil - } - } - } - return nil, nil -} - -func DecodeNodeDevices(str string) []*DeviceInfo { - if !strings.Contains(str, ":") { - return []*DeviceInfo{} - } - tmp := strings.Split(str, ":") - var retval []*DeviceInfo - for _, val := range tmp { - if strings.Contains(val, ",") { - items := strings.Split(val, ",") - count, _ := strconv.Atoi(items[1]) - devmem, _ := strconv.Atoi(items[2]) - health, _ := strconv.ParseBool(items[4]) - i := DeviceInfo{ - Id: items[0], - Count: int32(count), - Devmem: int32(devmem), - Type: items[3], - Health: health, - } - retval = append(retval, &i) - } - } - return retval -} - -func EncodeNodeDevices(dlist []*DeviceInfo) string { - tmp := "" - for _, val := range dlist { - tmp += val.Id + "," + strconv.FormatInt(int64(val.Count), 10) + "," + strconv.Itoa(int(val.Devmem)) + "," + val.Type + "," + strconv.FormatBool(val.Health) + ":" - } - klog.V(3).Infoln("Encoded node Devices", tmp) - return tmp -} - -func EncodeContainerDevices(cd ContainerDevices) string { - tmp := "" - for _, val := range cd { - tmp += val.UUID + "," + val.Type + "," + strconv.Itoa(int(val.Usedmem)) + "," + strconv.Itoa(int(val.Usedcores)) + ":" - } - fmt.Println("Encoded container Devices=", tmp) - return tmp - //return strings.Join(cd, ",") -} - -func EncodePodDevices(pd PodDevices) string { - var ss []string - for _, cd := range pd { - ss = append(ss, EncodeContainerDevices(cd)) - } - return strings.Join(ss, ";") -} - -func DecodeContainerDevices(str string) ContainerDevices { - if len(str) == 0 { - return ContainerDevices{} - } - cd := strings.Split(str, ":") - contdev := ContainerDevices{} - tmpdev := ContainerDevice{} - if len(str) == 0 { - return contdev - } - for _, val := range cd { - if strings.Contains(val, ",") { - tmpstr := strings.Split(val, ",") - tmpdev.UUID = tmpstr[0] - tmpdev.Type = tmpstr[1] - devmem, _ := strconv.ParseInt(tmpstr[2], 10, 32) - tmpdev.Usedmem = int32(devmem) - devcores, _ := strconv.ParseInt(tmpstr[3], 10, 32) - tmpdev.Usedcores = int32(devcores) - contdev = append(contdev, tmpdev) - } - } - return contdev -} - -func DecodePodDevices(str string) PodDevices { - if len(str) == 0 { - return PodDevices{} - } - var pd PodDevices - for _, s := range strings.Split(str, ";") { - cd := DecodeContainerDevices(s) - pd = append(pd, cd) - } - return pd -} - -func GetNextDeviceRequest(dtype string, p v1.Pod) (v1.Container, ContainerDevices, error) { - pdevices := DecodePodDevices(p.Annotations[AssignedIDsToAllocateAnnotations]) - klog.Infoln("pdevices=", pdevices) - res := ContainerDevices{} - for idx, val := range pdevices { - found := false - for _, dev := range val { - if strings.Compare(dtype, dev.Type) == 0 { - res = append(res, dev) - found = true - } - } - if found { - return p.Spec.Containers[idx], res, nil - } - } - return v1.Container{}, res, errors.New("device request not found") -} - -func EraseNextDeviceTypeFromAnnotation(dtype string, p v1.Pod) error { - pdevices := DecodePodDevices(p.Annotations[AssignedIDsToAllocateAnnotations]) - res := PodDevices{} - found := false - for _, val := range pdevices { - if found { - res = append(res, val) - continue - } else { - tmp := ContainerDevices{} - for _, dev := range val { - if strings.Compare(dtype, dev.Type) == 0 { - found = true - } else { - tmp = append(tmp, dev) - } - } - if !found { - res = append(res, val) - } else { - res = append(res, tmp) - } - } - } - klog.Infoln("After erase res=", res) - newannos := make(map[string]string) - newannos[AssignedIDsToAllocateAnnotations] = EncodePodDevices(res) - return PatchPodAnnotations(&p, newannos) -} - -func PodAllocationTrySuccess(nodeName string, pod *v1.Pod) { - refreshed, _ := lock.GetClient().CoreV1().Pods(pod.Namespace).Get(context.Background(), pod.Name, metav1.GetOptions{}) - annos := refreshed.Annotations[AssignedIDsToAllocateAnnotations] - klog.Infoln("TrySuccess:", annos) - for _, val := range DevicesToHandle { - if strings.Contains(annos, val) { - return - } - } - klog.Infoln("AllDevicesAllocateSuccess releasing lock") - PodAllocationSuccess(nodeName, pod) -} - -func PodAllocationSuccess(nodeName string, pod *v1.Pod) { - newannos := make(map[string]string) - newannos[DeviceBindPhase] = DeviceBindSuccess - err := PatchPodAnnotations(pod, newannos) - if err != nil { - klog.Errorf("patchPodAnnotations failed:%v", err.Error()) - } - err = lock.ReleaseNodeLock(nodeName, VGPUDeviceName) - if err != nil { - klog.Errorf("release lock failed:%v", err.Error()) - } -} - -func PodAllocationFailed(nodeName string, pod *v1.Pod) { - newannos := make(map[string]string) - newannos[DeviceBindPhase] = DeviceBindFailed - err := PatchPodAnnotations(pod, newannos) - if err != nil { - klog.Errorf("patchPodAnnotations failed:%v", err.Error()) - } - err = lock.ReleaseNodeLock(nodeName, VGPUDeviceName) - if err != nil { - klog.Errorf("release lock failed:%v", err.Error()) - } -} - -func PatchNodeAnnotations(node *v1.Node, annotations map[string]string) error { - type patchMetadata struct { - Annotations map[string]string `json:"annotations,omitempty"` - } - type patchPod struct { - Metadata patchMetadata `json:"metadata"` - //Spec patchSpec `json:"spec,omitempty"` - } - - p := patchPod{} - p.Metadata.Annotations = annotations - - bytes, err := json.Marshal(p) - if err != nil { - return err - } - _, err = lock.GetClient().CoreV1().Nodes(). - Patch(context.Background(), node.Name, k8stypes.StrategicMergePatchType, bytes, metav1.PatchOptions{}) - if err != nil { - klog.Infof("patch pod %v failed, %v", node.Name, err) - } - return err -} - -func PatchPodAnnotations(pod *v1.Pod, annotations map[string]string) error { - type patchMetadata struct { - Annotations map[string]string `json:"annotations,omitempty"` - } - type patchPod struct { - Metadata patchMetadata `json:"metadata"` - //Spec patchSpec `json:"spec,omitempty"` - } - - p := patchPod{} - p.Metadata.Annotations = annotations - - bytes, err := json.Marshal(p) - if err != nil { - return err - } - _, err = lock.GetClient().CoreV1().Pods(pod.Namespace). - Patch(context.Background(), pod.Name, k8stypes.StrategicMergePatchType, bytes, metav1.PatchOptions{}) - if err != nil { - klog.Infof("patch pod %v failed, %v", pod.Name, err) - } - return err -} diff --git a/pkg/lock/nodelock.go b/pkg/util/nodelock.go similarity index 88% rename from pkg/lock/nodelock.go rename to pkg/util/nodelock.go index 6ce83abc2..73021c3b4 100644 --- a/pkg/lock/nodelock.go +++ b/pkg/util/nodelock.go @@ -1,20 +1,4 @@ -/* -Copyright 2023 The Volcano Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package lock +package util import ( "context" diff --git a/volcano-device-plugin.yml b/volcano-device-plugin.yml index 1879e7664..febf4697d 100644 --- a/volcano-device-plugin.yml +++ b/volcano-device-plugin.yml @@ -32,7 +32,7 @@ rules: verbs: ["patch"] - apiGroups: [""] resources: ["pods"] - verbs: ["get", "list", "update","patch"] + verbs: ["get", "list", "update", "patch"] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 @@ -82,11 +82,10 @@ spec: priorityClassName: "system-node-critical" serviceAccount: volcano-device-plugin containers: - - image: volcanosh/volcano-device-plugin:1.0.0-ubuntu20.04 + - image: volcanosh/volcano-device-plugin:latest #args: ["--gpu-strategy=number"] - args: ["--gpu-strategy=share", "--gpu-memory-factor=1"] + #args: ["--gpu-strategy=share", "--gpu-memory-factor=1"] name: volcano-device-plugin - #command: ["sleep","infinity"] env: - name: NODE_NAME valueFrom: @@ -96,21 +95,10 @@ spec: allowPrivilegeEscalation: false capabilities: drop: ["ALL"] - add: ["SYS_ADMIN"] volumeMounts: - name: device-plugin mountPath: /var/lib/kubelet/device-plugins - - name: lib - mountPath: /usr/local/vgpu - - name: hosttmp - mountPath: /tmp volumes: - name: device-plugin hostPath: path: /var/lib/kubelet/device-plugins - - name: lib - hostPath: - path: /usr/local/vgpu - - name: hosttmp - hostPath: - path: /tmp diff --git a/volcano-vgpu-device-plugin.yml b/volcano-vgpu-device-plugin.yml deleted file mode 100644 index e71b42d88..000000000 --- a/volcano-vgpu-device-plugin.yml +++ /dev/null @@ -1,120 +0,0 @@ -# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: volcano-device-plugin - namespace: kube-system ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: volcano-device-plugin -rules: - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get","list", "watch","update","patch"] - - apiGroups: [""] - resources: ["nodes/status"] - verbs: ["patch"] - - apiGroups: [""] - resources: ["pods"] - verbs: ["get", "list", "update","patch"] ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: volcano-device-plugin -subjects: - - kind: ServiceAccount - name: volcano-device-plugin - namespace: kube-system -roleRef: - kind: ClusterRole - name: volcano-device-plugin - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: volcano-device-plugin - namespace: kube-system -spec: - selector: - matchLabels: - name: volcano-device-plugin - updateStrategy: - type: RollingUpdate - template: - metadata: - # This annotation is deprecated. Kept here for backward compatibility - # See https://kubernetes.io/docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods/ - annotations: - scheduler.alpha.kubernetes.io/critical-pod: "" - labels: - name: volcano-device-plugin - spec: - tolerations: - # This toleration is deprecated. Kept here for backward compatibility - # See https://kubernetes.io/docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods/ - - key: CriticalAddonsOnly - operator: Exists - - key: volcano.sh/gpu-memory - operator: Exists - effect: NoSchedule - # Mark this pod as a critical add-on; when enabled, the critical add-on - # scheduler reserves resources for critical add-on pods so that they can - # be rescheduled after a failure. - # See https://kubernetes.io/docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods/ - priorityClassName: "system-node-critical" - serviceAccount: volcano-device-plugin - containers: - - image: 4pdosc/volcano-vgpu-device-plugin:master - args: ["--device-split-count=10"] - lifecycle: - postStart: - exec: - command: ["/bin/sh","-c","cp -f /k8s-vgpu/lib/nvidia/* /usr/local/vgpu/"] - name: volcano-device-plugin - env: - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: HOOK_PATH - value: "/usr/local/vgpu" - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: ["ALL"] - add: ["SYS_ADMIN"] - volumeMounts: - - name: device-plugin - mountPath: /var/lib/kubelet/device-plugins - - name: lib - mountPath: /usr/local/vgpu - - name: hosttmp - mountPath: /tmp - volumes: - - name: device-plugin - hostPath: - path: /var/lib/kubelet/device-plugins - - name: lib - hostPath: - path: /usr/local/vgpu - - name: hosttmp - hostPath: - path: /tmp