graceful kernel vf handling for vlan trunking

pkg/networkservice/common/resourcepool/common.go

@@ -22,6 +22,7 @@ package resourcepool
 
 import (
 	"context"
+	"strconv"
 	"sync"
 
 	"github.com/pkg/errors"
@@ -57,10 +58,10 @@ type resourcePoolConfig struct {
 	selectedVFs  map[string]string
 }
 
-func (s *resourcePoolConfig) selectVF(connID string, vfConfig *vfconfig.VFConfig, tokenID string) (vf sriov.PCIFunction, err error) {
+func (s *resourcePoolConfig) selectVF(connID string, vfConfig *vfconfig.VFConfig, tokenID string) (vf sriov.PCIFunction, skipDriverCheck bool, err error) {
 	vfPCIAddr, err := s.resourcePool.Select(tokenID, s.driverType)
 	if err != nil {
-		return nil, errors.Wrapf(err, "failed to select VF for: %v", s.driverType)
+		return nil, false, errors.Wrapf(err, "failed to select VF for: %v", s.driverType)
 	}
 	s.selectedVFs[connID] = vfPCIAddr
 
@@ -72,25 +73,27 @@ func (s *resourcePoolConfig) selectVF(connID string, vfConfig *vfconfig.VFConfig
 
 			pf, err := s.pciPool.GetPCIFunction(pfPCIAddr)
 			if err != nil {
-				return nil, errors.Wrapf(err, "failed to get PF: %v", pfPCIAddr)
+				return nil, true, errors.Wrapf(err, "failed to get PF: %v", pfPCIAddr)
 			}
 			vfConfig.PFInterfaceName, err = pf.GetNetInterfaceName()
 			if err != nil {
-				return nil, errors.Errorf("failed to get PF net interface name: %v", pfPCIAddr)
+				return nil, true, errors.Errorf("failed to get PF net interface name: %v", pfPCIAddr)
 			}
 
 			vf, err := s.pciPool.GetPCIFunction(vfPCIAddr)
 			if err != nil {
-				return nil, errors.Wrapf(err, "failed to get VF: %v", vfPCIAddr)
+				return nil, true, errors.Wrapf(err, "failed to get VF: %v", vfPCIAddr)
 			}
 
 			vfConfig.VFNum = i
 
-			return vf, err
+			skipDriverCheck, _ = strconv.ParseBool(pfCfg.SkipDriverCheck)
+
+			return vf, skipDriverCheck, err
 		}
 	}
 
-	return nil, errors.Errorf("no VF with selected PCI address exists: %v", s.selectedVFs[connID])
+	return nil, true, errors.Errorf("no VF with selected PCI address exists: %v", s.selectedVFs[connID])
 }
 
 func (s *resourcePoolConfig) close(conn *networkservice.Connection) error {
@@ -113,7 +116,7 @@ func assignVF(ctx context.Context, logger log.Logger, conn *networkservice.Conne
 	vfConfig := &vfconfig.VFConfig{}
 
 	logger.Infof("trying to select VF for %v", resourcePool.driverType)
-	vf, err := resourcePool.selectVF(conn.GetId(), vfConfig, tokenID)
+	vf, skipDriverCheck, err := resourcePool.selectVF(conn.GetId(), vfConfig, tokenID)
 	if err != nil {
 		return err
 	}
@@ -131,7 +134,7 @@ func assignVF(ctx context.Context, logger log.Logger, conn *networkservice.Conne
 	switch resourcePool.driverType {
 	case sriov.KernelDriver:
 		vfConfig.VFInterfaceName, err = vf.GetNetInterfaceName()
-		if err != nil {
+		if err != nil && !skipDriverCheck {
 			return errors.Wrapf(err, "failed to get VF net interface name: %v", vf.GetPCIAddress())
 		}
 	case sriov.VFIOPCIDriver:

pkg/networkservice/common/resourcepool/config.yml

@@ -1,6 +1,7 @@
 ---
 physicalFunctions:
   0000:00:01.0:
+    skipDriverCheck: true
     pfKernelDriver: pf-1-driver
     vfKernelDriver: vf-1-driver
     capabilities:
@@ -14,6 +15,7 @@ physicalFunctions:
       - address: 0000:00:01.2
         iommuGroup: 1
   0000:00:02.0:
+    skipDriverCheck: true
     pfKernelDriver: pf-2-driver
     vfKernelDriver: vf-2-driver
     capabilities:

pkg/sriov/config/config.go

@@ -1,5 +1,7 @@
 // Copyright (c) 2020-2021 Doc.ai and/or its affiliates.
 //
+// Copyright (c) 2021 Nordix Foundation.
+//
 // SPDX-License-Identifier: Apache-2.0
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
@@ -20,6 +22,7 @@ package config
 import (
 	"context"
 	"fmt"
+	"strconv"
 	"strings"
 
 	"github.com/networkservicemesh/sdk/pkg/tools/log/logruslogger"
@@ -55,6 +58,7 @@ type PhysicalFunction struct {
 	VFKernelDriver   string             `yaml:"vfKernelDriver"`
 	Capabilities     []string           `yaml:"capabilities"`
 	ServiceDomains   []string           `yaml:"serviceDomains"`
+	SkipDriverCheck  string             `yaml:"skipDriverCheck"`
 	VirtualFunctions []*VirtualFunction `yaml:"virtualFunctions"`
 }
 
@@ -84,6 +88,9 @@ func (pf *PhysicalFunction) String() string {
 	_, _ = sb.WriteString(strings.Join(strs, " "))
 	_, _ = sb.WriteString("]")
 
+	_, _ = sb.WriteString(" SkipDriverCheck:")
+	_, _ = sb.WriteString(pf.SkipDriverCheck)
+
 	_, _ = sb.WriteString("}")
 	return sb.String()
 }
@@ -116,6 +123,9 @@ func ReadConfig(ctx context.Context, configFile string) (*Config, error) {
 		if len(pfCfg.ServiceDomains) == 0 {
 			return nil, errors.Errorf("%s has no ServiceDomains set", pciAddr)
 		}
+		if _, err := strconv.ParseBool(pfCfg.SkipDriverCheck); err != nil {
+			return nil, errors.Errorf("%s has invalid SkipDriverCheck set", pciAddr)
+		}
 	}
 
 	logger.WithField("Config", "ReadConfig").Infof("unmarshalled Config: %+v", cfg)

pkg/sriov/config/config.yml

@@ -1,6 +1,7 @@
 ---
 physicalFunctions:
   0000:01:00.0:
+    skipDriverCheck: true
     pfKernelDriver: pf-driver
     vfKernelDriver: vf-driver
     capabilities:
@@ -14,6 +15,7 @@ physicalFunctions:
       - address: 0000:01:00.2
         iommuGroup: 2
   0000:02:00.0:
+    skipDriverCheck: true
     pfKernelDriver: pf-driver
     vfKernelDriver: vf-driver
     capabilities:

pkg/sriov/config/config_test.go

@@ -1,4 +1,6 @@
-// Copyright (c) 2020 Doc.ai and/or its affiliates.
+// Copyright (c) 2020-2021 Doc.ai and/or its affiliates.
+//
+// Copyright (c) 2021 Nordix Foundation.
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -41,6 +43,7 @@ const (
 	vf21PciAddr     = "0000:02:00.1"
 	vf22PciAddr     = "0000:02:00.2"
 	vf23PciAddr     = "0000:02:00.3"
+	skipDriverCheck = "true"
 )
 
 func TestReadConfigFile(t *testing.T) {
@@ -68,6 +71,7 @@ func TestReadConfigFile(t *testing.T) {
 						IOMMUGroup: 2,
 					},
 				},
+				SkipDriverCheck: skipDriverCheck,
 			},
 			pf2PciAddr: {
 				PFKernelDriver: pfKernelDriver,
@@ -94,6 +98,7 @@ func TestReadConfigFile(t *testing.T) {
 						IOMMUGroup: 3,
 					},
 				},
+				SkipDriverCheck: skipDriverCheck,
 			},
 		},
 	}, cfg)

pkg/sriov/pci/pool.go

@@ -52,26 +52,20 @@ type Pool struct {
 	functions             map[string]*function // pciAddr -> *function
 	functionsByIOMMUGroup map[uint][]*function // iommuGroup -> []*function
 	vfioDir               string
-	skipDriverCheck       bool
 }
 
 type function struct {
-	function     pciFunction
-	kernelDriver string
+	function        pciFunction
+	kernelDriver    string
+	skipDriverCheck bool
 }
 
 // NewPool returns a new PCI Pool
 func NewPool(pciDevicesPath, pciDriversPath, vfioDir string, cfg *config.Config) (*Pool, error) {
-	return NewPCIPool(pciDevicesPath, pciDriversPath, vfioDir, cfg, false)
-}
-
-// NewPCIPool returns a new PCI Pool
-func NewPCIPool(pciDevicesPath, pciDriversPath, vfioDir string, cfg *config.Config, skipDriverCheck bool) (*Pool, error) {
 	p := &Pool{
 		functions:             map[string]*function{},
 		functionsByIOMMUGroup: map[uint][]*function{},
 		vfioDir:               vfioDir,
-		skipDriverCheck:       skipDriverCheck,
 	}
 
 	for pfPCIAddr, pfCfg := range cfg.PhysicalFunctions {
@@ -80,12 +74,14 @@ func NewPCIPool(pciDevicesPath, pciDriversPath, vfioDir string, cfg *config.Conf
 			return nil, err
 		}
 
-		if err := p.addFunction(&pf.Function, pfCfg.PFKernelDriver); err != nil {
+		skipDriverCheck, _ := strconv.ParseBool(pfCfg.SkipDriverCheck)
+
+		if err := p.addFunction(&pf.Function, pfCfg.PFKernelDriver, skipDriverCheck); err != nil {
 			return nil, err
 		}
 
 		for _, vf := range pf.GetVirtualFunctions() {
-			if err := p.addFunction(vf, pfCfg.VFKernelDriver); err != nil {
+			if err := p.addFunction(vf, pfCfg.VFKernelDriver, skipDriverCheck); err != nil {
 				return nil, err
 			}
 		}
@@ -99,7 +95,6 @@ func NewTestPool(physicalFunctions map[string]*sriovtest.PCIPhysicalFunction, cf
 	p := &Pool{
 		functions:             map[string]*function{},
 		functionsByIOMMUGroup: map[uint][]*function{},
-		skipDriverCheck:       true,
 	}
 
 	for pfPCIAddr, pfCfg := range cfg.PhysicalFunctions {
@@ -108,20 +103,21 @@ func NewTestPool(physicalFunctions map[string]*sriovtest.PCIPhysicalFunction, cf
 			return nil, errors.Errorf("PF doesn't exist: %v", pfPCIAddr)
 		}
 
-		_ = p.addFunction(&pf.PCIFunction, pfCfg.PFKernelDriver)
+		_ = p.addFunction(&pf.PCIFunction, pfCfg.PFKernelDriver, true)
 
 		for _, vf := range pf.Vfs {
-			_ = p.addFunction(vf, pfCfg.VFKernelDriver)
+			_ = p.addFunction(vf, pfCfg.VFKernelDriver, true)
 		}
 	}
 
 	return p, nil
 }
 
-func (p *Pool) addFunction(pcif pciFunction, kernelDriver string) (err error) {
+func (p *Pool) addFunction(pcif pciFunction, kernelDriver string, skipDriverCheck bool) (err error) {
 	f := &function{
-		function:     pcif,
-		kernelDriver: kernelDriver,
+		function:        pcif,
+		kernelDriver:    kernelDriver,
+		skipDriverCheck: skipDriverCheck,
 	}
 
 	p.functions[pcif.GetPCIAddress()] = f
@@ -162,18 +158,19 @@ func (p *Pool) BindDriver(ctx context.Context, iommuGroup uint, driverType sriov
 	}
 
 	for _, f := range p.functionsByIOMMUGroup[iommuGroup] {
-		if err := p.waitDriverGettingBound(ctx, f.function, driverType); err != nil {
+		if err := p.waitDriverGettingBound(ctx, f.function, driverType, f.skipDriverCheck); err != nil {
 			return err
 		}
 	}
 
 	return nil
 }
 
-func (p *Pool) waitDriverGettingBound(ctx context.Context, pcif pciFunction, driverType sriov.DriverType) error {
+func (p *Pool) waitDriverGettingBound(ctx context.Context, pcif pciFunction, driverType sriov.DriverType,
+	skipDriverCheck bool) error {
 	timeoutCh := time.After(driverBindTimeout)
 	for {
-		var driverCheck func(pciFunction) error
+		var driverCheck func(pciFunction, bool) error
 		switch driverType {
 		case sriov.KernelDriver:
 			driverCheck = p.kernelDriverCheck
@@ -183,7 +180,7 @@ func (p *Pool) waitDriverGettingBound(ctx context.Context, pcif pciFunction, dri
 			return errors.Errorf("driver type is not supported: %v", driverType)
 		}
 
-		err := driverCheck(pcif)
+		err := driverCheck(pcif, skipDriverCheck)
 		if err == nil {
 			return nil
 		}
@@ -198,17 +195,17 @@ func (p *Pool) waitDriverGettingBound(ctx context.Context, pcif pciFunction, dri
 	}
 }
 
-func (p *Pool) kernelDriverCheck(pcif pciFunction) error {
-	if p.skipDriverCheck {
+func (p *Pool) kernelDriverCheck(pcif pciFunction, skipDriverCheck bool) error {
+	if skipDriverCheck {
 		return nil
 	}
 
 	_, err := pcif.GetNetInterfaceName()
 	return err
 }
 
-func (p *Pool) vfioDriverCheck(pcif pciFunction) error {
-	if p.skipDriverCheck {
+func (p *Pool) vfioDriverCheck(pcif pciFunction, skipDriverCheck bool) error {
+	if skipDriverCheck {
 		return nil
 	}
 

pkg/sriov/resource/config.yml

@@ -1,6 +1,7 @@
 ---
 physicalFunctions:
   0000:01:00.0:
+    skipDriverCheck: true
     pfKernelDriver: pf-driver
     vfKernelDriver: vf-driver
     capabilities:
@@ -12,6 +13,7 @@ physicalFunctions:
       - address: 0000:01:00.1
         iommuGroup: 1
   0000:02:00.0:
+    skipDriverCheck: true
     pfKernelDriver: pf-driver
     vfKernelDriver: vf-driver
     capabilities:
@@ -25,6 +27,7 @@ physicalFunctions:
       - address: 0000:02:00.2
         iommuGroup: 2
   0000:03:00.0:
+    skipDriverCheck: true
     pfKernelDriver: pf-driver
     vfKernelDriver: vf-driver
     capabilities:

pkg/sriov/token/config.yml

@@ -1,6 +1,7 @@
 ---
 physicalFunctions:
   0000:01:00.0:
+    skipDriverCheck: true
     pfKernelDriver: pf-driver
     vfKernelDriver: vf-driver
     capabilities:
@@ -12,6 +13,7 @@ physicalFunctions:
       - address: 0000:01:00.1
         iommuGroup: 1
   0000:02:00.0:
+    skipDriverCheck: true
     pfKernelDriver: pf-driver
     vfKernelDriver: vf-driver
     capabilities: