Merge pull request kubernetes#46744 from karataliu/wincri4

Automatic merge from submit-queue

Support windows in dockershim

**What this PR does / why we need it**:
This is the 2nd part for kubernetes#45927 .

The non-cri implementation dockertools was removed from kubelet v1.7 .
Part of previous work for supporting windows container lies in v1.6 dockertools, this PR is to port them to dockershim.

Main reference file in v1.6 dockertools windows support:
https://github.com/kubernetes/kubernetes/blob/v1.6.4/pkg/kubelet/dockertools/docker_manager_windows.go

**Which issue this PR fixes**
45927, for now catching up the implementation of v1.6

**Special notes for your reviewer**:
The code change includes 4 parts, put them together as we discussed in kubernetes#46089

1. Update go-winio package to a newer version
  'go-winio' package is used by docker client.
  This change is to bring the support for Go v1.8, specifically included in the PR: microsoft/go-winio#48 
Otherwise it will produce a lot of error like in: fsouza/go-dockerclient#648 

2. Add os dependent getSecurityOpts helper method. 
seccomp not supported on windows
  Corresponding code in v1.6: https://github.com/kubernetes/kubernetes/blob/v1.6.4/pkg/kubelet/dockertools/docker_manager_windows.go#L78

3. Add updateCreateConfig.
Allow user specified network mode setting. This is to be compatible with what kube-proxy package does on Windows. 
  Also, there is a Linux section in both sandbox config and container config： LinuxPodSandboxConfig， LinuxContainerConfig.
And that section later goes to Config and HostConfig section under docker container createConfig. Ideally hostconfig section should be dependent on host os, while config should depend on container image os.
  To simplify the case, here it assumes that windows host supports windows type container image only. It needs to be updated when kubernetes is to support windows host running linux container image or the like.
  Corresponding code in v1.6: https://github.com/kubernetes/kubernetes/blob/v1.6.4/pkg/kubelet/dockertools/docker_manager_windows.go#L57

4. Add podIpCache in dockershim. 
  For v1.6 windows implementation, it still does not use sandbox, thus only allow single container to be exposed.
  Here added a cache for saving container IP, to get adapted to the new CRI api.
Corresponding code in v1.6:
No sandbox: https://github.com/kubernetes/kubernetes/blob/v1.6.4/pkg/kubelet/dockertools/docker_manager_windows.go#L66
Use container id as pod ip: https://github.com/kubernetes/kubernetes/blob/v1.6.4/pkg/kubelet/dockertools/docker_manager.go#L2727

**Release note**:

pkg/kubelet/dockershim/docker_container.go

@@ -132,47 +132,12 @@ func (ds *dockerService) CreateContainer(podSandboxID string, config *runtimeapi
 			StdinOnce: config.StdinOnce,
 			Tty:       config.Tty,
 		},
+		HostConfig: &dockercontainer.HostConfig{
+			Binds: generateMountBindings(config.GetMounts()),
+		},
 	}
 
-	// Fill the HostConfig.
-	hc := &dockercontainer.HostConfig{
-		Binds: generateMountBindings(config.GetMounts()),
-	}
-
-	// Apply Linux-specific options if applicable.
-	if lc := config.GetLinux(); lc != nil {
-		// TODO: Check if the units are correct.
-		// TODO: Can we assume the defaults are sane?
-		rOpts := lc.GetResources()
-		if rOpts != nil {
-			hc.Resources = dockercontainer.Resources{
-				Memory:     rOpts.MemoryLimitInBytes,
-				MemorySwap: DefaultMemorySwap(),
-				CPUShares:  rOpts.CpuShares,
-				CPUQuota:   rOpts.CpuQuota,
-				CPUPeriod:  rOpts.CpuPeriod,
-			}
-			hc.OomScoreAdj = int(rOpts.OomScoreAdj)
-		}
-		// Note: ShmSize is handled in kube_docker_client.go
-
-		// Apply security context.
-		if err = applyContainerSecurityContext(lc, podSandboxID, createConfig.Config, hc, securityOptSep); err != nil {
-			return "", fmt.Errorf("failed to apply container security context for container %q: %v", config.Metadata.Name, err)
-		}
-		modifyPIDNamespaceOverrides(ds.disableSharedPID, apiVersion, hc)
-	}
-
-	// Apply cgroupsParent derived from the sandbox config.
-	if lc := sandboxConfig.GetLinux(); lc != nil {
-		// Apply Cgroup options.
-		cgroupParent, err := ds.GenerateExpectedCgroupParent(lc.CgroupParent)
-		if err != nil {
-			return "", fmt.Errorf("failed to generate cgroup parent in expected syntax for container %q: %v", config.Metadata.Name, err)
-		}
-		hc.CgroupParent = cgroupParent
-	}
-
+	hc := createConfig.HostConfig
 	// Set devices for container.
 	devices := make([]dockercontainer.DeviceMapping, len(config.Devices))
 	for i, device := range config.Devices {
@@ -183,15 +148,15 @@ func (ds *dockerService) CreateContainer(podSandboxID string, config *runtimeapi
 		}
 	}
 	hc.Resources.Devices = devices
+	ds.updateCreateConfig(&createConfig, config, sandboxConfig, podSandboxID, securityOptSep, apiVersion)
 
-	// Apply seccomp options.
-	seccompSecurityOpts, err := getSeccompSecurityOpts(config.Metadata.Name, sandboxConfig, ds.seccompProfileRoot, securityOptSep)
+	securityOpts, err := ds.getSecurityOpts(config.Metadata.Name, sandboxConfig, securityOptSep)
 	if err != nil {
-		return "", fmt.Errorf("failed to generate seccomp security options for container %q: %v", config.Metadata.Name, err)
+		return "", fmt.Errorf("failed to generate security options for container %q: %v", config.Metadata.Name, err)
 	}
-	hc.SecurityOpt = append(hc.SecurityOpt, seccompSecurityOpts...)
 
-	createConfig.HostConfig = hc
+	hc.SecurityOpt = append(hc.SecurityOpt, securityOpts...)
+
 	createResp, err := ds.client.CreateContainer(createConfig)
 	if err != nil {
 		createResp, err = recoverFromCreationConflictIfNeeded(ds.client, createConfig, err)

pkg/kubelet/dockershim/docker_sandbox.go

@@ -317,9 +317,15 @@ func (ds *dockerService) PodSandboxStatus(podSandboxID string) (*runtimeapi.PodS
 	if r.State.Running {
 		state = runtimeapi.PodSandboxState_SANDBOX_READY
 	}
-	IP, err := ds.getIP(r)
-	if err != nil {
-		return nil, err
+
+	var IP string
+	// TODO: Remove this when sandbox is available on windows
+	// This is a workaround for windows, where sandbox is not in use, and pod IP is determined through containers belonging to the Pod.
+	if IP = ds.determinePodIPBySandboxID(podSandboxID); IP == "" {
+		IP, err = ds.getIP(r)
+		if err != nil {
+			return nil, err
+		}
 	}
 	network := &runtimeapi.PodSandboxNetworkStatus{Ip: IP}
 	hostNetwork := sharesHostNetwork(r)
@@ -537,7 +543,7 @@ func (ds *dockerService) makeSandboxDockerConfig(c *runtimeapi.PodSandboxConfig,
 	}
 
 	// Set security options.
-	securityOpts, err := getSeccompSecurityOpts(sandboxContainerName, c, ds.seccompProfileRoot, securityOptSep)
+	securityOpts, err := ds.getSecurityOpts(sandboxContainerName, c, securityOptSep)
 	if err != nil {
 		return nil, fmt.Errorf("failed to generate sandbox security options for sandbox %q: %v", c.Metadata.Name, err)
 	}

pkg/kubelet/dockershim/helpers_linux.go

@@ -18,6 +18,71 @@ limitations under the License.
 
 package dockershim
 
+import (
+	"fmt"
+
+	"github.com/blang/semver"
+	dockertypes "github.com/docker/engine-api/types"
+	dockercontainer "github.com/docker/engine-api/types/container"
+	runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1"
+)
+
 func DefaultMemorySwap() int64 {
 	return 0
 }
+
+func (ds *dockerService) getSecurityOpts(containerName string, sandboxConfig *runtimeapi.PodSandboxConfig, separator rune) ([]string, error) {
+	// Apply seccomp options.
+	seccompSecurityOpts, err := getSeccompSecurityOpts(containerName, sandboxConfig, ds.seccompProfileRoot, separator)
+	if err != nil {
+		return nil, fmt.Errorf("failed to generate seccomp security options for container %q: %v", containerName, err)
+	}
+
+	return seccompSecurityOpts, nil
+}
+
+func (ds *dockerService) updateCreateConfig(
+	createConfig *dockertypes.ContainerCreateConfig,
+	config *runtimeapi.ContainerConfig,
+	sandboxConfig *runtimeapi.PodSandboxConfig,
+	podSandboxID string, securityOptSep rune, apiVersion *semver.Version) error {
+	// Apply Linux-specific options if applicable.
+	if lc := config.GetLinux(); lc != nil {
+		// TODO: Check if the units are correct.
+		// TODO: Can we assume the defaults are sane?
+		rOpts := lc.GetResources()
+		if rOpts != nil {
+			createConfig.HostConfig.Resources = dockercontainer.Resources{
+				Memory:     rOpts.MemoryLimitInBytes,
+				MemorySwap: DefaultMemorySwap(),
+				CPUShares:  rOpts.CpuShares,
+				CPUQuota:   rOpts.CpuQuota,
+				CPUPeriod:  rOpts.CpuPeriod,
+			}
+			createConfig.HostConfig.OomScoreAdj = int(rOpts.OomScoreAdj)
+		}
+		// Note: ShmSize is handled in kube_docker_client.go
+
+		// Apply security context.
+		if err := applyContainerSecurityContext(lc, podSandboxID, createConfig.Config, createConfig.HostConfig, securityOptSep); err != nil {
+			return fmt.Errorf("failed to apply container security context for container %q: %v", config.Metadata.Name, err)
+		}
+		modifyPIDNamespaceOverrides(ds.disableSharedPID, apiVersion, createConfig.HostConfig)
+	}
+
+	// Apply cgroupsParent derived from the sandbox config.
+	if lc := sandboxConfig.GetLinux(); lc != nil {
+		// Apply Cgroup options.
+		cgroupParent, err := ds.GenerateExpectedCgroupParent(lc.CgroupParent)
+		if err != nil {
+			return fmt.Errorf("failed to generate cgroup parent in expected syntax for container %q: %v", config.Metadata.Name, err)
+		}
+		createConfig.HostConfig.CgroupParent = cgroupParent
+	}
+
+	return nil
+}
+
+func (ds *dockerService) determinePodIPBySandboxID(uid string) string {
+	return ""
+}

pkg/kubelet/dockershim/helpers_unsupported.go

@@ -18,6 +18,32 @@ limitations under the License.
 
 package dockershim
 
+import (
+	"github.com/blang/semver"
+	dockertypes "github.com/docker/engine-api/types"
+	"github.com/golang/glog"
+	runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1"
+)
+
 func DefaultMemorySwap() int64 {
 	return -1
 }
+
+func (ds *dockerService) getSecurityOpts(containerName string, sandboxConfig *runtimeapi.PodSandboxConfig, separator rune) ([]string, error) {
+	glog.Warningf("getSecurityOpts is unsupported in this build")
+	return nil, nil
+}
+
+func (ds *dockerService) updateCreateConfig(
+	createConfig *dockertypes.ContainerCreateConfig,
+	config *runtimeapi.ContainerConfig,
+	sandboxConfig *runtimeapi.PodSandboxConfig,
+	podSandboxID string, securityOptSep rune, apiVersion *semver.Version) error {
+	glog.Warningf("updateCreateConfig is unsupported in this build")
+	return nil
+}
+
+func (ds *dockerService) determinePodIPBySandboxID(uid string) string {
+	glog.Warningf("determinePodIPBySandboxID is unsupported in this build")
+	return ""
+}

pkg/kubelet/dockershim/helpers_windows.go

@@ -18,6 +18,84 @@ limitations under the License.
 
 package dockershim
 
+import (
+	"os"
+
+	"github.com/blang/semver"
+	dockertypes "github.com/docker/engine-api/types"
+	dockercontainer "github.com/docker/engine-api/types/container"
+	dockerfilters "github.com/docker/engine-api/types/filters"
+	"github.com/golang/glog"
+	"k8s.io/kubernetes/pkg/api/v1"
+	runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1"
+)
+
 func DefaultMemorySwap() int64 {
 	return 0
 }
+
+func (ds *dockerService) getSecurityOpts(containerName string, sandboxConfig *runtimeapi.PodSandboxConfig, separator rune) ([]string, error) {
+	hasSeccompSetting := false
+	annotations := sandboxConfig.GetAnnotations()
+	if _, ok := annotations[v1.SeccompContainerAnnotationKeyPrefix+containerName]; !ok {
+		_, hasSeccompSetting = annotations[v1.SeccompPodAnnotationKey]
+	} else {
+		hasSeccompSetting = true
+	}
+
+	if hasSeccompSetting {
+		glog.Warningf("seccomp annotations found, but it is not supported on windows")
+	}
+
+	return nil, nil
+}
+
+func (ds *dockerService) updateCreateConfig(
+	createConfig *dockertypes.ContainerCreateConfig,
+	config *runtimeapi.ContainerConfig,
+	sandboxConfig *runtimeapi.PodSandboxConfig,
+	podSandboxID string, securityOptSep rune, apiVersion *semver.Version) error {
+	if networkMode := os.Getenv("CONTAINER_NETWORK"); networkMode != "" {
+		createConfig.HostConfig.NetworkMode = dockercontainer.NetworkMode(networkMode)
+	}
+
+	return nil
+}
+
+func (ds *dockerService) determinePodIPBySandboxID(sandboxID string) string {
+	opts := dockertypes.ContainerListOptions{
+		All:    true,
+		Filter: dockerfilters.NewArgs(),
+	}
+
+	f := newDockerFilter(&opts.Filter)
+	f.AddLabel(containerTypeLabelKey, containerTypeLabelContainer)
+	f.AddLabel(sandboxIDLabelKey, sandboxID)
+	containers, err := ds.client.ListContainers(opts)
+	if err != nil {
+		return ""
+	}
+
+	for _, c := range containers {
+		r, err := ds.client.InspectContainer(c.ID)
+		if err != nil {
+			continue
+		}
+		if containerIP := getContainerIP(r); containerIP != "" {
+			return containerIP
+		}
+	}
+
+	return ""
+}
+
+func getContainerIP(container *dockertypes.ContainerJSON) string {
+	if container.NetworkSettings != nil {
+		for _, network := range container.NetworkSettings.Networks {
+			if network.IPAddress != "" {
+				return network.IPAddress
+			}
+		}
+	}
+	return ""
+}

vendor/BUILD

@@ -337,6 +337,7 @@ filegroup(
         "//vendor/golang.org/x/net/websocket:all-srcs",
         "//vendor/golang.org/x/oauth2:all-srcs",
         "//vendor/golang.org/x/sys/unix:all-srcs",
+        "//vendor/golang.org/x/sys/windows:all-srcs",
         "//vendor/golang.org/x/text/cases:all-srcs",
         "//vendor/golang.org/x/text/encoding:all-srcs",
         "//vendor/golang.org/x/text/internal/tag:all-srcs",