diff --git a/kubeadm/flannel/Dockerfile b/kubeadm/flannel/Dockerfile new file mode 100644 index 00000000..9525bd52 --- /dev/null +++ b/kubeadm/flannel/Dockerfile @@ -0,0 +1,23 @@ +ARG servercoreTag="ltsc2019" +ARG cniVersion="0.8.5" + +FROM mcr.microsoft.com/windows/servercore:${servercoreTag} +SHELL ["powershell", "-NoLogo", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"] + +ARG cniVersion + +# Stuck on a prerelease flannel until https://github.com/coreos/flannel/issues/1231 is resolved +RUN mkdir -force C:\k\flannel; \ + pushd C:\k\flannel; \ + curl.exe -LO https://github.com/benmoss/flannel/releases/download/v0.12.0-rc1/flanneld.exe + +RUN mkdir C:\cni; \ + pushd C:\cni; \ + curl.exe -Lo cni.tgz https://github.com/containernetworking/plugins/releases/download/v${env:cniVersion}/cni-plugins-windows-amd64-v${env:cniVersion}.tgz; \ + tar -xf cni.tgz; \ + rm cni.tgz + +RUN mkdir C:\utils; \ + curl.exe -Lo C:\utils\wins.exe https://github.com/rancher/wins/releases/download/v0.0.4/wins.exe; \ + curl.exe -Lo C:\utils\yq.exe https://github.com/mikefarah/yq/releases/download/2.4.1/yq_windows_amd64.exe; \ + "[Environment]::SetEnvironmentVariable('PATH', $env:PATH + ';C:\utils', [EnvironmentVariableTarget]::Machine)" diff --git a/kubeadm/flannel/flannel-host-gw.yml b/kubeadm/flannel/flannel-host-gw.yml new file mode 100644 index 00000000..1a7221dd --- /dev/null +++ b/kubeadm/flannel/flannel-host-gw.yml @@ -0,0 +1,171 @@ +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: kube-flannel-windows-cfg + namespace: kube-system + labels: + tier: node + app: flannel +data: + run.ps1: | + $ErrorActionPreference = "Stop"; + + mkdir -force /host/etc/cni/net.d + mkdir -force /host/etc/kube-flannel + mkdir -force /host/opt/cni/bin + mkdir -force /host/k/flannel + mkdir -force /host/k/flannel/var/run/secrets/kubernetes.io/serviceaccount + + $cniJson = get-content /etc/kube-flannel-windows/cni-conf.json | ConvertFrom-Json + $serviceSubnet = yq r /etc/kubeadm-config/ClusterConfiguration networking.serviceSubnet + $podSubnet = yq r /etc/kubeadm-config/ClusterConfiguration networking.podSubnet + $networkJson = wins cli net get | convertfrom-json + + $cniJson.delegate.policies[0].Value.ExceptionList = $serviceSubnet, $podSubnet, $networkJson.SubnetCIDR + $cniJson.delegate.policies[1].Value.DestinationPrefix = $serviceSubnet + $cniJson.delegate.policies[2].Value.DestinationPrefix = $networkJson.AddressCIDR + Set-Content -Path /host/etc/cni/net.d/10-flannel.conf ($cniJson | ConvertTo-Json -depth 100) + + cp -force /etc/kube-flannel/net-conf.json /host/etc/kube-flannel + cp -force -recurse /cni/* /host/opt/cni/bin + cp -force /k/flannel/flanneld.exe /host/k/flannel/flanneld.exe + cp -force /kube-proxy/kubeconfig.conf /host/k/flannel/kubeconfig.yml + cp -force /var/run/secrets/kubernetes.io/serviceaccount/* /host/k/flannel/var/run/secrets/kubernetes.io/serviceaccount/ + wins cli process run --path /k/flannel/flanneld.exe --args "--kube-subnet-mgr --kubeconfig-file /k/flannel/kubeconfig.yml" --envs "POD_NAME=$env:POD_NAME POD_NAMESPACE=$env:POD_NAMESPACE" + cni-conf.json: | + { + "name": "cbr0", + "cniVersion": "0.3.0", + "type": "flannel", + "capabilities": { + "dns": true + }, + "delegate": { + "type": "win-bridge", + "hairpinMode": true, + "isDefaultGateway": true, + "policies": [ + { + "Name": "EndpointPolicy", + "Value": { + "Type": "OutBoundNAT", + "ExceptionList": [] + } + }, + { + "Name": "EndpointPolicy", + "Value": { + "Type": "ROUTE", + "DestinationPrefix": "", + "NeedEncap": true + } + }, + { + "Name": "EndpointPolicy", + "Value": { + "Type": "ROUTE", + "DestinationPrefix": "", + "NeedEncap": true + } + } + ] + } + } +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: kube-flannel-ds-windows-amd64 + labels: + tier: node + app: flannel + namespace: kube-system +spec: + selector: + matchLabels: + app: flannel + template: + metadata: + labels: + tier: node + app: flannel + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: beta.kubernetes.io/os + operator: In + values: + - windows + - key: beta.kubernetes.io/arch + operator: In + values: + - amd64 + hostNetwork: true + serviceAccountName: flannel + tolerations: + - operator: Exists + effect: NoSchedule + containers: + - name: kube-flannel + image: sigwindowstools/flannel:0.12.0-rc1 + command: + - powershell + args: + - -file + - /etc/kube-flannel-windows/run.ps1 + volumeMounts: + - name: wins + mountPath: \\.\pipe\rancher_wins + - name: host + mountPath: /host + - name: kube-proxy + mountPath: /kube-proxy + - name: cni + mountPath: /etc/cni/net.d + - name: flannel-cfg + mountPath: /etc/kube-flannel/ + - name: flannel-windows-cfg + mountPath: /etc/kube-flannel-windows/ + - name: kubeadm-config + mountPath: /etc/kubeadm-config/ + env: + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + volumes: + - name: opt + hostPath: + path: /opt + - name: host + hostPath: + path: / + - name: cni + hostPath: + path: /etc + - name: flannel-cfg + configMap: + name: kube-flannel-cfg + - name: flannel-windows-cfg + configMap: + name: kube-flannel-windows-cfg + - name: kube-proxy + configMap: + name: kube-proxy + - name: kubeadm-config + configMap: + name: kubeadm-config + - name: wins + hostPath: + path: \\.\pipe\rancher_wins + type: null diff --git a/kubeadm/flannel/flannel-overlay.yml b/kubeadm/flannel/flannel-overlay.yml new file mode 100644 index 00000000..e5f654d3 --- /dev/null +++ b/kubeadm/flannel/flannel-overlay.yml @@ -0,0 +1,160 @@ +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: kube-flannel-windows-cfg + namespace: kube-system + labels: + tier: node + app: flannel +data: + run.ps1: | + $ErrorActionPreference = "Stop"; + + mkdir -force /host/etc/cni/net.d + mkdir -force /host/etc/kube-flannel + mkdir -force /host/opt/cni/bin + mkdir -force /host/k/flannel + mkdir -force /host/k/flannel/var/run/secrets/kubernetes.io/serviceaccount + + $cniJson = get-content /etc/kube-flannel-windows/cni-conf.json | ConvertFrom-Json + $serviceSubnet = yq r /etc/kubeadm-config/ClusterConfiguration networking.serviceSubnet + $podSubnet = yq r /etc/kubeadm-config/ClusterConfiguration networking.podSubnet + $networkJson = wins cli net get | convertfrom-json + + $cniJson.delegate.policies[0].Value.ExceptionList = $serviceSubnet, $podSubnet + $cniJson.delegate.policies[1].Value.DestinationPrefix = $serviceSubnet + Set-Content -Path /host/etc/cni/net.d/10-flannel.conf ($cniJson | ConvertTo-Json -depth 100) + + cp -force /etc/kube-flannel/net-conf.json /host/etc/kube-flannel + cp -force -recurse /cni/* /host/opt/cni/bin + cp -force /k/flannel/flanneld.exe /host/k/flannel/flanneld.exe + cp -force /kube-proxy/kubeconfig.conf /host/k/flannel/kubeconfig.yml + cp -force /var/run/secrets/kubernetes.io/serviceaccount/* /host/k/flannel/var/run/secrets/kubernetes.io/serviceaccount/ + wins cli process run --path /k/flannel/flanneld.exe --args "--kube-subnet-mgr --kubeconfig-file /k/flannel/kubeconfig.yml" --envs "POD_NAME=$env:POD_NAME POD_NAMESPACE=$env:POD_NAMESPACE" + cni-conf.json: | + { + "name": "flannel.4096", + "cniVersion": "0.3.0", + "type": "flannel", + "capabilities": { + "dns": true + }, + "delegate": { + "type": "win-overlay", + "policies": [ + { + "Name": "EndpointPolicy", + "Value": { + "Type": "OutBoundNAT", + "ExceptionList": [] + } + }, + { + "Name": "EndpointPolicy", + "Value": { + "Type": "ROUTE", + "DestinationPrefix": "", + "NeedEncap": true + } + } + ] + } + } +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: kube-flannel-ds-windows-amd64 + labels: + tier: node + app: flannel + namespace: kube-system +spec: + selector: + matchLabels: + app: flannel + template: + metadata: + labels: + tier: node + app: flannel + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: beta.kubernetes.io/os + operator: In + values: + - windows + - key: beta.kubernetes.io/arch + operator: In + values: + - amd64 + hostNetwork: true + serviceAccountName: flannel + tolerations: + - operator: Exists + effect: NoSchedule + containers: + - name: kube-flannel + image: sigwindowstools/flannel:0.12.0-rc1 + command: + - powershell + args: + - -file + - /etc/kube-flannel-windows/run.ps1 + volumeMounts: + - name: wins + mountPath: \\.\pipe\rancher_wins + - name: host + mountPath: /host + - name: kube-proxy + mountPath: /kube-proxy + - name: cni + mountPath: /etc/cni/net.d + - name: flannel-cfg + mountPath: /etc/kube-flannel/ + - name: flannel-windows-cfg + mountPath: /etc/kube-flannel-windows/ + - name: kubeadm-config + mountPath: /etc/kubeadm-config/ + env: + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + volumes: + - name: opt + hostPath: + path: /opt + - name: host + hostPath: + path: / + - name: cni + hostPath: + path: /etc + - name: flannel-cfg + configMap: + name: kube-flannel-cfg + - name: flannel-windows-cfg + configMap: + name: kube-flannel-windows-cfg + - name: kube-proxy + configMap: + name: kube-proxy + - name: kubeadm-config + configMap: + name: kubeadm-config + - name: wins + hostPath: + path: \\.\pipe\rancher_wins + type: null diff --git a/kubeadm/kube-proxy/Dockerfile b/kubeadm/kube-proxy/Dockerfile new file mode 100644 index 00000000..341199da --- /dev/null +++ b/kubeadm/kube-proxy/Dockerfile @@ -0,0 +1,16 @@ +ARG k8sVersion="v1.17.3" +ARG servercoreTag="ltsc2019" + +FROM mcr.microsoft.com/windows/servercore:${servercoreTag} +SHELL ["powershell", "-NoLogo", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"] + +ARG k8sVersion + +RUN mkdir -force C:\k\kube-proxy; \ + pushd C:\k\kube-proxy; \ + curl.exe -LO https://dl.k8s.io/${env:k8sVersion}/bin/windows/amd64/kube-proxy.exe + +RUN mkdir C:\utils; \ + curl.exe -Lo C:\utils\wins.exe https://github.com/rancher/wins/releases/download/v0.0.4/wins.exe; \ + curl.exe -Lo C:\utils\yq.exe https://github.com/mikefarah/yq/releases/download/2.4.1/yq_windows_amd64.exe; \ + "[Environment]::SetEnvironmentVariable('PATH', $env:PATH + ';C:\utils', [EnvironmentVariableTarget]::Machine)" diff --git a/kubeadm/kube-proxy/kube-proxy.yml b/kubeadm/kube-proxy/kube-proxy.yml new file mode 100644 index 00000000..171ab5a8 --- /dev/null +++ b/kubeadm/kube-proxy/kube-proxy.yml @@ -0,0 +1,94 @@ +apiVersion: v1 +data: + run-script.ps1: |- + $ErrorActionPreference = "Stop"; + mkdir -force /host/var/lib/kube-proxy/var/run/secrets/kubernetes.io/serviceaccount + mkdir -force /host/k/kube-proxy + + cp -force /k/kube-proxy/* /host/k/kube-proxy + cp -force /var/lib/kube-proxy/* /host/var/lib/kube-proxy + cp -force /var/run/secrets/kubernetes.io/serviceaccount/* /host/var/lib/kube-proxy/var/run/secrets/kubernetes.io/serviceaccount #FIXME? + + $networkName = (Get-Content /host/etc/cni/net.d/* | ConvertFrom-Json).name + $sourceVip = ($env:POD_IP -split "\.")[0..2] + 0 -join "." + yq w -i /host/var/lib/kube-proxy/config.conf winkernel.sourceVip $sourceVip + yq w -i /host/var/lib/kube-proxy/config.conf winkernel.networkName $networkName + yq w -i /host/var/lib/kube-proxy/config.conf featureGates.WinOverlay true + yq w -i /host/var/lib/kube-proxy/config.conf mode "kernelspace" + wins cli process run --path /k/kube-proxy/kube-proxy.exe --args "--v=6 --config=/var/lib/kube-proxy/config.conf --hostname-override=$env:NODE_NAME --feature-gates=WinOverlay=true" + +kind: ConfigMap +apiVersion: v1 +metadata: + labels: + app: kube-proxy + name: kube-proxy-windows + namespace: kube-system +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + k8s-app: kube-proxy + name: kube-proxy-windows + namespace: kube-system +spec: + selector: + matchLabels: + k8s-app: kube-proxy-windows + template: + metadata: + labels: + k8s-app: kube-proxy-windows + spec: + serviceAccountName: kube-proxy + containers: + - command: + - powershell + args: + - -file + - /var/lib/kube-proxy-windows/run-script.ps1 + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: sigwindowstools/kube-proxy:VERSION + name: kube-proxy + volumeMounts: + - name: host + mountPath: /host + - name: wins + mountPath: \\.\pipe\rancher_wins + - mountPath: /var/lib/kube-proxy + name: kube-proxy + - mountPath: /var/lib/kube-proxy-windows + name: kube-proxy-windows + nodeSelector: + beta.kubernetes.io/os: windows + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - operator: Exists + volumes: + - configMap: + defaultMode: 420 + name: kube-proxy-windows + name: kube-proxy-windows + - configMap: + name: kube-proxy + name: kube-proxy + - hostPath: + path: / + name: host + - name: wins + hostPath: + path: \\.\pipe\rancher_wins + type: null + updateStrategy: + type: RollingUpdate diff --git a/kubeadm/kube-proxy/publish.sh b/kubeadm/kube-proxy/publish.sh new file mode 100755 index 00000000..41aaccd7 --- /dev/null +++ b/kubeadm/kube-proxy/publish.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +set -e + +INITIAL_MAJOR=1 +INITIAL_MINOR=17 +INITIAL_PATCH=0 + +dir=$(cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P) + +# https://github.com/google/go-containerregistry/tree/master/cmd/crane +versions=$(crane ls k8s.gcr.io/kube-proxy 2>/dev/null) +for version in $versions; do + if [[ $version =~ alpha|beta|rc ]]; then + continue + fi + major=`echo $version | cut -d. -f1 | tr -d v` + minor=`echo $version | cut -d. -f2` + patch=`echo $version | cut -d. -f3` + if [ $major -lt $INITIAL_MAJOR ]; then + continue + fi + if [ $minor -lt $INITIAL_MINOR ]; then + continue + fi + + echo "building $major.$minor.$patch" + + docker build --pull --build-arg k8sVersion="$version" --tag sigwindowstools/kube-proxy:$version $dir +done + +docker push sigwindowstools/kube-proxy diff --git a/kubeadm/scripts/PrepareNode.ps1 b/kubeadm/scripts/PrepareNode.ps1 new file mode 100644 index 00000000..054af411 --- /dev/null +++ b/kubeadm/scripts/PrepareNode.ps1 @@ -0,0 +1,96 @@ +<# +.SYNOPSIS +Assists with preparing a Windows VM prior to calling kubeadm join + +.DESCRIPTION +This script assists with joining a Windows node to a cluster. +- Downloads Kubernetes binaries (kubelet, kubeadm) at the version specified +- Registers wins as a service in order to run kube-proxy and cni as DaemonSets. +- Registers kubelet as an nssm service. More info on nssm: https://nssm.cc/ + +.PARAMETER KubernetesVersion +Kubernetes version to download and use + +.EXAMPLE +PS> .\PrepareNode.ps1 -KubernetesVersion v1.17.0 + +#> + +Param( + [parameter(Mandatory = $true, HelpMessage="Kubernetes version to use")] + [string] $KubernetesVersion +) +$ErrorActionPreference = 'Stop' + +function DownloadFile($destination, $source) { + Write-Host("Downloading $source to $destination") + curl.exe --silent --fail -Lo $destination $source + + if (!$?) { + Write-Error "Download $source failed" + exit 1 + } +} + +if (!$KubernetesVersion.StartsWith("v")) { + $KubernetesVersion = "v" + $KubernetesVersion +} +Write-Host "Using Kubernetes version: $KubernetesVersion" +$global:Powershell = (Get-Command powershell).Source +$global:PowershellArgs = "-ExecutionPolicy Bypass -NoProfile" +$global:KubernetesPath = "$env:SystemDrive\k" +$global:StartKubeletScript = "$global:KubernetesPath\StartKubelet.ps1" +$global:NssmInstallDirectory = "$env:ProgramFiles\nssm" +$kubeletBinPath = "$global:KubernetesPath\kubelet.exe" + +mkdir -force "$global:KubernetesPath" +$env:Path += ";$global:KubernetesPath" +[Environment]::SetEnvironmentVariable("Path", $env:Path, [System.EnvironmentVariableTarget]::Machine) + +DownloadFile $kubeletBinPath https://dl.k8s.io/$KubernetesVersion/bin/windows/amd64/kubelet.exe +DownloadFile "$global:KubernetesPath\kubeadm.exe" https://dl.k8s.io/$KubernetesVersion/bin/windows/amd64/kubeadm.exe +DownloadFile "$global:KubernetesPath\wins.exe" https://github.com/rancher/wins/releases/download/v0.0.4/wins.exe + +# Create host network to allow kubelet to schedule hostNetwork pods +Write-Host "Creating Docker host network" +docker network create -d nat host + +Write-Host "Registering wins service" +wins.exe srv app run --register +start-service rancher-wins + +mkdir -force C:\var\log\kubelet +mkdir -force C:\var\lib\kubelet\etc\kubernetes +mkdir -force C:\etc\kubernetes\pki +New-Item -path C:\var\lib\kubelet\etc\kubernetes\pki -type SymbolicLink -value C:\etc\kubernetes\pki\ + +$StartKubeletFileContent = '$FileContent = Get-Content -Path "/var/lib/kubelet/kubeadm-flags.env" +$global:KubeletArgs = $FileContent.Trim("KUBELET_KUBEADM_ARGS=`"") + +$cmd = "C:\k\kubelet.exe $global:KubeletArgs --cert-dir=$env:SYSTEMDRIVE\var\lib\kubelet\pki --config=/var/lib/kubelet/config.yaml --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf --hostname-override=$(hostname) --pod-infra-container-image=`"mcr.microsoft.com/k8s/core/pause:1.2.0`" --enable-debugging-handlers --cgroups-per-qos=false --enforce-node-allocatable=`"`" --network-plugin=cni --resolv-conf=`"`" --log-dir=/var/log/kubelet --logtostderr=false --image-pull-progress-deadline=20m" + +Invoke-Expression $cmd' +Set-Content -Path $global:StartKubeletScript -Value $StartKubeletFileContent + +Write-Host "Installing nssm" +$arch = "win32" +if ([Environment]::Is64BitOperatingSystem) { + $arch = "win64" +} + +mkdir -Force $global:NssmInstallDirectory +DownloadFile nssm.zip https://k8stestinfrabinaries.blob.core.windows.net/nssm-mirror/nssm-2.24.zip +tar C $global:NssmInstallDirectory -xvf .\nssm.zip --strip-components 2 */$arch/*.exe +Remove-Item -Force .\nssm.zip + +$env:path += ";$global:NssmInstallDirectory" +$newPath = "$global:NssmInstallDirectory;" + +[Environment]::GetEnvironmentVariable("PATH", [EnvironmentVariableTarget]::Machine) + +[Environment]::SetEnvironmentVariable("PATH", $newPath, [EnvironmentVariableTarget]::Machine) + +Write-Host "Registering kubelet service" +nssm install kubelet $global:Powershell $global:PowershellArgs $global:StartKubeletScript +nssm set kubelet DependOnService docker + +New-NetFirewallRule -Name kubelet -DisplayName 'kubelet' -Enabled True -Direction Inbound -Protocol TCP -Action Allow -LocalPort 10250