From ea35456c0793d6bab43fc0f8ab449c422712ef6a Mon Sep 17 00:00:00 2001
From: Gab Satchi <gsatchithanantham@pivotal.io>
Date: Fri, 7 Feb 2020 12:32:58 -0500
Subject: [PATCH 01/18] Adds flannel (overlay + l2bridge) and kube-proxy yml

---
 kubeadm/flannel/flannel-host-gw.yml | 171 ++++++++++++++++++++++++++++
 kubeadm/flannel/flannel-overlay.yml | 160 ++++++++++++++++++++++++++
 kubeadm/kube-proxy/kube-proxy.yml   |  94 +++++++++++++++
 3 files changed, 425 insertions(+)
 create mode 100644 kubeadm/flannel/flannel-host-gw.yml
 create mode 100644 kubeadm/flannel/flannel-overlay.yml
 create mode 100644 kubeadm/kube-proxy/kube-proxy.yml

diff --git a/kubeadm/flannel/flannel-host-gw.yml b/kubeadm/flannel/flannel-host-gw.yml
new file mode 100644
index 00000000..3a575043
--- /dev/null
+++ b/kubeadm/flannel/flannel-host-gw.yml
@@ -0,0 +1,171 @@
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: kube-flannel-windows-cfg
+  namespace: kube-system
+  labels:
+    tier: node
+    app: flannel
+data:
+  run.ps1: |
+    $ErrorActionPreference = "Stop";
+
+    mkdir -force /host/etc/cni/net.d
+    mkdir -force /host/etc/kube-flannel
+    mkdir -force /host/opt/cni/bin
+    mkdir -force /host/flannel
+    mkdir -force /host/flannel/var/run/secrets/kubernetes.io/serviceaccount
+
+    $cniJson = get-content /etc/kube-flannel-windows/cni-conf.json | ConvertFrom-Json
+    $serviceSubnet = yq r /etc/kubeadm-config/ClusterConfiguration networking.serviceSubnet
+    $podSubnet = yq r /etc/kubeadm-config/ClusterConfiguration networking.podSubnet
+    $networkJson = wins cli net get | convertfrom-json
+
+    $cniJson.delegate.policies[0].Value.ExceptionList = $serviceSubnet, $podSubnet, $networkJson.SubnetCIDR
+    $cniJson.delegate.policies[1].Value.DestinationPrefix = $serviceSubnet
+    $cniJson.delegate.policies[2].Value.DestinationPrefix = $networkJson.AddressCIDR
+    Set-Content -Path /host/etc/cni/net.d/10-flannel.conf ($cniJson | ConvertTo-Json -depth 100)
+
+    cp -force /etc/kube-flannel/net-conf.json /host/etc/kube-flannel
+    cp -force -recurse /cni/* /host/opt/cni/bin
+    cp -force /flannel/flanneld.exe /host/flannel/flanneld.exe
+    cp -force /kube-proxy/kubeconfig.conf /host/flannel/kubeconfig.yml
+    cp -force /var/run/secrets/kubernetes.io/serviceaccount/* /host/flannel/var/run/secrets/kubernetes.io/serviceaccount/
+    wins cli process run --path C:\flannel\flanneld.exe --args "--kube-subnet-mgr --kubeconfig-file /flannel/kubeconfig.yml" --envs "POD_NAME=$env:POD_NAME POD_NAMESPACE=$env:POD_NAMESPACE"
+  cni-conf.json: |
+    {
+      "name": "cbr0",
+      "cniVersion": "0.3.0",
+      "type": "flannel",
+      "capabilities": {
+        "dns": true
+      },
+      "delegate": {
+        "type": "win-bridge",
+        "hairpinMode": true,
+        "isDefaultGateway": true,
+        "policies": [
+          {
+            "Name": "EndpointPolicy",
+            "Value": {
+              "Type": "OutBoundNAT",
+              "ExceptionList": []
+            }
+          },
+          {
+            "Name": "EndpointPolicy",
+            "Value": {
+              "Type": "ROUTE",
+              "DestinationPrefix": "",
+              "NeedEncap": true
+            }
+          },
+          {
+            "Name": "EndpointPolicy",
+            "Value": {
+              "Type": "ROUTE",
+              "DestinationPrefix": "",
+              "NeedEncap": true
+            }
+          }
+        ]
+      }
+    }
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: kube-flannel-ds-windows-amd64
+  labels:
+    tier: node
+    app: flannel
+  namespace: kube-system
+spec:
+  selector:
+    matchLabels:
+      app: flannel
+  template:
+    metadata:
+      labels:
+        tier: node
+        app: flannel
+    spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: beta.kubernetes.io/os
+                    operator: In
+                    values:
+                      - windows
+                  - key: beta.kubernetes.io/arch
+                    operator: In
+                    values:
+                      - amd64
+      hostNetwork: true
+      serviceAccountName: flannel
+      tolerations:
+      - operator: Exists
+        effect: NoSchedule
+      containers:
+      - name: kube-flannel
+        image: gcr.io/cf-london-servces-k8s/windows-images/flannel:latest
+        command:
+        - powershell
+        args:
+        - -file
+        - /etc/kube-flannel-windows/run.ps1
+        volumeMounts:
+        - name: wins
+          mountPath: \\.\pipe\rancher_wins
+        - name: host
+          mountPath: /host
+        - name: kube-proxy
+          mountPath: /kube-proxy
+        - name: cni
+          mountPath: /etc/cni/net.d
+        - name: flannel-cfg
+          mountPath: /etc/kube-flannel/
+        - name: flannel-windows-cfg
+          mountPath: /etc/kube-flannel-windows/
+        - name: kubeadm-config
+          mountPath: /etc/kubeadm-config/
+        env:
+        - name: POD_NAME
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: metadata.name
+        - name: POD_NAMESPACE
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: metadata.namespace
+      volumes:
+      - name: opt
+        hostPath:
+          path: /opt
+      - name: host
+        hostPath:
+          path: /
+      - name: cni
+        hostPath:
+          path: /etc
+      - name: flannel-cfg
+        configMap:
+          name: kube-flannel-cfg
+      - name: flannel-windows-cfg
+        configMap:
+          name: kube-flannel-windows-cfg
+      - name: kube-proxy
+        configMap:
+          name: kube-proxy
+      - name: kubeadm-config
+        configMap:
+          name: kubeadm-config
+      - name: wins
+        hostPath:
+          path: \\.\pipe\rancher_wins
+          type: null
diff --git a/kubeadm/flannel/flannel-overlay.yml b/kubeadm/flannel/flannel-overlay.yml
new file mode 100644
index 00000000..c72d7245
--- /dev/null
+++ b/kubeadm/flannel/flannel-overlay.yml
@@ -0,0 +1,160 @@
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: kube-flannel-windows-cfg
+  namespace: kube-system
+  labels:
+    tier: node
+    app: flannel
+data:
+  run.ps1: |
+    $ErrorActionPreference = "Stop";
+
+    mkdir -force /host/etc/cni/net.d
+    mkdir -force /host/etc/kube-flannel
+    mkdir -force /host/opt/cni/bin
+    mkdir -force /host/flannel
+    mkdir -force /host/flannel/var/run/secrets/kubernetes.io/serviceaccount
+
+    $cniJson = get-content /etc/kube-flannel-windows/cni-conf.json | ConvertFrom-Json
+    $serviceSubnet = yq r /etc/kubeadm-config/ClusterConfiguration networking.serviceSubnet
+    $podSubnet = yq r /etc/kubeadm-config/ClusterConfiguration networking.podSubnet
+    $networkJson = wins cli net get | convertfrom-json
+
+    $cniJson.delegate.policies[0].Value.ExceptionList = $serviceSubnet, $podSubnet
+    $cniJson.delegate.policies[1].Value.DestinationPrefix = $serviceSubnet
+    Set-Content -Path /host/etc/cni/net.d/10-flannel.conf ($cniJson | ConvertTo-Json -depth 100)
+
+    cp -force /etc/kube-flannel/net-conf.json /host/etc/kube-flannel
+    cp -force -recurse /cni/* /host/opt/cni/bin
+    cp -force /flannel/flanneld.exe /host/flannel/flanneld.exe
+    cp -force /kube-proxy/kubeconfig.conf /host/flannel/kubeconfig.yml
+    cp -force /var/run/secrets/kubernetes.io/serviceaccount/* /host/flannel/var/run/secrets/kubernetes.io/serviceaccount/
+    wins cli process run --path C:\flannel\flanneld.exe --args "--kube-subnet-mgr --kubeconfig-file /flannel/kubeconfig.yml" --envs "POD_NAME=$env:POD_NAME POD_NAMESPACE=$env:POD_NAMESPACE"
+  cni-conf.json: |
+    {
+      "name": "flannel.4096",
+      "cniVersion": "0.3.0",
+      "type": "flannel",
+      "capabilities": {
+        "dns": true
+      },
+      "delegate": {
+        "type": "win-overlay",
+        "policies": [
+          {
+            "Name": "EndpointPolicy",
+            "Value": {
+              "Type": "OutBoundNAT",
+              "ExceptionList": []
+            }
+          },
+          {
+            "Name": "EndpointPolicy",
+            "Value": {
+              "Type": "ROUTE",
+              "DestinationPrefix": "",
+              "NeedEncap": true
+            }
+          }
+        ]
+      }
+    }
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: kube-flannel-ds-windows-amd64
+  labels:
+    tier: node
+    app: flannel
+  namespace: kube-system
+spec:
+  selector:
+    matchLabels:
+      app: flannel
+  template:
+    metadata:
+      labels:
+        tier: node
+        app: flannel
+    spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: beta.kubernetes.io/os
+                    operator: In
+                    values:
+                      - windows
+                  - key: beta.kubernetes.io/arch
+                    operator: In
+                    values:
+                      - amd64
+      hostNetwork: true
+      serviceAccountName: flannel
+      tolerations:
+      - operator: Exists
+        effect: NoSchedule
+      containers:
+      - name: kube-flannel
+        image: gcr.io/cf-london-servces-k8s/windows-images/flannel:latest
+        command:
+        - powershell
+        args:
+        - -file
+        - /etc/kube-flannel-windows/run.ps1
+        volumeMounts:
+        - name: wins
+          mountPath: \\.\pipe\rancher_wins
+        - name: host
+          mountPath: /host
+        - name: kube-proxy
+          mountPath: /kube-proxy
+        - name: cni
+          mountPath: /etc/cni/net.d
+        - name: flannel-cfg
+          mountPath: /etc/kube-flannel/
+        - name: flannel-windows-cfg
+          mountPath: /etc/kube-flannel-windows/
+        - name: kubeadm-config
+          mountPath: /etc/kubeadm-config/
+        env:
+        - name: POD_NAME
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: metadata.name
+        - name: POD_NAMESPACE
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: metadata.namespace
+      volumes:
+      - name: opt
+        hostPath:
+          path: /opt
+      - name: host
+        hostPath:
+          path: /
+      - name: cni
+        hostPath:
+          path: /etc
+      - name: flannel-cfg
+        configMap:
+          name: kube-flannel-cfg
+      - name: flannel-windows-cfg
+        configMap:
+          name: kube-flannel-windows-cfg
+      - name: kube-proxy
+        configMap:
+          name: kube-proxy
+      - name: kubeadm-config
+        configMap:
+          name: kubeadm-config
+      - name: wins
+        hostPath:
+          path: \\.\pipe\rancher_wins
+          type: null
diff --git a/kubeadm/kube-proxy/kube-proxy.yml b/kubeadm/kube-proxy/kube-proxy.yml
new file mode 100644
index 00000000..65b53126
--- /dev/null
+++ b/kubeadm/kube-proxy/kube-proxy.yml
@@ -0,0 +1,94 @@
+apiVersion: v1
+data:
+  run-script.ps1: |-
+    $ErrorActionPreference = "Stop";
+    mkdir -force /host/var/lib/kube-proxy/var/run/secrets/kubernetes.io/serviceaccount
+    mkdir -force /host/kube-proxy
+
+    cp -force /kube-proxy/* /host/kube-proxy
+    cp -force /var/lib/kube-proxy/* /host/var/lib/kube-proxy
+    cp -force /var/run/secrets/kubernetes.io/serviceaccount/* /host/var/lib/kube-proxy/var/run/secrets/kubernetes.io/serviceaccount #FIXME?
+
+    $networkName = (Get-Content /host/etc/cni/net.d/* | ConvertFrom-Json).name
+    $sourceVip = ($env:POD_IP -split "\.")[0..2] + 0 -join "."
+    yq w -i /host/var/lib/kube-proxy/config.conf winkernel.sourceVip $sourceVip
+    yq w -i /host/var/lib/kube-proxy/config.conf winkernel.networkName $networkName
+    yq w -i /host/var/lib/kube-proxy/config.conf featureGates.WinOverlay true
+    yq w -i /host/var/lib/kube-proxy/config.conf mode "kernelspace"
+    wins cli process run --path /kube-proxy/kube-proxy.exe --args "--v=6 --config=/var/lib/kube-proxy/config.conf --hostname-override=$env:NODE_NAME --feature-gates=WinOverlay=true"
+
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  labels:
+    app: kube-proxy
+  name: kube-proxy-windows
+  namespace: kube-system
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  labels:
+    k8s-app: kube-proxy
+  name: kube-proxy-windows
+  namespace: kube-system
+spec:
+  selector:
+    matchLabels:
+      k8s-app: kube-proxy-windows
+  template:
+    metadata:
+      labels:
+        k8s-app: kube-proxy-windows
+    spec:
+      serviceAccountName: kube-proxy
+      containers:
+      - command:
+        - powershell
+        args:
+        - -file
+        - /var/lib/kube-proxy-windows/run-script.ps1
+        env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: spec.nodeName
+        - name: POD_IP
+          valueFrom:
+            fieldRef:
+              fieldPath: status.podIP
+        image: gcr.io/cf-london-servces-k8s/windows-images/kube-proxy:latest
+        name: kube-proxy
+        volumeMounts:
+        - name: host
+          mountPath: /host
+        - name: wins
+          mountPath: \\.\pipe\rancher_wins
+        - mountPath: /var/lib/kube-proxy
+          name: kube-proxy
+        - mountPath: /var/lib/kube-proxy-windows
+          name: kube-proxy-windows
+      nodeSelector:
+        beta.kubernetes.io/os: windows
+      tolerations:
+      - key: CriticalAddonsOnly
+        operator: Exists
+      - operator: Exists
+      volumes:
+      - configMap:
+          defaultMode: 420
+          name: kube-proxy-windows
+        name: kube-proxy-windows
+      - configMap:
+          name: kube-proxy
+        name: kube-proxy
+      - hostPath:
+          path: /
+        name: host
+      - name: wins
+        hostPath:
+          path: \\.\pipe\rancher_wins
+          type: null
+  updateStrategy:
+    type: RollingUpdate

From b9ac979b325d127d219a925d8c84fcef276ab7ec Mon Sep 17 00:00:00 2001
From: Gab Satchi <gsatchithanantham@pivotal.io>
Date: Fri, 14 Feb 2020 13:55:03 -0500
Subject: [PATCH 02/18] Adds scripts to prepare and start kubelet

---
 kubeadm/scripts/PrepareNode.ps1  | 30 ++++++++++++++++++++++++++++++
 kubeadm/scripts/StartKubelet.ps1 | 21 +++++++++++++++++++++
 2 files changed, 51 insertions(+)
 create mode 100644 kubeadm/scripts/PrepareNode.ps1
 create mode 100644 kubeadm/scripts/StartKubelet.ps1

diff --git a/kubeadm/scripts/PrepareNode.ps1 b/kubeadm/scripts/PrepareNode.ps1
new file mode 100644
index 00000000..e9fc3292
--- /dev/null
+++ b/kubeadm/scripts/PrepareNode.ps1
@@ -0,0 +1,30 @@
+$global:Powershell = (Get-Command powershell).Source
+$global:PowershellArgs = "-ExecutionPolicy Bypass -NoProfile"
+$global:KubernetesPath = "$env:SystemDrive\k"
+$global:StartKubeletScript = "$global:KubernetesPath\StartKubelet.ps1"
+$kubeletBinPath = "$global:KubernetesPath\kubelet.exe"
+
+mkdir -force "$global:KubernetesPath"
+$env:Path += ";$global:KubernetesPath"
+[Environment]::SetEnvironmentVariable("Path", $env:Path, [System.EnvironmentVariableTarget]::Machine)
+
+curl.exe -Lo $kubeletBinPath https://dl.k8s.io/v1.17.0/bin/windows/amd64/kubelet.exe
+curl.exe -Lo "$global:KubernetesPath\kubeadm.exe" https://dl.k8s.io/v1.17.0/bin/windows/amd64/kubeadm.exe
+curl.exe -Lo "$global:KubernetesPath\wins.exe" https://github.com/rancher/wins/releases/download/v0.0.4/wins.exe
+
+#Create host network to allow kubelet to schedule hostNetwork pods
+docker network create -d nat host
+
+wins.exe srv app run --register
+start-service rancher-wins
+
+mkdir -force C:\var\log\kubelet
+mkdir -force C:\var\lib\kubelet\etc\kubernetes
+mkdir -force C:\etc\kubernetes\pki
+New-Item -path C:\var\lib\kubelet\etc\kubernetes\pki -type SymbolicLink -value C:\etc\kubernetes\pki\
+
+cp $PSScriptRoot/StartKubelet.ps1 $global:StartKubeletScript
+nssm install kubelet $global:Powershell $global:PowershellArgs $global:StartKubeletScript
+nssm set kubelet DependOnService docker
+
+New-NetFirewallRule -Name kubelet -DisplayName 'kubelet' -Enabled True -Direction Inbound -Protocol TCP -Action Allow -LocalPort 10250
diff --git a/kubeadm/scripts/StartKubelet.ps1 b/kubeadm/scripts/StartKubelet.ps1
new file mode 100644
index 00000000..7576658a
--- /dev/null
+++ b/kubeadm/scripts/StartKubelet.ps1
@@ -0,0 +1,21 @@
+#read kubelet args file
+$FileContent = Get-Content -Path "/var/lib/kubelet/kubeadm-flags.env"
+$KubeletArgs = $FileContent.Trim('KUBELET_KUBEADM_ARGS=')
+
+
+kubelet.exe `
+--cert-dir=$env:SYSTEMDRIVE\var\lib\kubelet\pki `
+--config=/var/lib/kubelet/config.yaml `
+--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf `
+--kubeconfig=/etc/kubernetes/kubelet.conf `
+--hostname-override=$(hostname) `
+--pod-infra-container-image=`"mcr.microsoft.com/k8s/core/pause:1.2.0`" `
+--enable-debugging-handlers `
+--cgroups-per-qos=false `
+--enforce-node-allocatable=`"`" `
+--network-plugin=cni `
+--resolv-conf=`"`" `
+--log-dir=/var/log/kubelet `
+--logtostderr=false `
+--image-pull-progress-deadline=20m `
+$KubeletArgs

From c1d65e33bb57af8ddd4e3a2372707e0cd1c49ece Mon Sep 17 00:00:00 2001
From: Gab Satchi <gsatchithanantham@pivotal.io>
Date: Tue, 18 Feb 2020 10:32:59 -0500
Subject: [PATCH 03/18] Kubelet service wrapper reads kubeadm kubelet flags

---
 kubeadm/scripts/StartKubelet.ps1 | 21 +++------------------
 1 file changed, 3 insertions(+), 18 deletions(-)

diff --git a/kubeadm/scripts/StartKubelet.ps1 b/kubeadm/scripts/StartKubelet.ps1
index 7576658a..ee3787a6 100644
--- a/kubeadm/scripts/StartKubelet.ps1
+++ b/kubeadm/scripts/StartKubelet.ps1
@@ -1,21 +1,6 @@
-#read kubelet args file
 $FileContent = Get-Content -Path "/var/lib/kubelet/kubeadm-flags.env"
-$KubeletArgs = $FileContent.Trim('KUBELET_KUBEADM_ARGS=')
+$global:KubeletArgs = $FileContent.Trim('KUBELET_KUBEADM_ARGS="')
 
+$cmd = "C:\k\kubelet.exe --cert-dir=$env:SYSTEMDRIVE\var\lib\kubelet\pki --config=/var/lib/kubelet/config.yaml --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf --hostname-override=$(hostname) --pod-infra-container-image=`"mcr.microsoft.com/k8s/core/pause:1.2.0`" --enable-debugging-handlers --cgroups-per-qos=false --enforce-node-allocatable=`"`" --network-plugin=cni --resolv-conf=`"`" --log-dir=/var/log/kubelet --logtostderr=false --image-pull-progress-deadline=20m $global:KubeletArgs"
 
-kubelet.exe `
---cert-dir=$env:SYSTEMDRIVE\var\lib\kubelet\pki `
---config=/var/lib/kubelet/config.yaml `
---bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf `
---kubeconfig=/etc/kubernetes/kubelet.conf `
---hostname-override=$(hostname) `
---pod-infra-container-image=`"mcr.microsoft.com/k8s/core/pause:1.2.0`" `
---enable-debugging-handlers `
---cgroups-per-qos=false `
---enforce-node-allocatable=`"`" `
---network-plugin=cni `
---resolv-conf=`"`" `
---log-dir=/var/log/kubelet `
---logtostderr=false `
---image-pull-progress-deadline=20m `
-$KubeletArgs
+Invoke-Expression $cmd

From edbbfa0d50d7c811947f226e39db95f5307b74b6 Mon Sep 17 00:00:00 2001
From: Gab Satchi <gsatchithanantham@pivotal.io>
Date: Thu, 20 Feb 2020 15:48:02 -0500
Subject: [PATCH 04/18] Use nssm, support specifying k8s version

---
 kubeadm/scripts/InstallNssm.ps1 | 19 ++++++++++++
 kubeadm/scripts/PrepareNode.ps1 | 52 +++++++++++++++++++++++++++++++--
 2 files changed, 68 insertions(+), 3 deletions(-)
 create mode 100644 kubeadm/scripts/InstallNssm.ps1

diff --git a/kubeadm/scripts/InstallNssm.ps1 b/kubeadm/scripts/InstallNssm.ps1
new file mode 100644
index 00000000..d6233965
--- /dev/null
+++ b/kubeadm/scripts/InstallNssm.ps1
@@ -0,0 +1,19 @@
+$ErrorActionPreference = "Stop"
+$global:NssmInstallDirectory = "$env:ProgramFiles\nssm"
+$arch = "win32"
+if ([Environment]::Is64BitOperatingSystem) {
+    $arch = "win64"
+}
+
+mkdir -Force $global:NssmInstallDirectory
+curl.exe -Lo nssm.zip https://nssm.cc/release/nssm-2.24.zip
+tar C $global:NssmInstallDirectory -xvf .\nssm.zip --strip-components 2 */$arch/*.exe
+Remove-Item -Force .\nssm.zip
+
+$env:path += ";$global:NssmInstallDirectory"
+$newPath = "$global:NssmInstallDirectory;" +
+[Environment]::GetEnvironmentVariable("PATH",
+[EnvironmentVariableTarget]::Machine)
+
+[Environment]::SetEnvironmentVariable("PATH", $newPath,
+[EnvironmentVariableTarget]::Machine)
diff --git a/kubeadm/scripts/PrepareNode.ps1 b/kubeadm/scripts/PrepareNode.ps1
index e9fc3292..b516a06c 100644
--- a/kubeadm/scripts/PrepareNode.ps1
+++ b/kubeadm/scripts/PrepareNode.ps1
@@ -1,3 +1,41 @@
+<#
+.SYNOPSIS
+Assists with preparing a Windows VM prior to calling kubeadm join
+
+.DESCRIPTION
+This script assists with joining a Windows node to a cluster.
+- Downloads Kubernetes binaries (kubelet, kubeadm) at the version specified
+- Registers wins as a service in order to run kube-proxy and cni as DaemonSets. More info here: 
+- Registers kubelet as an nssm service. More info on nssm: https://nssm.cc/
+
+.PARAMETER KubernetesVersion
+Kubernetes version to download and use
+
+.EXAMPLE
+PS> .\PrepareNode.ps1 -KubernetesVersion v1.17.0
+
+#>
+
+Param(
+    [parameter(Mandatory = $true, HelpMessage="Kubernetes version to use")]
+    [string] $KubernetesVersion
+)
+$ErrorActionPreference = 'Stop'
+
+function DownloadFile($destination, $source) {
+    Write-Host("Downloading $source to $destination")
+    curl.exe --fail -Lo $destination $source
+
+    if (!$?) {
+        Write-Error "Download $source failed"
+        exit 1
+    }
+}
+
+if (!$KubernetesVersion.StartsWith("v")) {
+    $KubernetesVersion = "v" + $KubernetesVersion
+}
+Write-Host "Using Kubernete version: $KubernetesVersion"
 $global:Powershell = (Get-Command powershell).Source
 $global:PowershellArgs = "-ExecutionPolicy Bypass -NoProfile"
 $global:KubernetesPath = "$env:SystemDrive\k"
@@ -8,13 +46,15 @@ mkdir -force "$global:KubernetesPath"
 $env:Path += ";$global:KubernetesPath"
 [Environment]::SetEnvironmentVariable("Path", $env:Path, [System.EnvironmentVariableTarget]::Machine)
 
-curl.exe -Lo $kubeletBinPath https://dl.k8s.io/v1.17.0/bin/windows/amd64/kubelet.exe
-curl.exe -Lo "$global:KubernetesPath\kubeadm.exe" https://dl.k8s.io/v1.17.0/bin/windows/amd64/kubeadm.exe
-curl.exe -Lo "$global:KubernetesPath\wins.exe" https://github.com/rancher/wins/releases/download/v0.0.4/wins.exe
+DownloadFile $kubeletBinPath https://dl.k8s.io/$KubernetesVersion/bin/windows/amd64/kubelet.exe
+DownloadFile "$global:KubernetesPath\kubeadm.exe" https://dl.k8s.io/$KubernetesVersion/bin/windows/amd64/kubeadm.exe
+DownloadFile "$global:KubernetesPath\wins.exe" https://github.com/rancher/wins/releases/download/v0.0.4/wins.exe
 
 #Create host network to allow kubelet to schedule hostNetwork pods
+Write-Host "Creating Docker host network"
 docker network create -d nat host
 
+Write-Host "Registering wins service"
 wins.exe srv app run --register
 start-service rancher-wins
 
@@ -24,6 +64,12 @@ mkdir -force C:\etc\kubernetes\pki
 New-Item -path C:\var\lib\kubelet\etc\kubernetes\pki -type SymbolicLink -value C:\etc\kubernetes\pki\
 
 cp $PSScriptRoot/StartKubelet.ps1 $global:StartKubeletScript
+Write-Host "Registering kubelet service"
+Get-Command nssm
+if (!$?) {
+    Write-Error "NSSM is not installed."
+    exit 1
+}
 nssm install kubelet $global:Powershell $global:PowershellArgs $global:StartKubeletScript
 nssm set kubelet DependOnService docker
 

From bfdfaa8015a46a70561e30679e1842900ddcfe74 Mon Sep 17 00:00:00 2001
From: Ben Moss <bmoss@pivotal.io>
Date: Thu, 20 Feb 2020 16:14:01 -0500
Subject: [PATCH 05/18] Add Dockerfiles for flannel & kube-proxy

---
 kubeadm/flannel/Dockerfile    | 16 ++++++++++++++++
 kubeadm/kube-proxy/Dockerfile | 10 ++++++++++
 2 files changed, 26 insertions(+)
 create mode 100644 kubeadm/flannel/Dockerfile
 create mode 100644 kubeadm/kube-proxy/Dockerfile

diff --git a/kubeadm/flannel/Dockerfile b/kubeadm/flannel/Dockerfile
new file mode 100644
index 00000000..79c07f05
--- /dev/null
+++ b/kubeadm/flannel/Dockerfile
@@ -0,0 +1,16 @@
+FROM mcr.microsoft.com/windows/servercore:ltsc2019
+SHELL ["powershell", "-NoLogo", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"]
+
+RUN mkdir C:\flannel; \
+  pushd C:\flannel; \
+  curl.exe -LO https://github.com/benmoss/flannel/releases/download/v0.12.0-rc1/flanneld.exe
+
+RUN mkdir C:\cni; \
+  pushd C:\cni; \
+  curl.exe -Lo cni.tgz https://github.com/containernetworking/plugins/releases/download/v0.8.2/cni-plugins-windows-amd64-v0.8.2.tgz; \
+  tar -xf cni.tgz; \
+  rm cni.tgz
+
+RUN curl.exe -Lo C:\windows\wins.exe https://github.com/rancher/wins/releases/download/v0.0.4/wins.exe
+
+RUN curl.exe -Lo C:\windows\yq.exe https://github.com/mikefarah/yq/releases/download/2.4.1/yq_windows_amd64.exe
diff --git a/kubeadm/kube-proxy/Dockerfile b/kubeadm/kube-proxy/Dockerfile
new file mode 100644
index 00000000..583ef280
--- /dev/null
+++ b/kubeadm/kube-proxy/Dockerfile
@@ -0,0 +1,10 @@
+FROM mcr.microsoft.com/windows/servercore:ltsc2019
+SHELL ["powershell", "-NoLogo", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"]
+
+RUN mkdir C:\kube-proxy; \
+  pushd C:\kube-proxy; \
+  curl.exe -LO https://dl.k8s.io/v1.16.2/bin/windows/amd64/kube-proxy.exe
+
+RUN curl.exe -Lo C:\windows\wins.exe https://github.com/rancher/wins/releases/download/v0.0.4/wins.exe
+
+RUN curl.exe -Lo C:\windows\yq.exe https://github.com/mikefarah/yq/releases/download/2.4.1/yq_windows_amd64.exe

From 6a0cdcab6a0c8791435aa012ed687b3be1f13ea0 Mon Sep 17 00:00:00 2001
From: Ben Moss <moss.127@gmail.com>
Date: Thu, 20 Feb 2020 17:28:31 -0500
Subject: [PATCH 06/18] Fix typo

---
 kubeadm/scripts/PrepareNode.ps1 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/kubeadm/scripts/PrepareNode.ps1 b/kubeadm/scripts/PrepareNode.ps1
index b516a06c..5e4cc91d 100644
--- a/kubeadm/scripts/PrepareNode.ps1
+++ b/kubeadm/scripts/PrepareNode.ps1
@@ -35,7 +35,7 @@ function DownloadFile($destination, $source) {
 if (!$KubernetesVersion.StartsWith("v")) {
     $KubernetesVersion = "v" + $KubernetesVersion
 }
-Write-Host "Using Kubernete version: $KubernetesVersion"
+Write-Host "Using Kubernetes version: $KubernetesVersion"
 $global:Powershell = (Get-Command powershell).Source
 $global:PowershellArgs = "-ExecutionPolicy Bypass -NoProfile"
 $global:KubernetesPath = "$env:SystemDrive\k"

From 4850bb5d2a2a1c6169b5dbbc9b3f5a1d42eb90e4 Mon Sep 17 00:00:00 2001
From: Gab Satchi <gsatchithanantham@pivotal.io>
Date: Fri, 21 Feb 2020 10:22:44 -0500
Subject: [PATCH 07/18] Start kubelet script now generated in PrepareNode

---
 kubeadm/scripts/PrepareNode.ps1  | 9 ++++++++-
 kubeadm/scripts/StartKubelet.ps1 | 6 ------
 2 files changed, 8 insertions(+), 7 deletions(-)
 delete mode 100644 kubeadm/scripts/StartKubelet.ps1

diff --git a/kubeadm/scripts/PrepareNode.ps1 b/kubeadm/scripts/PrepareNode.ps1
index 5e4cc91d..45ee6653 100644
--- a/kubeadm/scripts/PrepareNode.ps1
+++ b/kubeadm/scripts/PrepareNode.ps1
@@ -63,7 +63,14 @@ mkdir -force C:\var\lib\kubelet\etc\kubernetes
 mkdir -force C:\etc\kubernetes\pki
 New-Item -path C:\var\lib\kubelet\etc\kubernetes\pki -type SymbolicLink -value C:\etc\kubernetes\pki\
 
-cp $PSScriptRoot/StartKubelet.ps1 $global:StartKubeletScript
+$StartKubeletFileContent = '$FileContent = Get-Content -Path "/var/lib/kubelet/kubeadm-flags.env"
+$global:KubeletArgs = $FileContent.Trim("KUBELET_KUBEADM_ARGS=`"")
+
+$cmd = "C:\k\kubelet.exe --cert-dir=$env:SYSTEMDRIVE\var\lib\kubelet\pki --config=/var/lib/kubelet/config.yaml --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf --hostname-override=$(hostname) --pod-infra-container-image=`"mcr.microsoft.com/k8s/core/pause:1.2.0`" --enable-debugging-handlers --cgroups-per-qos=false --enforce-node-allocatable=`"`" --network-plugin=cni --resolv-conf=`"`" --log-dir=/var/log/kubelet --logtostderr=false --image-pull-progress-deadline=20m $global:KubeletArgs"
+
+Invoke-Expression $cmd'
+Set-Content -Path $global:StartKubeletScript -Value $StartKubeletFileContent
+
 Write-Host "Registering kubelet service"
 Get-Command nssm
 if (!$?) {
diff --git a/kubeadm/scripts/StartKubelet.ps1 b/kubeadm/scripts/StartKubelet.ps1
deleted file mode 100644
index ee3787a6..00000000
--- a/kubeadm/scripts/StartKubelet.ps1
+++ /dev/null
@@ -1,6 +0,0 @@
-$FileContent = Get-Content -Path "/var/lib/kubelet/kubeadm-flags.env"
-$global:KubeletArgs = $FileContent.Trim('KUBELET_KUBEADM_ARGS="')
-
-$cmd = "C:\k\kubelet.exe --cert-dir=$env:SYSTEMDRIVE\var\lib\kubelet\pki --config=/var/lib/kubelet/config.yaml --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf --hostname-override=$(hostname) --pod-infra-container-image=`"mcr.microsoft.com/k8s/core/pause:1.2.0`" --enable-debugging-handlers --cgroups-per-qos=false --enforce-node-allocatable=`"`" --network-plugin=cni --resolv-conf=`"`" --log-dir=/var/log/kubelet --logtostderr=false --image-pull-progress-deadline=20m $global:KubeletArgs"
-
-Invoke-Expression $cmd

From 325bd115ac8a344c5e51c8c819e925676ff4e4fa Mon Sep 17 00:00:00 2001
From: Ben Moss <bmoss@pivotal.io>
Date: Mon, 24 Feb 2020 12:33:42 -0500
Subject: [PATCH 08/18] Parameterize flannel/kube-proxy dockerfiles

---
 kubeadm/flannel/Dockerfile    | 10 ++++++++--
 kubeadm/kube-proxy/Dockerfile |  9 +++++++--
 2 files changed, 15 insertions(+), 4 deletions(-)

diff --git a/kubeadm/flannel/Dockerfile b/kubeadm/flannel/Dockerfile
index 79c07f05..2b0f45ba 100644
--- a/kubeadm/flannel/Dockerfile
+++ b/kubeadm/flannel/Dockerfile
@@ -1,13 +1,19 @@
-FROM mcr.microsoft.com/windows/servercore:ltsc2019
+ARG servercoreTag="ltsc2019"
+ARG cniVersion="0.8.5"
+
+FROM mcr.microsoft.com/windows/servercore:${servercoreTag}
 SHELL ["powershell", "-NoLogo", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"]
 
+ARG cniVersion
+
+# Stuck on a prerelease flannel until https://github.com/coreos/flannel/issues/1231 is resolved
 RUN mkdir C:\flannel; \
   pushd C:\flannel; \
   curl.exe -LO https://github.com/benmoss/flannel/releases/download/v0.12.0-rc1/flanneld.exe
 
 RUN mkdir C:\cni; \
   pushd C:\cni; \
-  curl.exe -Lo cni.tgz https://github.com/containernetworking/plugins/releases/download/v0.8.2/cni-plugins-windows-amd64-v0.8.2.tgz; \
+  curl.exe -Lo cni.tgz https://github.com/containernetworking/plugins/releases/download/v${env:cniVersion}/cni-plugins-windows-amd64-v${env:cniVersion}.tgz; \
   tar -xf cni.tgz; \
   rm cni.tgz
 
diff --git a/kubeadm/kube-proxy/Dockerfile b/kubeadm/kube-proxy/Dockerfile
index 583ef280..d66f077f 100644
--- a/kubeadm/kube-proxy/Dockerfile
+++ b/kubeadm/kube-proxy/Dockerfile
@@ -1,9 +1,14 @@
-FROM mcr.microsoft.com/windows/servercore:ltsc2019
+ARG k8sVersion="1.17.3"
+ARG servercoreTag="ltsc2019"
+
+FROM mcr.microsoft.com/windows/servercore:${servercoreTag}
 SHELL ["powershell", "-NoLogo", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"]
 
+ARG k8sVersion
+
 RUN mkdir C:\kube-proxy; \
   pushd C:\kube-proxy; \
-  curl.exe -LO https://dl.k8s.io/v1.16.2/bin/windows/amd64/kube-proxy.exe
+  curl.exe -LO https://dl.k8s.io/v${env:k8sVersion}/bin/windows/amd64/kube-proxy.exe
 
 RUN curl.exe -Lo C:\windows\wins.exe https://github.com/rancher/wins/releases/download/v0.0.4/wins.exe
 

From a8c6f2c6937ef79537f7e87ed5d24725c41acbe0 Mon Sep 17 00:00:00 2001
From: Ben Moss <bmoss@pivotal.io>
Date: Mon, 24 Feb 2020 13:56:22 -0500
Subject: [PATCH 09/18] Update image locations to sigwindowstools dockerhub
 account

---
 kubeadm/flannel/flannel-host-gw.yml | 2 +-
 kubeadm/flannel/flannel-overlay.yml | 2 +-
 kubeadm/kube-proxy/kube-proxy.yml   | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/kubeadm/flannel/flannel-host-gw.yml b/kubeadm/flannel/flannel-host-gw.yml
index 3a575043..53445b46 100644
--- a/kubeadm/flannel/flannel-host-gw.yml
+++ b/kubeadm/flannel/flannel-host-gw.yml
@@ -111,7 +111,7 @@ spec:
         effect: NoSchedule
       containers:
       - name: kube-flannel
-        image: gcr.io/cf-london-servces-k8s/windows-images/flannel:latest
+        image: sigwindowstools/flannel:0.12.0-rc1
         command:
         - powershell
         args:
diff --git a/kubeadm/flannel/flannel-overlay.yml b/kubeadm/flannel/flannel-overlay.yml
index c72d7245..a302d08a 100644
--- a/kubeadm/flannel/flannel-overlay.yml
+++ b/kubeadm/flannel/flannel-overlay.yml
@@ -100,7 +100,7 @@ spec:
         effect: NoSchedule
       containers:
       - name: kube-flannel
-        image: gcr.io/cf-london-servces-k8s/windows-images/flannel:latest
+        image: sigwindowstools/flannel:0.12.0-rc1
         command:
         - powershell
         args:
diff --git a/kubeadm/kube-proxy/kube-proxy.yml b/kubeadm/kube-proxy/kube-proxy.yml
index 65b53126..160900fe 100644
--- a/kubeadm/kube-proxy/kube-proxy.yml
+++ b/kubeadm/kube-proxy/kube-proxy.yml
@@ -58,7 +58,7 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: status.podIP
-        image: gcr.io/cf-london-servces-k8s/windows-images/kube-proxy:latest
+        image: sigwindowstools/kube-proxy:1.17.3
         name: kube-proxy
         volumeMounts:
         - name: host

From e36109bb328f2a0e95f64ad44c474990a87200d2 Mon Sep 17 00:00:00 2001
From: Ben Moss <bmoss@pivotal.io>
Date: Mon, 24 Feb 2020 16:25:16 -0500
Subject: [PATCH 10/18] Move $kubeletArgs to beginning of kubelet invocation

This allows subsequent args (namely --pod-infra-container-image) to
override any linux-specific flags we get
---
 kubeadm/scripts/PrepareNode.ps1 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/kubeadm/scripts/PrepareNode.ps1 b/kubeadm/scripts/PrepareNode.ps1
index 45ee6653..f94dbb20 100644
--- a/kubeadm/scripts/PrepareNode.ps1
+++ b/kubeadm/scripts/PrepareNode.ps1
@@ -66,7 +66,7 @@ New-Item -path C:\var\lib\kubelet\etc\kubernetes\pki -type SymbolicLink -value C
 $StartKubeletFileContent = '$FileContent = Get-Content -Path "/var/lib/kubelet/kubeadm-flags.env"
 $global:KubeletArgs = $FileContent.Trim("KUBELET_KUBEADM_ARGS=`"")
 
-$cmd = "C:\k\kubelet.exe --cert-dir=$env:SYSTEMDRIVE\var\lib\kubelet\pki --config=/var/lib/kubelet/config.yaml --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf --hostname-override=$(hostname) --pod-infra-container-image=`"mcr.microsoft.com/k8s/core/pause:1.2.0`" --enable-debugging-handlers --cgroups-per-qos=false --enforce-node-allocatable=`"`" --network-plugin=cni --resolv-conf=`"`" --log-dir=/var/log/kubelet --logtostderr=false --image-pull-progress-deadline=20m $global:KubeletArgs"
+$cmd = "C:\k\kubelet.exe $global:KubeletArgs --cert-dir=$env:SYSTEMDRIVE\var\lib\kubelet\pki --config=/var/lib/kubelet/config.yaml --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf --hostname-override=$(hostname) --pod-infra-container-image=`"mcr.microsoft.com/k8s/core/pause:1.2.0`" --enable-debugging-handlers --cgroups-per-qos=false --enforce-node-allocatable=`"`" --network-plugin=cni --resolv-conf=`"`" --log-dir=/var/log/kubelet --logtostderr=false --image-pull-progress-deadline=20m"
 
 Invoke-Expression $cmd'
 Set-Content -Path $global:StartKubeletScript -Value $StartKubeletFileContent

From 5fc42e884541c1bd9ce18ef60808c0ce5b4e59ab Mon Sep 17 00:00:00 2001
From: Ben Moss <bmoss@pivotal.io>
Date: Tue, 25 Feb 2020 09:49:00 -0500
Subject: [PATCH 11/18] Move wins and yq to C:\utils

Avoid using the Windows directory to store non-official binaries
---
 kubeadm/flannel/Dockerfile    | 7 ++++---
 kubeadm/kube-proxy/Dockerfile | 7 ++++---
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/kubeadm/flannel/Dockerfile b/kubeadm/flannel/Dockerfile
index 2b0f45ba..561976e7 100644
--- a/kubeadm/flannel/Dockerfile
+++ b/kubeadm/flannel/Dockerfile
@@ -17,6 +17,7 @@ RUN mkdir C:\cni; \
   tar -xf cni.tgz; \
   rm cni.tgz
 
-RUN curl.exe -Lo C:\windows\wins.exe https://github.com/rancher/wins/releases/download/v0.0.4/wins.exe
-
-RUN curl.exe -Lo C:\windows\yq.exe https://github.com/mikefarah/yq/releases/download/2.4.1/yq_windows_amd64.exe
+RUN mkdir C:\utils; \
+  curl.exe -Lo C:\utils\wins.exe https://github.com/rancher/wins/releases/download/v0.0.4/wins.exe; \
+  curl.exe -Lo C:\utils\yq.exe https://github.com/mikefarah/yq/releases/download/2.4.1/yq_windows_amd64.exe; \
+  "[Environment]::SetEnvironmentVariable('PATH', $env:PATH + ';C:\utils', [EnvironmentVariableTarget]::Machine)"
diff --git a/kubeadm/kube-proxy/Dockerfile b/kubeadm/kube-proxy/Dockerfile
index d66f077f..0cfc4a53 100644
--- a/kubeadm/kube-proxy/Dockerfile
+++ b/kubeadm/kube-proxy/Dockerfile
@@ -10,6 +10,7 @@ RUN mkdir C:\kube-proxy; \
   pushd C:\kube-proxy; \
   curl.exe -LO https://dl.k8s.io/v${env:k8sVersion}/bin/windows/amd64/kube-proxy.exe
 
-RUN curl.exe -Lo C:\windows\wins.exe https://github.com/rancher/wins/releases/download/v0.0.4/wins.exe
-
-RUN curl.exe -Lo C:\windows\yq.exe https://github.com/mikefarah/yq/releases/download/2.4.1/yq_windows_amd64.exe
+RUN mkdir C:\utils; \
+  curl.exe -Lo C:\utils\wins.exe https://github.com/rancher/wins/releases/download/v0.0.4/wins.exe; \
+  curl.exe -Lo C:\utils\yq.exe https://github.com/mikefarah/yq/releases/download/2.4.1/yq_windows_amd64.exe; \
+  "[Environment]::SetEnvironmentVariable('PATH', $env:PATH + ';C:\utils', [EnvironmentVariableTarget]::Machine)"

From d2203e7658d28bef43547f0c4187b19227ca2b27 Mon Sep 17 00:00:00 2001
From: Ben Moss <bmoss@pivotal.io>
Date: Tue, 25 Feb 2020 10:37:19 -0500
Subject: [PATCH 12/18] Move kube-proxy and flannel to under C:\k directory

---
 kubeadm/flannel/flannel-host-gw.yml | 12 ++++++------
 kubeadm/flannel/flannel-overlay.yml | 12 ++++++------
 kubeadm/kube-proxy/kube-proxy.yml   |  6 +++---
 3 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/kubeadm/flannel/flannel-host-gw.yml b/kubeadm/flannel/flannel-host-gw.yml
index 53445b46..009a16c2 100644
--- a/kubeadm/flannel/flannel-host-gw.yml
+++ b/kubeadm/flannel/flannel-host-gw.yml
@@ -14,8 +14,8 @@ data:
     mkdir -force /host/etc/cni/net.d
     mkdir -force /host/etc/kube-flannel
     mkdir -force /host/opt/cni/bin
-    mkdir -force /host/flannel
-    mkdir -force /host/flannel/var/run/secrets/kubernetes.io/serviceaccount
+    mkdir -force /host/k/flannel
+    mkdir -force /host/k/flannel/var/run/secrets/kubernetes.io/serviceaccount
 
     $cniJson = get-content /etc/kube-flannel-windows/cni-conf.json | ConvertFrom-Json
     $serviceSubnet = yq r /etc/kubeadm-config/ClusterConfiguration networking.serviceSubnet
@@ -29,10 +29,10 @@ data:
 
     cp -force /etc/kube-flannel/net-conf.json /host/etc/kube-flannel
     cp -force -recurse /cni/* /host/opt/cni/bin
-    cp -force /flannel/flanneld.exe /host/flannel/flanneld.exe
-    cp -force /kube-proxy/kubeconfig.conf /host/flannel/kubeconfig.yml
-    cp -force /var/run/secrets/kubernetes.io/serviceaccount/* /host/flannel/var/run/secrets/kubernetes.io/serviceaccount/
-    wins cli process run --path C:\flannel\flanneld.exe --args "--kube-subnet-mgr --kubeconfig-file /flannel/kubeconfig.yml" --envs "POD_NAME=$env:POD_NAME POD_NAMESPACE=$env:POD_NAMESPACE"
+    cp -force /flannel/flanneld.exe /host/k/flannel/flanneld.exe
+    cp -force /kube-proxy/kubeconfig.conf /host/k/flannel/kubeconfig.yml
+    cp -force /var/run/secrets/kubernetes.io/serviceaccount/* /host/k/flannel/var/run/secrets/kubernetes.io/serviceaccount/
+    wins cli process run --path /k/flannel/flanneld.exe --args "--kube-subnet-mgr --kubeconfig-file /k/flannel/kubeconfig.yml" --envs "POD_NAME=$env:POD_NAME POD_NAMESPACE=$env:POD_NAMESPACE"
   cni-conf.json: |
     {
       "name": "cbr0",
diff --git a/kubeadm/flannel/flannel-overlay.yml b/kubeadm/flannel/flannel-overlay.yml
index a302d08a..b08849ad 100644
--- a/kubeadm/flannel/flannel-overlay.yml
+++ b/kubeadm/flannel/flannel-overlay.yml
@@ -14,8 +14,8 @@ data:
     mkdir -force /host/etc/cni/net.d
     mkdir -force /host/etc/kube-flannel
     mkdir -force /host/opt/cni/bin
-    mkdir -force /host/flannel
-    mkdir -force /host/flannel/var/run/secrets/kubernetes.io/serviceaccount
+    mkdir -force /host/k/flannel
+    mkdir -force /host/k/flannel/var/run/secrets/kubernetes.io/serviceaccount
 
     $cniJson = get-content /etc/kube-flannel-windows/cni-conf.json | ConvertFrom-Json
     $serviceSubnet = yq r /etc/kubeadm-config/ClusterConfiguration networking.serviceSubnet
@@ -28,10 +28,10 @@ data:
 
     cp -force /etc/kube-flannel/net-conf.json /host/etc/kube-flannel
     cp -force -recurse /cni/* /host/opt/cni/bin
-    cp -force /flannel/flanneld.exe /host/flannel/flanneld.exe
-    cp -force /kube-proxy/kubeconfig.conf /host/flannel/kubeconfig.yml
-    cp -force /var/run/secrets/kubernetes.io/serviceaccount/* /host/flannel/var/run/secrets/kubernetes.io/serviceaccount/
-    wins cli process run --path C:\flannel\flanneld.exe --args "--kube-subnet-mgr --kubeconfig-file /flannel/kubeconfig.yml" --envs "POD_NAME=$env:POD_NAME POD_NAMESPACE=$env:POD_NAMESPACE"
+    cp -force /flannel/flanneld.exe /host/k/flannel/flanneld.exe
+    cp -force /kube-proxy/kubeconfig.conf /host/k/flannel/kubeconfig.yml
+    cp -force /var/run/secrets/kubernetes.io/serviceaccount/* /host/k/flannel/var/run/secrets/kubernetes.io/serviceaccount/
+    wins cli process run --path /k/flannel/flanneld.exe --args "--kube-subnet-mgr --kubeconfig-file /k/flannel/kubeconfig.yml" --envs "POD_NAME=$env:POD_NAME POD_NAMESPACE=$env:POD_NAMESPACE"
   cni-conf.json: |
     {
       "name": "flannel.4096",
diff --git a/kubeadm/kube-proxy/kube-proxy.yml b/kubeadm/kube-proxy/kube-proxy.yml
index 160900fe..996607e8 100644
--- a/kubeadm/kube-proxy/kube-proxy.yml
+++ b/kubeadm/kube-proxy/kube-proxy.yml
@@ -3,9 +3,9 @@ data:
   run-script.ps1: |-
     $ErrorActionPreference = "Stop";
     mkdir -force /host/var/lib/kube-proxy/var/run/secrets/kubernetes.io/serviceaccount
-    mkdir -force /host/kube-proxy
+    mkdir -force /host/k/kube-proxy
 
-    cp -force /kube-proxy/* /host/kube-proxy
+    cp -force /kube-proxy/* /host/k/kube-proxy
     cp -force /var/lib/kube-proxy/* /host/var/lib/kube-proxy
     cp -force /var/run/secrets/kubernetes.io/serviceaccount/* /host/var/lib/kube-proxy/var/run/secrets/kubernetes.io/serviceaccount #FIXME?
 
@@ -15,7 +15,7 @@ data:
     yq w -i /host/var/lib/kube-proxy/config.conf winkernel.networkName $networkName
     yq w -i /host/var/lib/kube-proxy/config.conf featureGates.WinOverlay true
     yq w -i /host/var/lib/kube-proxy/config.conf mode "kernelspace"
-    wins cli process run --path /kube-proxy/kube-proxy.exe --args "--v=6 --config=/var/lib/kube-proxy/config.conf --hostname-override=$env:NODE_NAME --feature-gates=WinOverlay=true"
+    wins cli process run --path /k/kube-proxy/kube-proxy.exe --args "--v=6 --config=/var/lib/kube-proxy/config.conf --hostname-override=$env:NODE_NAME --feature-gates=WinOverlay=true"
 
 kind: ConfigMap
 apiVersion: v1

From c9ce0bcc36119c5429cc281155778f9aaf57a4d9 Mon Sep 17 00:00:00 2001
From: Ben Moss <bmoss@pivotal.io>
Date: Tue, 25 Feb 2020 14:24:59 -0500
Subject: [PATCH 13/18] Move binaries inside containers to C:\k

wins for whatever reason validates the path both inside the container
and on the host, meaning the file has to exist and have the same
checksum inside as well as outside.
---
 kubeadm/flannel/Dockerfile          | 4 ++--
 kubeadm/flannel/flannel-host-gw.yml | 2 +-
 kubeadm/flannel/flannel-overlay.yml | 2 +-
 kubeadm/kube-proxy/Dockerfile       | 4 ++--
 kubeadm/kube-proxy/kube-proxy.yml   | 2 +-
 5 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/kubeadm/flannel/Dockerfile b/kubeadm/flannel/Dockerfile
index 561976e7..9525bd52 100644
--- a/kubeadm/flannel/Dockerfile
+++ b/kubeadm/flannel/Dockerfile
@@ -7,8 +7,8 @@ SHELL ["powershell", "-NoLogo", "-Command", "$ErrorActionPreference = 'Stop'; $P
 ARG cniVersion
 
 # Stuck on a prerelease flannel until https://github.com/coreos/flannel/issues/1231 is resolved
-RUN mkdir C:\flannel; \
-  pushd C:\flannel; \
+RUN mkdir -force C:\k\flannel; \
+  pushd C:\k\flannel; \
   curl.exe -LO https://github.com/benmoss/flannel/releases/download/v0.12.0-rc1/flanneld.exe
 
 RUN mkdir C:\cni; \
diff --git a/kubeadm/flannel/flannel-host-gw.yml b/kubeadm/flannel/flannel-host-gw.yml
index 009a16c2..1a7221dd 100644
--- a/kubeadm/flannel/flannel-host-gw.yml
+++ b/kubeadm/flannel/flannel-host-gw.yml
@@ -29,7 +29,7 @@ data:
 
     cp -force /etc/kube-flannel/net-conf.json /host/etc/kube-flannel
     cp -force -recurse /cni/* /host/opt/cni/bin
-    cp -force /flannel/flanneld.exe /host/k/flannel/flanneld.exe
+    cp -force /k/flannel/flanneld.exe /host/k/flannel/flanneld.exe
     cp -force /kube-proxy/kubeconfig.conf /host/k/flannel/kubeconfig.yml
     cp -force /var/run/secrets/kubernetes.io/serviceaccount/* /host/k/flannel/var/run/secrets/kubernetes.io/serviceaccount/
     wins cli process run --path /k/flannel/flanneld.exe --args "--kube-subnet-mgr --kubeconfig-file /k/flannel/kubeconfig.yml" --envs "POD_NAME=$env:POD_NAME POD_NAMESPACE=$env:POD_NAMESPACE"
diff --git a/kubeadm/flannel/flannel-overlay.yml b/kubeadm/flannel/flannel-overlay.yml
index b08849ad..e5f654d3 100644
--- a/kubeadm/flannel/flannel-overlay.yml
+++ b/kubeadm/flannel/flannel-overlay.yml
@@ -28,7 +28,7 @@ data:
 
     cp -force /etc/kube-flannel/net-conf.json /host/etc/kube-flannel
     cp -force -recurse /cni/* /host/opt/cni/bin
-    cp -force /flannel/flanneld.exe /host/k/flannel/flanneld.exe
+    cp -force /k/flannel/flanneld.exe /host/k/flannel/flanneld.exe
     cp -force /kube-proxy/kubeconfig.conf /host/k/flannel/kubeconfig.yml
     cp -force /var/run/secrets/kubernetes.io/serviceaccount/* /host/k/flannel/var/run/secrets/kubernetes.io/serviceaccount/
     wins cli process run --path /k/flannel/flanneld.exe --args "--kube-subnet-mgr --kubeconfig-file /k/flannel/kubeconfig.yml" --envs "POD_NAME=$env:POD_NAME POD_NAMESPACE=$env:POD_NAMESPACE"
diff --git a/kubeadm/kube-proxy/Dockerfile b/kubeadm/kube-proxy/Dockerfile
index 0cfc4a53..b3f7a2e6 100644
--- a/kubeadm/kube-proxy/Dockerfile
+++ b/kubeadm/kube-proxy/Dockerfile
@@ -6,8 +6,8 @@ SHELL ["powershell", "-NoLogo", "-Command", "$ErrorActionPreference = 'Stop'; $P
 
 ARG k8sVersion
 
-RUN mkdir C:\kube-proxy; \
-  pushd C:\kube-proxy; \
+RUN mkdir -force C:\k\kube-proxy; \
+  pushd C:\k\kube-proxy; \
   curl.exe -LO https://dl.k8s.io/v${env:k8sVersion}/bin/windows/amd64/kube-proxy.exe
 
 RUN mkdir C:\utils; \
diff --git a/kubeadm/kube-proxy/kube-proxy.yml b/kubeadm/kube-proxy/kube-proxy.yml
index 996607e8..cf89e508 100644
--- a/kubeadm/kube-proxy/kube-proxy.yml
+++ b/kubeadm/kube-proxy/kube-proxy.yml
@@ -5,7 +5,7 @@ data:
     mkdir -force /host/var/lib/kube-proxy/var/run/secrets/kubernetes.io/serviceaccount
     mkdir -force /host/k/kube-proxy
 
-    cp -force /kube-proxy/* /host/k/kube-proxy
+    cp -force /k/kube-proxy/* /host/k/kube-proxy
     cp -force /var/lib/kube-proxy/* /host/var/lib/kube-proxy
     cp -force /var/run/secrets/kubernetes.io/serviceaccount/* /host/var/lib/kube-proxy/var/run/secrets/kubernetes.io/serviceaccount #FIXME?
 

From a9b407af594ab770581d58d17c4d05548a138241 Mon Sep 17 00:00:00 2001
From: Ben Moss <bmoss@pivotal.io>
Date: Wed, 26 Feb 2020 11:07:20 -0500
Subject: [PATCH 14/18] Replace the kube-proxy image tag with a placeholder

Docs will specify users need to substitute this with their k8s version,
this avoids people accidentally using the wrong version since now it
will break.
---
 kubeadm/kube-proxy/kube-proxy.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/kubeadm/kube-proxy/kube-proxy.yml b/kubeadm/kube-proxy/kube-proxy.yml
index cf89e508..171ab5a8 100644
--- a/kubeadm/kube-proxy/kube-proxy.yml
+++ b/kubeadm/kube-proxy/kube-proxy.yml
@@ -58,7 +58,7 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: status.podIP
-        image: sigwindowstools/kube-proxy:1.17.3
+        image: sigwindowstools/kube-proxy:VERSION
         name: kube-proxy
         volumeMounts:
         - name: host

From f235dd177e7680f6e0a3192d08f2e67bb9ffe2d1 Mon Sep 17 00:00:00 2001
From: Ben Moss <bmoss@pivotal.io>
Date: Wed, 26 Feb 2020 13:43:32 -0500
Subject: [PATCH 15/18] Add a script to publish kube-proxy images

---
 kubeadm/kube-proxy/Dockerfile |  4 ++--
 kubeadm/kube-proxy/publish.sh | 32 ++++++++++++++++++++++++++++++++
 2 files changed, 34 insertions(+), 2 deletions(-)
 create mode 100755 kubeadm/kube-proxy/publish.sh

diff --git a/kubeadm/kube-proxy/Dockerfile b/kubeadm/kube-proxy/Dockerfile
index b3f7a2e6..341199da 100644
--- a/kubeadm/kube-proxy/Dockerfile
+++ b/kubeadm/kube-proxy/Dockerfile
@@ -1,4 +1,4 @@
-ARG k8sVersion="1.17.3"
+ARG k8sVersion="v1.17.3"
 ARG servercoreTag="ltsc2019"
 
 FROM mcr.microsoft.com/windows/servercore:${servercoreTag}
@@ -8,7 +8,7 @@ ARG k8sVersion
 
 RUN mkdir -force C:\k\kube-proxy; \
   pushd C:\k\kube-proxy; \
-  curl.exe -LO https://dl.k8s.io/v${env:k8sVersion}/bin/windows/amd64/kube-proxy.exe
+  curl.exe -LO https://dl.k8s.io/${env:k8sVersion}/bin/windows/amd64/kube-proxy.exe
 
 RUN mkdir C:\utils; \
   curl.exe -Lo C:\utils\wins.exe https://github.com/rancher/wins/releases/download/v0.0.4/wins.exe; \
diff --git a/kubeadm/kube-proxy/publish.sh b/kubeadm/kube-proxy/publish.sh
new file mode 100755
index 00000000..41aaccd7
--- /dev/null
+++ b/kubeadm/kube-proxy/publish.sh
@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+
+set -e
+
+INITIAL_MAJOR=1
+INITIAL_MINOR=17
+INITIAL_PATCH=0
+
+dir=$(cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)
+
+# https://github.com/google/go-containerregistry/tree/master/cmd/crane
+versions=$(crane ls k8s.gcr.io/kube-proxy 2>/dev/null)
+for version in $versions; do
+  if [[ $version =~ alpha|beta|rc ]]; then
+    continue
+  fi
+  major=`echo $version | cut -d. -f1 | tr -d v`
+  minor=`echo $version | cut -d. -f2`
+  patch=`echo $version | cut -d. -f3`
+  if [ $major -lt $INITIAL_MAJOR ]; then
+    continue
+  fi
+  if [ $minor -lt $INITIAL_MINOR ]; then
+    continue
+  fi
+
+  echo "building $major.$minor.$patch"
+
+  docker build --pull --build-arg k8sVersion="$version" --tag sigwindowstools/kube-proxy:$version $dir
+done
+
+docker push sigwindowstools/kube-proxy

From def34f5bb6b2f0d763b799efdf2bbfbd0b0f7ca7 Mon Sep 17 00:00:00 2001
From: Ben Moss <bmoss@pivotal.io>
Date: Wed, 26 Feb 2020 10:48:36 -0500
Subject: [PATCH 16/18] Merge installing nssm into the PrepareNode script

No reason to run two scripts when one will do
---
 kubeadm/scripts/InstallNssm.ps1 | 19 -------------------
 kubeadm/scripts/PrepareNode.ps1 | 27 ++++++++++++++++++++-------
 2 files changed, 20 insertions(+), 26 deletions(-)
 delete mode 100644 kubeadm/scripts/InstallNssm.ps1

diff --git a/kubeadm/scripts/InstallNssm.ps1 b/kubeadm/scripts/InstallNssm.ps1
deleted file mode 100644
index d6233965..00000000
--- a/kubeadm/scripts/InstallNssm.ps1
+++ /dev/null
@@ -1,19 +0,0 @@
-$ErrorActionPreference = "Stop"
-$global:NssmInstallDirectory = "$env:ProgramFiles\nssm"
-$arch = "win32"
-if ([Environment]::Is64BitOperatingSystem) {
-    $arch = "win64"
-}
-
-mkdir -Force $global:NssmInstallDirectory
-curl.exe -Lo nssm.zip https://nssm.cc/release/nssm-2.24.zip
-tar C $global:NssmInstallDirectory -xvf .\nssm.zip --strip-components 2 */$arch/*.exe
-Remove-Item -Force .\nssm.zip
-
-$env:path += ";$global:NssmInstallDirectory"
-$newPath = "$global:NssmInstallDirectory;" +
-[Environment]::GetEnvironmentVariable("PATH",
-[EnvironmentVariableTarget]::Machine)
-
-[Environment]::SetEnvironmentVariable("PATH", $newPath,
-[EnvironmentVariableTarget]::Machine)
diff --git a/kubeadm/scripts/PrepareNode.ps1 b/kubeadm/scripts/PrepareNode.ps1
index f94dbb20..29e75af7 100644
--- a/kubeadm/scripts/PrepareNode.ps1
+++ b/kubeadm/scripts/PrepareNode.ps1
@@ -5,7 +5,7 @@ Assists with preparing a Windows VM prior to calling kubeadm join
 .DESCRIPTION
 This script assists with joining a Windows node to a cluster.
 - Downloads Kubernetes binaries (kubelet, kubeadm) at the version specified
-- Registers wins as a service in order to run kube-proxy and cni as DaemonSets. More info here: 
+- Registers wins as a service in order to run kube-proxy and cni as DaemonSets.
 - Registers kubelet as an nssm service. More info on nssm: https://nssm.cc/
 
 .PARAMETER KubernetesVersion
@@ -50,7 +50,7 @@ DownloadFile $kubeletBinPath https://dl.k8s.io/$KubernetesVersion/bin/windows/am
 DownloadFile "$global:KubernetesPath\kubeadm.exe" https://dl.k8s.io/$KubernetesVersion/bin/windows/amd64/kubeadm.exe
 DownloadFile "$global:KubernetesPath\wins.exe" https://github.com/rancher/wins/releases/download/v0.0.4/wins.exe
 
-#Create host network to allow kubelet to schedule hostNetwork pods
+# Create host network to allow kubelet to schedule hostNetwork pods
 Write-Host "Creating Docker host network"
 docker network create -d nat host
 
@@ -71,12 +71,25 @@ $cmd = "C:\k\kubelet.exe $global:KubeletArgs --cert-dir=$env:SYSTEMDRIVE\var\lib
 Invoke-Expression $cmd'
 Set-Content -Path $global:StartKubeletScript -Value $StartKubeletFileContent
 
-Write-Host "Registering kubelet service"
-Get-Command nssm
-if (!$?) {
-    Write-Error "NSSM is not installed."
-    exit 1
+Write-Host "Installing nssm"
+$global:NssmInstallDirectory = "$env:ProgramFiles\nssm"
+$arch = "win32"
+if ([Environment]::Is64BitOperatingSystem) {
+    $arch = "win64"
 }
+
+mkdir -Force $global:NssmInstallDirectory
+curl.exe -Lo nssm.zip https://nssm.cc/release/nssm-2.24.zip
+tar C $global:NssmInstallDirectory -xvf .\nssm.zip --strip-components 2 */$arch/*.exe
+Remove-Item -Force .\nssm.zip
+
+$env:path += ";$global:NssmInstallDirectory"
+$newPath = "$global:NssmInstallDirectory;" +
+[Environment]::GetEnvironmentVariable("PATH", [EnvironmentVariableTarget]::Machine)
+
+[Environment]::SetEnvironmentVariable("PATH", $newPath, [EnvironmentVariableTarget]::Machine)
+
+Write-Host "Registering kubelet service"
 nssm install kubelet $global:Powershell $global:PowershellArgs $global:StartKubeletScript
 nssm set kubelet DependOnService docker
 

From a127bd42619f32f70124ba27c54b2fc96042a2f1 Mon Sep 17 00:00:00 2001
From: Ben Moss <bmoss@pivotal.io>
Date: Wed, 26 Feb 2020 16:28:14 -0500
Subject: [PATCH 17/18] Switch to MS mirror of nssm

Apparently the official site has some stability issues
---
 kubeadm/scripts/PrepareNode.ps1 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/kubeadm/scripts/PrepareNode.ps1 b/kubeadm/scripts/PrepareNode.ps1
index 29e75af7..1d36697b 100644
--- a/kubeadm/scripts/PrepareNode.ps1
+++ b/kubeadm/scripts/PrepareNode.ps1
@@ -79,7 +79,7 @@ if ([Environment]::Is64BitOperatingSystem) {
 }
 
 mkdir -Force $global:NssmInstallDirectory
-curl.exe -Lo nssm.zip https://nssm.cc/release/nssm-2.24.zip
+curl.exe -Lo nssm.zip https://k8stestinfrabinaries.blob.core.windows.net/nssm-mirror/nssm-2.24.zip
 tar C $global:NssmInstallDirectory -xvf .\nssm.zip --strip-components 2 */$arch/*.exe
 Remove-Item -Force .\nssm.zip
 

From c73c053a9c88f42214cf40bc5ff1365c630eeb64 Mon Sep 17 00:00:00 2001
From: Ben Moss <bmoss@pivotal.io>
Date: Wed, 26 Feb 2020 17:58:32 -0500
Subject: [PATCH 18/18] Hide curl progress bar

This seems to mess up cloudbase-init somehow, the stderr from curl
causes a "RemoteException"/"NativeCommandError"
---
 kubeadm/scripts/PrepareNode.ps1 | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/kubeadm/scripts/PrepareNode.ps1 b/kubeadm/scripts/PrepareNode.ps1
index 1d36697b..054af411 100644
--- a/kubeadm/scripts/PrepareNode.ps1
+++ b/kubeadm/scripts/PrepareNode.ps1
@@ -24,7 +24,7 @@ $ErrorActionPreference = 'Stop'
 
 function DownloadFile($destination, $source) {
     Write-Host("Downloading $source to $destination")
-    curl.exe --fail -Lo $destination $source
+    curl.exe --silent --fail -Lo $destination $source
 
     if (!$?) {
         Write-Error "Download $source failed"
@@ -40,6 +40,7 @@ $global:Powershell = (Get-Command powershell).Source
 $global:PowershellArgs = "-ExecutionPolicy Bypass -NoProfile"
 $global:KubernetesPath = "$env:SystemDrive\k"
 $global:StartKubeletScript = "$global:KubernetesPath\StartKubelet.ps1"
+$global:NssmInstallDirectory = "$env:ProgramFiles\nssm"
 $kubeletBinPath = "$global:KubernetesPath\kubelet.exe"
 
 mkdir -force "$global:KubernetesPath"
@@ -72,14 +73,13 @@ Invoke-Expression $cmd'
 Set-Content -Path $global:StartKubeletScript -Value $StartKubeletFileContent
 
 Write-Host "Installing nssm"
-$global:NssmInstallDirectory = "$env:ProgramFiles\nssm"
 $arch = "win32"
 if ([Environment]::Is64BitOperatingSystem) {
     $arch = "win64"
 }
 
 mkdir -Force $global:NssmInstallDirectory
-curl.exe -Lo nssm.zip https://k8stestinfrabinaries.blob.core.windows.net/nssm-mirror/nssm-2.24.zip
+DownloadFile nssm.zip https://k8stestinfrabinaries.blob.core.windows.net/nssm-mirror/nssm-2.24.zip
 tar C $global:NssmInstallDirectory -xvf .\nssm.zip --strip-components 2 */$arch/*.exe
 Remove-Item -Force .\nssm.zip