From 46023660e9d5d775d28bc43c54a81d99c3df601c Mon Sep 17 00:00:00 2001 From: Travis Hall Date: Thu, 21 Mar 2019 12:28:44 -0700 Subject: [PATCH] Move windows release (#315) * Revert "Add support for syncing Windows binaries" This reverts commit a2a5be6e2c756f533f37f5201bcbd21287a0d2ef. * Remove Windows jobs to their own release With the Windows and Linux jobs intermixed, we would no longer be able to distribute precompiled releases. Even with two precompiled releases (one for Linux and one for Windows), we'd be unable to differentiate which release the instance group pulls from. --- config/blobs.yml | 20 --- jobs/flanneld-windows/monit | 11 -- jobs/flanneld-windows/spec | 30 ---- .../templates/bin/flanneld_ctl.ps1.erb | 86 ----------- .../templates/config/etcd-ca.crt.erb | 2 - .../templates/config/etcd-client.crt.erb | 2 - .../templates/config/etcd-client.key.erb | 2 - jobs/kube-proxy-windows/monit | 11 -- jobs/kube-proxy-windows/spec | 38 ----- .../templates/bin/kube_proxy_ctl.ps1.erb | 51 ------- .../templates/config/ca.pem.erb | 1 - .../templates/config/config.yml.erb | 13 -- .../templates/config/kubeconfig.erb | 17 --- jobs/kubelet-windows/monit | 11 -- jobs/kubelet-windows/spec | 64 -------- .../templates/bin/drain.ps1.erb | 69 --------- .../templates/bin/kubelet_ctl.ps1.erb | 144 ------------------ .../templates/bin/post-start.ps1 | 29 ---- .../templates/config/Dockerfile | 3 - .../templates/config/apiserver-ca.pem.erb | 1 - .../templates/config/cloud-provider.ini.erb | 1 - .../templates/config/kubeconfig-drain.erb | 17 --- .../templates/config/kubeconfig.erb | 17 --- .../config/kubelet-client-ca.pem.erb | 1 - .../templates/config/kubelet-key.pem.erb | 1 - .../templates/config/kubelet.pem.erb | 2 - .../templates/config/kubeletconfig.yml.erb | 1 - .../templates/config/openstack-ca.crt.erb | 1 - .../templates/config/service_key.json.erb | 1 - packages/cni-windows/packaging | 8 - packages/cni-windows/spec | 6 - packages/flanneld-windows/packaging | 9 -- packages/flanneld-windows/spec | 8 - packages/kubernetes-windows/packaging | 10 -- packages/kubernetes-windows/spec | 6 - scripts/download_k8s_binaries | 70 +++------ src/exiter.ps1 | 6 - 37 files changed, 22 insertions(+), 748 deletions(-) delete mode 100644 jobs/flanneld-windows/monit delete mode 100644 jobs/flanneld-windows/spec delete mode 100644 jobs/flanneld-windows/templates/bin/flanneld_ctl.ps1.erb delete mode 100644 jobs/flanneld-windows/templates/config/etcd-ca.crt.erb delete mode 100644 jobs/flanneld-windows/templates/config/etcd-client.crt.erb delete mode 100644 jobs/flanneld-windows/templates/config/etcd-client.key.erb delete mode 100644 jobs/kube-proxy-windows/monit delete mode 100644 jobs/kube-proxy-windows/spec delete mode 100644 jobs/kube-proxy-windows/templates/bin/kube_proxy_ctl.ps1.erb delete mode 100644 jobs/kube-proxy-windows/templates/config/ca.pem.erb delete mode 100644 jobs/kube-proxy-windows/templates/config/config.yml.erb delete mode 100644 jobs/kube-proxy-windows/templates/config/kubeconfig.erb delete mode 100644 jobs/kubelet-windows/monit delete mode 100644 jobs/kubelet-windows/spec delete mode 100644 jobs/kubelet-windows/templates/bin/drain.ps1.erb delete mode 100644 jobs/kubelet-windows/templates/bin/kubelet_ctl.ps1.erb delete mode 100644 jobs/kubelet-windows/templates/bin/post-start.ps1 delete mode 100644 jobs/kubelet-windows/templates/config/Dockerfile delete mode 100644 jobs/kubelet-windows/templates/config/apiserver-ca.pem.erb delete mode 120000 jobs/kubelet-windows/templates/config/cloud-provider.ini.erb delete mode 100644 jobs/kubelet-windows/templates/config/kubeconfig-drain.erb delete mode 100644 jobs/kubelet-windows/templates/config/kubeconfig.erb delete mode 100644 jobs/kubelet-windows/templates/config/kubelet-client-ca.pem.erb delete mode 100644 jobs/kubelet-windows/templates/config/kubelet-key.pem.erb delete mode 100644 jobs/kubelet-windows/templates/config/kubelet.pem.erb delete mode 100644 jobs/kubelet-windows/templates/config/kubeletconfig.yml.erb delete mode 120000 jobs/kubelet-windows/templates/config/openstack-ca.crt.erb delete mode 120000 jobs/kubelet-windows/templates/config/service_key.json.erb delete mode 100644 packages/cni-windows/packaging delete mode 100644 packages/cni-windows/spec delete mode 100644 packages/flanneld-windows/packaging delete mode 100644 packages/flanneld-windows/spec delete mode 100644 packages/kubernetes-windows/packaging delete mode 100644 packages/kubernetes-windows/spec delete mode 100644 src/exiter.ps1 diff --git a/config/blobs.yml b/config/blobs.yml index 5f8b822ce..1cc51e62a 100644 --- a/config/blobs.yml +++ b/config/blobs.yml @@ -14,10 +14,6 @@ cni/cni-plugins-amd64-v0.7.1.tgz: size: 17108856 object_id: 50dc58eb-107d-483a-4e37-5a243ffed520 sha: fb29e20401d3e9598a1d8e8d7992970a36de5e05 -cni/cni-plugins-windows-amd64-c74e0e996.tgz: - size: 9678154 - object_id: ed601155-ac1f-4682-684d-76cd6eaa1a95 - sha: 02c4d67a17a28c26cc471e56bf01dcacd104b346 cni/util-linux_2.27.1-6ubuntu3_amd64.deb: size: 847730 object_id: 27eb3ec3-220f-4a25-6c4f-ec219c37ac8d @@ -58,10 +54,6 @@ flannel-v0.11.0-linux-amd64.tar.gz: size: 9565743 object_id: 83064ebe-b45d-4d41-694a-b1e61a781250 sha: 3024034b2202d5940acf5353037161ea1770a9c5 -flannel-v0.11.0-windows-amd64.tar.gz: - size: 9488010 - object_id: f1a21fd3-b6c3-4f70-48c4-017e55a061c3 - sha: 71cb61a487c40df8eeb4d2bd68d7e891b51b44bb ipset/ipset_6.20.1-1_amd64.deb: size: 34156 object_id: 5dfcce8f-366e-49f0-59e6-9e2d48178750 @@ -98,18 +90,6 @@ kubernetes-1.13.4/kubelet: size: 113031192 object_id: 0d7db619-15e6-4f17-6469-88058024a62b sha: 6c43640cd0f4494b4406296fe35579027176b2f7 -kubernetes-windows-1.13.4/kube-proxy.exe: - size: 34466816 - object_id: 56e1ecb1-47aa-467c-60a4-4c2557c411c8 - sha: e08e17e6370a2ef53c3d6a74cc89ea28d083b55d -kubernetes-windows-1.13.4/kubectl.exe: - size: 39670272 - object_id: 367d644b-470f-425d-6a24-bb8faa905bd3 - sha: fa5c7a0d3c7ae7edb8d6fb276dcfcf9a232b17ec -kubernetes-windows-1.13.4/kubelet.exe: - size: 102774272 - object_id: 458f8def-b633-4ead-48b4-8e7720cc609e - sha: 30c8cab80e5940503f9cdd36b51c56f47f0411d2 libmnl0_1.0.3-3.deb: size: 11416 object_id: 245e085f-1ad2-447d-67bb-1894b960c571 diff --git a/jobs/flanneld-windows/monit b/jobs/flanneld-windows/monit deleted file mode 100644 index 7eaaf469d..000000000 --- a/jobs/flanneld-windows/monit +++ /dev/null @@ -1,11 +0,0 @@ -{ - "processes": [ - { - "name": "flanneld", - "executable": "powershell", - "args": ["C:\\var\\vcap\\jobs\\flanneld-windows\\bin\\flanneld_ctl.ps1"], - "env": {} - } - ] -} - diff --git a/jobs/flanneld-windows/spec b/jobs/flanneld-windows/spec deleted file mode 100644 index 1c8d3a6fd..000000000 --- a/jobs/flanneld-windows/spec +++ /dev/null @@ -1,30 +0,0 @@ ---- -name: flanneld-windows - -templates: - bin/flanneld_ctl.ps1.erb: bin/flanneld_ctl.ps1 - config/etcd-ca.crt.erb: config/etcd-ca.crt - config/etcd-client.crt.erb: config/etcd-client.crt - config/etcd-client.key.erb: config/etcd-client.key - -packages: -- flanneld-windows -- cni-windows - -properties: - pod-network-cidr: - description: The pod networking cidr for pod network overlay - default: "10.200.0.0/16" - backend-type: - description: The network backend to use - default: "win-overlay" - -consumes: -- name: etcd - type: etcd - -provides: -- name: flanneld-windows - type: flanneld-windows - properties: - - backend-type diff --git a/jobs/flanneld-windows/templates/bin/flanneld_ctl.ps1.erb b/jobs/flanneld-windows/templates/bin/flanneld_ctl.ps1.erb deleted file mode 100644 index 2e8c93c6e..000000000 --- a/jobs/flanneld-windows/templates/bin/flanneld_ctl.ps1.erb +++ /dev/null @@ -1,86 +0,0 @@ -trap { $host.SetShouldExit(1) } - -<%- - def get_url(server, port) - if link('etcd').p('etcd.dns_suffix', false) != false - node_name = "#{server.name.gsub('_','-')}-#{server.index}" - return "https://#{node_name}.#{link('etcd').p('etcd.dns_suffix')}:#{port}" - else - return "https://#{server.address}:#{port}" - end - end --%> -<%- - def get_network_name - if p("backend-type") == "win-overlay" - "vxlan0" - else - "cbr0" - end - end --%> - -function start_flanneld { - <% etcd_endpoints = link('etcd').instances.map { |server| get_url(server, 2379) }.join(",") %> - - mkdir -force /etc/cni/net.d - $confFile= ' -{ - "name": "<%= get_network_name %>", - "plugins": [ - { - "type": "flannel", - "delegate": { - "hairpinMode": true, - "isDefaultGateway": true, - "type": "<%= p('backend-type') %>", - "dns": { - "nameservers": ["10.100.200.10"], - "search": ["svc.cluster.local"] - }, - "policies": [ - { - "Name": "EndpointPolicy", - "Value": { - "Type": "OutBoundNAT", - "ExceptionList": [ - "<%= p('pod-network-cidr') %>", - "10.100.200.0/12", - "<%= spec.ip.split(".")[0...-1].append(0).join(".") %>/24" - ] - } - }, - { - "Name": "EndpointPolicy", - "Value": { - "Type": "ROUTE", - "DestinationPrefix": "10.100.200.0/24", - "NeedEncap": true - } - }, - { - "Name": "EndpointPolicy", - "Value": { - "Type": "ROUTE", - "DestinationPrefix": "<%= spec.ip %>/32", - "NeedEncap": true - } - } - ] - } - } - ] -} -' - Set-Content -Path /etc/cni/net.d/50-flannel.conflist -Value $confFile - - - /var/vcap/packages/flanneld-windows/flanneld.exe ` - --etcd-endpoints=<%= etcd_endpoints %> ` - --ip-masq ` - --etcd-certfile=/var/vcap/jobs/flanneld-windows/config/etcd-client.crt ` - --etcd-keyfile=/var/vcap/jobs/flanneld-windows/config/etcd-client.key ` - --etcd-cafile=/var/vcap/jobs/flanneld-windows/config/etcd-ca.crt -} - -start_flanneld diff --git a/jobs/flanneld-windows/templates/config/etcd-ca.crt.erb b/jobs/flanneld-windows/templates/config/etcd-ca.crt.erb deleted file mode 100644 index 5c42f93b6..000000000 --- a/jobs/flanneld-windows/templates/config/etcd-ca.crt.erb +++ /dev/null @@ -1,2 +0,0 @@ -<%= link('etcd').p('tls.etcdctl.ca') %> - diff --git a/jobs/flanneld-windows/templates/config/etcd-client.crt.erb b/jobs/flanneld-windows/templates/config/etcd-client.crt.erb deleted file mode 100644 index 83ba4bf73..000000000 --- a/jobs/flanneld-windows/templates/config/etcd-client.crt.erb +++ /dev/null @@ -1,2 +0,0 @@ -<%= link('etcd').p('tls.etcdctl.certificate') %> - diff --git a/jobs/flanneld-windows/templates/config/etcd-client.key.erb b/jobs/flanneld-windows/templates/config/etcd-client.key.erb deleted file mode 100644 index 48964f297..000000000 --- a/jobs/flanneld-windows/templates/config/etcd-client.key.erb +++ /dev/null @@ -1,2 +0,0 @@ -<%= link('etcd').p('tls.etcdctl.private_key') %> - diff --git a/jobs/kube-proxy-windows/monit b/jobs/kube-proxy-windows/monit deleted file mode 100644 index b710d030b..000000000 --- a/jobs/kube-proxy-windows/monit +++ /dev/null @@ -1,11 +0,0 @@ -{ - "processes": [ - { - "name": "kube-proxy", - "executable": "powershell", - "args": ["C:\\var\\vcap\\jobs\\kube-proxy-windows\\bin\\kube_proxy_ctl.ps1"], - "env": {} - } - ] -} - diff --git a/jobs/kube-proxy-windows/spec b/jobs/kube-proxy-windows/spec deleted file mode 100644 index cdccbaf42..000000000 --- a/jobs/kube-proxy-windows/spec +++ /dev/null @@ -1,38 +0,0 @@ ---- -name: kube-proxy-windows - -templates: - bin/kube_proxy_ctl.ps1.erb: bin/kube_proxy_ctl.ps1 - config/kubeconfig.erb: config/kubeconfig - config/config.yml.erb: config/config.yml - config/ca.pem.erb: config/ca.pem - -packages: -- kubernetes-windows - -properties: - api-token: - description: The password for the kube-proxy user - cloud-provider: - description: The type of cloud-provider that is being deployed - tls.kubernetes: - description: Certificate and private key for the Kubernetes master - kube-proxy-configuration: - description: The Kube-proxy will load its initial configuration from this. - Omit this to use the built-in default configuration values. - Command-line flags override configuration. - This is the recommended way to configure kube-proxy as the command line flags for kube-proxy are being deprecated. - example: | - kube-proxy-configuration: - feature-gates: - CPUManager: true - DryRun: false - cleanup: false - k8s-args: - description: Pass-through options for Kubernetes runtime arguments. See docs https://kubernetes.io/docs/reference/command-line-tools-reference/kube-proxy/ for reference. - example: | - k8s-args: - feature-gates: - CPUManager: true - DryRun: false - cleanup: false diff --git a/jobs/kube-proxy-windows/templates/bin/kube_proxy_ctl.ps1.erb b/jobs/kube-proxy-windows/templates/bin/kube_proxy_ctl.ps1.erb deleted file mode 100644 index da6ff5caf..000000000 --- a/jobs/kube-proxy-windows/templates/bin/kube_proxy_ctl.ps1.erb +++ /dev/null @@ -1,51 +0,0 @@ -trap { $host.SetShouldExit(1) } - -function ensure_kubelet_is_running { - curl.exe --fail http://localhost:10248/healthz - if (-not $?) { - throw "kubelet is not available" - } -} - -function start_kube_proxy { - $network = Get-HnsNetwork | ? Type -Eq L2Bridge - $env:KUBE_NETWORK = $network.Name - - C:\var\vcap\packages\kubernetes-windows\bin\kube-proxy.exe -v 5 --config /var/vcap/jobs/kube-proxy-windows/config/config.yml -} - -function check_for_networking { - $subnetConfig="/run/flannel/subnet.env" - - if (-not ([System.IO.File]::Exists($subnetConfig))) - { - throw "$subnetConfig does not exist, waiting for flannel initialization" - } -} - -function misc_setup { - # Prestart doesn't get run again on recreate, so need to do this in ctl - Set-NetFirewallProfile -Profile Domain, Public, Private -Enabled False - - <% if_p('cloud-provider') do |cloud_provider| %> - <% if cloud_provider == "vsphere" %> - # Override the hostname to work around - # vSphere cloud provider ignoring hostname override - # and kubernetes requiring all-lowercase node names - $ComputerName = (cat C:\var\vcap\bosh\settings.json | ConvertFrom-Json).agent_id - Set-ItemProperty -Path "HKLM:\SYSTEM\CurrentControlSet\Services\Tcpip\Parameters" -name "Hostname" -value $ComputerName - <% end %> - <% end %> - - # Needed until https://github.com/kubernetes/kubernetes/pull/71147 is merged - mkdir -force /sys/class/dmi/id - (wmic csproduct get IdentifyingNumber).Split([Environment]::Newline)[2] | Out-File -Encoding ASCII /sys/class/dmi/id/product_serial -} - -function main() { - misc_setup - check_for_networking - start_kube_proxy -} - -main diff --git a/jobs/kube-proxy-windows/templates/config/ca.pem.erb b/jobs/kube-proxy-windows/templates/config/ca.pem.erb deleted file mode 100644 index b8b5e229b..000000000 --- a/jobs/kube-proxy-windows/templates/config/ca.pem.erb +++ /dev/null @@ -1 +0,0 @@ -<%= p('tls.kubernetes.ca') %> diff --git a/jobs/kube-proxy-windows/templates/config/config.yml.erb b/jobs/kube-proxy-windows/templates/config/config.yml.erb deleted file mode 100644 index e04506a67..000000000 --- a/jobs/kube-proxy-windows/templates/config/config.yml.erb +++ /dev/null @@ -1,13 +0,0 @@ -<% - hostname = "" - if_p('cloud-provider') do |cloud_provider| - if cloud_provider == "gce" || cloud_provider == "azure" || cloud_provider == "vsphere" - else - hostname = spec.ip - end - end.else do - hostname = spec.ip - end -%> -<% require 'yaml' %><%= p('kube-proxy-configuration').to_yaml %> -hostnameOverride: <%= hostname %> diff --git a/jobs/kube-proxy-windows/templates/config/kubeconfig.erb b/jobs/kube-proxy-windows/templates/config/kubeconfig.erb deleted file mode 100644 index a1d3cb3f8..000000000 --- a/jobs/kube-proxy-windows/templates/config/kubeconfig.erb +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: v1 -kind: Config -clusters: -- cluster: - certificate-authority: "C:\\var\\vcap\\jobs\\kube-proxy-windows\\config\\ca.pem" - server: https://master.cfcr.internal:8443 - name: kubernetes -contexts: -- context: - cluster: kubernetes - user: kube-proxy - name: kube-proxy -current-context: kube-proxy -users: -- name: kube-proxy - user: - token: "<%= p("api-token") %>" diff --git a/jobs/kubelet-windows/monit b/jobs/kubelet-windows/monit deleted file mode 100644 index 5d639d930..000000000 --- a/jobs/kubelet-windows/monit +++ /dev/null @@ -1,11 +0,0 @@ -{ - "processes": [ - { - "name": "kubelet", - "executable": "powershell", - "args": ["C:\\var\\vcap\\jobs\\kubelet-windows\\bin\\kubelet_ctl.ps1"], - "env": {} - } - ] -} - diff --git a/jobs/kubelet-windows/spec b/jobs/kubelet-windows/spec deleted file mode 100644 index 16ac362a1..000000000 --- a/jobs/kubelet-windows/spec +++ /dev/null @@ -1,64 +0,0 @@ -name: kubelet-windows -templates: - bin/drain.ps1.erb: bin/drain.ps1 - bin/kubelet_ctl.ps1.erb: bin/kubelet_ctl.ps1 - config/apiserver-ca.pem.erb: config/apiserver-ca.pem - config/cloud-provider.ini.erb: config/cloud-provider.ini - config/Dockerfile: config/Dockerfile - config/kubeconfig-drain.erb: config/kubeconfig-drain - config/kubeconfig.erb: config/kubeconfig - config/kubelet-client-ca.pem.erb: config/kubelet-client-ca.pem - config/kubelet-key.pem.erb: config/kubelet-key.pem - config/kubelet.pem.erb: config/kubelet.pem - config/kubeletconfig.yml.erb: config/kubeletconfig.yml - config/openstack-ca.crt.erb: config/openstack-ca.crt - config/service_key.json.erb: config/service_key.json -packages: -- kubernetes-windows -- cni-windows -properties: - api-token: - description: The token to access Kubernetes API - cloud-provider: - description: "The type of cloud-provider that is being deployed" - drain-api-token: - description: The token to access Kubernetes API used to drain the kubelet. - http_proxy: - description: http_proxy env var for cloud provider interactions, i.e. for the - kubelet - https_proxy: - description: https_proxy env var for cloud provider interactions, i.e. for the - kubelet - kubelet-configuration: - description: The Kubelet will load its initial configuration from this. - Omit this to use the built-in default configuration values. - Command-line flags override configuration. - k8s-args: - description: "Pass-through options for Kubernetes runtime arguments. See docs https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/ for reference." - example: | - k8s-args: - address: 10.0.0.1 - docker-only: null - no_proxy: - description: no_proxy env var for cloud provider interactions, i.e. for the kubelet - labels: - description: ' Labels to add when registering the node - in the cluster. Labels must be key=value pairs separated by '',''.' - tls.kubelet: - description: Certificate and private key for the Kubernetes worker - parameters: - duration: {} - key_length: {} - type: {} - tls.kubelet-client-ca.certificate: - description: CA certificate of the authority granting access to kubelet server - tls.kubernetes: - description: Certificate and private key for the Kubernetes master -consumes: -- name: cloud-provider - optional: true - type: cloud-provider -provides: -- name: kubernetes-workers - type: kubernetes-workers - diff --git a/jobs/kubelet-windows/templates/bin/drain.ps1.erb b/jobs/kubelet-windows/templates/bin/drain.ps1.erb deleted file mode 100644 index dc2731a7b..000000000 --- a/jobs/kubelet-windows/templates/bin/drain.ps1.erb +++ /dev/null @@ -1,69 +0,0 @@ -trap { $host.SetShouldExit(1) } - -$OutLog = "C:\var\vcap\sys\log\kubelet-windows\drain.stdout.log" -$ErrLog = "C:\var\vcap\sys\log\kubelet-windows\drain.stderr.log" - - -function main { - if (kubelet_is_running) { - retry "drain_kubelet" $function:drain_kubelet - # watch_disks TODO(BM/LH): implement this? - retry "delete_drained_node" $function:delete_drained_node - } - echo "0" -} - -function drain_kubelet() { - $nodes=(/var/vcap/packages/kubernetes-windows/bin/kubectl ` - --kubeconfig /var/vcap/jobs/kubelet-windows/config/kubeconfig-drain ` - get nodes -o json | ConvertFrom-Json).Items - $node_name=($nodes | ? { $_.metadata.labels."bosh.id" -eq "<%= spec.id %>" }).metadata.name - - if (!$node_name) { - return $true - } - /var/vcap/packages/kubernetes-windows/bin/kubectl ` - --kubeconfig /var/vcap/jobs/kubelet-windows/config/kubeconfig-drain drain "${node_name}" ` - --grace-period 10 --force --delete-local-data --ignore-daemonsets - return $? -} - -function delete_drained_node() { - $nodes=(/var/vcap/packages/kubernetes-windows/bin/kubectl ` - --kubeconfig /var/vcap/jobs/kubelet-windows/config/kubeconfig-drain ` - get nodes -o json | ConvertFrom-Json).Items - $node_name=($nodes | ? { $_.metadata.labels."bosh.id" -eq "<%= spec.id %>" }).metadata.name - - if (!$node_name) { - return $true - } - /var/vcap/packages/kubernetes-windows/bin/kubectl ` - --kubeconfig /var/vcap/jobs/kubelet-windows/config/kubeconfig-drain delete node "${node_name}" ` - --ignore-not-found | Out-Null - return $? -} - -function retry($name, $func) { - $attempt_number=1 - $max_attempts=10 - - do { - $result=$func.Invoke() - if ($result) { - echo "Successfully $name" | Out-File -FilePath $OutLog -encoding ascii - return - } - echo ("[{0}] Unsuccessful {1}, retrying attempt {2} out of {3}" -f (Get-Date -UFormat %s), $name, $attempt_number, $max_attempts) | Out-File -FilePath $OutLog -encoding ascii - $attempt_number=$attempt_number + 1 - sleep 1 - } while ($attempt_number -le $max_attempts) - - throw "Failed all retry attempts for $name" -} - -function kubelet_is_running() { - curl.exe --silent --fail http://localhost:10248/healthz - return $? -} - -main diff --git a/jobs/kubelet-windows/templates/bin/kubelet_ctl.ps1.erb b/jobs/kubelet-windows/templates/bin/kubelet_ctl.ps1.erb deleted file mode 100644 index e8b58bc2f..000000000 --- a/jobs/kubelet-windows/templates/bin/kubelet_ctl.ps1.erb +++ /dev/null @@ -1,144 +0,0 @@ -trap { $host.SetShouldExit(1) } - -$env:PATH+=";C:\var\vcap\packages\docker\docker\;" - -<% iaas = nil %> -<% if_p('cloud-provider') do |cloud_provider| %> - <% iaas = cloud_provider %> - $cloud_provider="<%= cloud_provider %>" -<% end %> - -<% if_link('cloud-provider') do |cloud_provider| %> - $cloud_config="/var/vcap/jobs/kubelet-windows/config/cloud-provider.ini" - <% cloud_provider.if_p('cloud-provider.gce.service_key') do |service_key| %> - $env:GOOGLE_APPLICATION_CREDENTIALS="/var/vcap/jobs/kubelet-windows/config/service_key.json" - <% end %> - <% cloud_provider.if_p('cloud-provider.aws.access_key_id') do |access_key_id| %> - $env:AWS_ACCESS_KEY_ID="<%= access_key_id %>" - <% end %> - <% cloud_provider.if_p('cloud-provider.aws.secret_access_key') do |secret_access_key| %> - $env:AWS_SECRET_ACCESS_KEY="<%= secret_access_key %>" - <% end %> -<% end %> - -<% - labels = ["spec.ip=#{spec.ip}","bosh.id=#{spec.id}","bosh.zone=#{spec.az}"] - if iaas=="vsphere" - labels << "failure-domain.beta.kubernetes.io/zone=#{spec.az}" - end - if_p("labels") do |node_labels| - labels += node_labels.map {|k,v| "#{k}=#{v}"} - end - labels = labels.join(',') -%> - -<% if_p('http_proxy') do |http_proxy| %> -$env:HTTP_PROXY=<%= http_proxy %> -<% end %> -<% if_p('https_proxy') do |https_proxy| %> -$env:HTTPS_PROXY=<%= https_proxy %> -<% end %> -<% if_p('no_proxy') do |no_proxy| %> -$env:NO_PROXY=<%= no_proxy %> -<% end %> - -function delete_stale_drained_node { - $nodes=(/var/vcap/packages/kubernetes-windows/bin/kubectl --kubeconfig /var/vcap/jobs/kubelet-windows/config/kubeconfig-drain get nodes -o json | ConvertFrom-Json).Items - $nodes | ForEach-Object { - if ($_.metadata.labels."bosh.id" -eq "<= spec.id %>") { - if (($_.status.conditions | ? type -eq "Ready").status -ne "True") { - /var/vcap/packages/kubernetes-windows/bin/kubectl --kubeconfig /var/vcap/jobs/kubelet-windows/config/kubeconfig-drain delete node "${node_name}" --ignore-not-found - } - } - } -} - -function get_hostname_override { - if ($cloud_provider -in "gce", "azure") { - return "" - } else { - return "<%= spec.ip %>" - } -} - -function start_kubelet { - <%- - include_config = false - if !iaas.nil? and iaas != "vsphere" - if_link('cloud-provider') do - include_config = true - end - end - -%> - mkdir -force C:\tmp # workaround for conformance tests - - /var/vcap/packages/kubernetes-windows/bin/kubelet ` - <%- - if_p('k8s-args') do |args| - args.each do |flag, value| - valueString = "" - - if value.nil? - # Do nothing to supports args-less flags (--example) - elsif value.is_a? Array - valueString = "=#{value.join(",")}" - elsif value.is_a? Hash - valueString = "=#{value.map { |k,v| "#{k}=#{v}" }.join(",")}" - else - valueString = "=#{value}" - end - -%> - <%= "--#{flag}#{valueString}" %> ` - <%- - end - end - -%> - <% if include_config -%>--cloud-config=${cloud_config}<% end %> ` - <% if !iaas.nil? -%>--cloud-provider=${cloud_provider}<% end %> ` - --hostname-override=$(get_hostname_override) ` - --node-labels=<%= labels %> ` - --config="/var/vcap/jobs/kubelet-windows/config/kubeletconfig.yml" -} - -function check_for_networking { - $subnetConfig="/run/flannel/subnet.env" - - if (-not ([System.IO.File]::Exists($subnetConfig))) - { - throw "$subnetConfig does not exist, waiting for flannel initialization" - } -} - -function load_base_images { - # Temporary workaround until we bring our own images - docker version - if (-not $?) { - throw "docker is not available" - } - - docker pull mcr.microsoft.com/windows/nanoserver:1809 - docker tag (docker images mcr.microsoft.com/windows/nanoserver:1809 -q) mcr.microsoft.com/windows/nanoserver - docker build -t kubeletwin/pause /var/vcap/jobs/kubelet-windows/config - - Start-Job -ScriptBlock { - docker pull mcr.microsoft.com/windows/servercore:ltsc2019 - docker tag (docker images mcr.microsoft.com/windows/servercore:ltsc2019 -q) mcr.microsoft.com/windows/servercore - } -} - -function set_acls { - $ar = New-Object System.Security.AccessControl.FileSystemAccessRule("BUILTIN\Users", "ReadAndExecute,CreateFiles,AppendData", "ContainerInherit, ObjectInherit", "None", "Allow") - $acl = Get-Acl C:\var - $acl.SetAccessRule($ar) - Set-Acl C:\var $acl -} - -function main { - delete_stale_drained_node - check_for_networking - load_base_images - set_acls - start_kubelet -} - -main diff --git a/jobs/kubelet-windows/templates/bin/post-start.ps1 b/jobs/kubelet-windows/templates/bin/post-start.ps1 deleted file mode 100644 index 53daeeb72..000000000 --- a/jobs/kubelet-windows/templates/bin/post-start.ps1 +++ /dev/null @@ -1,29 +0,0 @@ -trap { $host.SetShouldExit(1) } - -function kubelet_is_running() { - curl.exe --fail http://localhost:10248/healthz - return $? -} - -function main() { - retry "passed kubelet healthcheck" $function:kubelet_is_running -} - -function retry($name, $func) { - $attempt_number=1 - $max_attempts=10 - - do { - $result=$func.Invoke() - if ($result) { - echo "Successfully $name" - return $true - } - echo ("[{0}] Unsuccessful {1}, retrying attempt {2} out of {3}" -f (Get-Date -UFormat %s), $name, $attempt_number, $max_attempts) - $attempt_number=$attempt_number + 1 - sleep 1 - } while ($attempt_number -le $max_attempts) - - echo "Failed all retry attempts for $name" - return $false -} diff --git a/jobs/kubelet-windows/templates/config/Dockerfile b/jobs/kubelet-windows/templates/config/Dockerfile deleted file mode 100644 index abaddf0aa..000000000 --- a/jobs/kubelet-windows/templates/config/Dockerfile +++ /dev/null @@ -1,3 +0,0 @@ -FROM mcr.microsoft.com/windows/nanoserver - -CMD cmd /c ping -t localhost diff --git a/jobs/kubelet-windows/templates/config/apiserver-ca.pem.erb b/jobs/kubelet-windows/templates/config/apiserver-ca.pem.erb deleted file mode 100644 index be33ccabb..000000000 --- a/jobs/kubelet-windows/templates/config/apiserver-ca.pem.erb +++ /dev/null @@ -1 +0,0 @@ -<%= p('tls.kubernetes.ca') %> \ No newline at end of file diff --git a/jobs/kubelet-windows/templates/config/cloud-provider.ini.erb b/jobs/kubelet-windows/templates/config/cloud-provider.ini.erb deleted file mode 120000 index 1d10c9f3b..000000000 --- a/jobs/kubelet-windows/templates/config/cloud-provider.ini.erb +++ /dev/null @@ -1 +0,0 @@ -../../../kube-apiserver/templates/config/cloud-provider.ini.erb \ No newline at end of file diff --git a/jobs/kubelet-windows/templates/config/kubeconfig-drain.erb b/jobs/kubelet-windows/templates/config/kubeconfig-drain.erb deleted file mode 100644 index b5a42d458..000000000 --- a/jobs/kubelet-windows/templates/config/kubeconfig-drain.erb +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: v1 -kind: Config -clusters: -- cluster: - certificate-authority: C:\var\vcap\jobs\kubelet-windows\config\apiserver-ca.pem - server: https://master.cfcr.internal:8443 - name: kubernetes -contexts: -- context: - cluster: kubernetes - user: kubelet-drain - name: kubelet-drain -current-context: kubelet-drain -users: -- name: kubelet-drain - user: - token: "<%= p("drain-api-token") %>" diff --git a/jobs/kubelet-windows/templates/config/kubeconfig.erb b/jobs/kubelet-windows/templates/config/kubeconfig.erb deleted file mode 100644 index 828f1afb1..000000000 --- a/jobs/kubelet-windows/templates/config/kubeconfig.erb +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: v1 -kind: Config -clusters: -- cluster: - certificate-authority: C:\var\vcap\jobs\kubelet-windows\config\apiserver-ca.pem - server: https://master.cfcr.internal:8443 - name: kubernetes -contexts: -- context: - cluster: kubernetes - user: kubelet - name: kubelet -current-context: kubelet -users: -- name: kubelet - user: - token: "<%= p("api-token") %>" diff --git a/jobs/kubelet-windows/templates/config/kubelet-client-ca.pem.erb b/jobs/kubelet-windows/templates/config/kubelet-client-ca.pem.erb deleted file mode 100644 index dc108e47d..000000000 --- a/jobs/kubelet-windows/templates/config/kubelet-client-ca.pem.erb +++ /dev/null @@ -1 +0,0 @@ -<%= p('tls.kubelet-client-ca.certificate') %> diff --git a/jobs/kubelet-windows/templates/config/kubelet-key.pem.erb b/jobs/kubelet-windows/templates/config/kubelet-key.pem.erb deleted file mode 100644 index d2226410f..000000000 --- a/jobs/kubelet-windows/templates/config/kubelet-key.pem.erb +++ /dev/null @@ -1 +0,0 @@ -<%= p("tls.kubelet.private_key") %> \ No newline at end of file diff --git a/jobs/kubelet-windows/templates/config/kubelet.pem.erb b/jobs/kubelet-windows/templates/config/kubelet.pem.erb deleted file mode 100644 index dda602a6d..000000000 --- a/jobs/kubelet-windows/templates/config/kubelet.pem.erb +++ /dev/null @@ -1,2 +0,0 @@ -<%= p("tls.kubelet.certificate") %> -<%= p("tls.kubelet.ca") %> \ No newline at end of file diff --git a/jobs/kubelet-windows/templates/config/kubeletconfig.yml.erb b/jobs/kubelet-windows/templates/config/kubeletconfig.yml.erb deleted file mode 100644 index a651e2a81..000000000 --- a/jobs/kubelet-windows/templates/config/kubeletconfig.yml.erb +++ /dev/null @@ -1 +0,0 @@ -<% require 'yaml' %><%= p('kubelet-configuration').to_yaml %> diff --git a/jobs/kubelet-windows/templates/config/openstack-ca.crt.erb b/jobs/kubelet-windows/templates/config/openstack-ca.crt.erb deleted file mode 120000 index af78a3025..000000000 --- a/jobs/kubelet-windows/templates/config/openstack-ca.crt.erb +++ /dev/null @@ -1 +0,0 @@ -../../../kube-apiserver/templates/config/openstack-ca.crt.erb \ No newline at end of file diff --git a/jobs/kubelet-windows/templates/config/service_key.json.erb b/jobs/kubelet-windows/templates/config/service_key.json.erb deleted file mode 120000 index 4954f82c4..000000000 --- a/jobs/kubelet-windows/templates/config/service_key.json.erb +++ /dev/null @@ -1 +0,0 @@ -../../../kube-apiserver/templates/config/service_key.json.erb \ No newline at end of file diff --git a/packages/cni-windows/packaging b/packages/cni-windows/packaging deleted file mode 100644 index cc96e3672..000000000 --- a/packages/cni-windows/packaging +++ /dev/null @@ -1,8 +0,0 @@ -. ./exiter.ps1 - -$ErrorActionPreference = "Stop"; -trap { $host.SetShouldExit(1) } - -mkdir ${env:BOSH_INSTALL_TARGET}/bin -$tgz=(get-childitem cni/cni-plugins-windows-amd64*.tgz).Name -tar -xzf cni/$tgz -C ${env:BOSH_INSTALL_TARGET}/bin/ diff --git a/packages/cni-windows/spec b/packages/cni-windows/spec deleted file mode 100644 index 86fb04cce..000000000 --- a/packages/cni-windows/spec +++ /dev/null @@ -1,6 +0,0 @@ ---- -name: cni-windows - -files: -- cni/cni-plugins-windows-amd64-c74e0e996.tgz -- exiter.ps1 diff --git a/packages/flanneld-windows/packaging b/packages/flanneld-windows/packaging deleted file mode 100644 index 23168bd17..000000000 --- a/packages/flanneld-windows/packaging +++ /dev/null @@ -1,9 +0,0 @@ -. ./exiter.ps1 - -$ErrorActionPreference = "Stop"; -trap { $host.SetShouldExit(1) } - - -$FLANNELD_VERSION="0.11.0" -tar xvf flannel-v${FLANNELD_VERSION}-windows-amd64.tar.gz -Copy-Item flanneld.exe "${env:BOSH_INSTALL_TARGET}/flanneld.exe" diff --git a/packages/flanneld-windows/spec b/packages/flanneld-windows/spec deleted file mode 100644 index b5948e629..000000000 --- a/packages/flanneld-windows/spec +++ /dev/null @@ -1,8 +0,0 @@ ---- -name: flanneld-windows - -dependencies: - -files: -- flannel-v0.11.0-windows-amd64.tar.gz -- exiter.ps1 diff --git a/packages/kubernetes-windows/packaging b/packages/kubernetes-windows/packaging deleted file mode 100644 index 4f4a292b1..000000000 --- a/packages/kubernetes-windows/packaging +++ /dev/null @@ -1,10 +0,0 @@ -. ./exiter.ps1 - -$ErrorActionPreference = "Stop"; -trap { $host.SetShouldExit(1) } - -$KUBERNETES_VERSION="1.13.4" - -New-Item -Path "${env:BOSH_INSTALL_TARGET}" -Name "bin" -ItemType "directory" - -Copy-Item "kubernetes-windows-${KUBERNETES_VERSION}/*" "${env:BOSH_INSTALL_TARGET}/bin" diff --git a/packages/kubernetes-windows/spec b/packages/kubernetes-windows/spec deleted file mode 100644 index 2f858c167..000000000 --- a/packages/kubernetes-windows/spec +++ /dev/null @@ -1,6 +0,0 @@ ---- -name: kubernetes-windows - -files: -- kubernetes-windows-1.13.4/* -- exiter.ps1 diff --git a/scripts/download_k8s_binaries b/scripts/download_k8s_binaries index c012eff56..639b6a4f2 100755 --- a/scripts/download_k8s_binaries +++ b/scripts/download_k8s_binaries @@ -14,7 +14,7 @@ main() { trap '{ rm -rf "$staging_dir"; }' EXIT - linux_binaries=( + binaries=( "kube-apiserver" "kube-controller-manager" "kube-proxy" @@ -22,72 +22,46 @@ main() { "kubectl" "kubelet" ) - windows_binaries=( - "kube-proxy.exe" - "kubectl.exe" - "kubelet.exe" - ) - platforms=( - "windows" - "linux" - ) pushd "$(dirname "${BASH_SOURCE[0]}")/.." - for platform in "${platforms[@]}"; do - package_name=$(package_name "${platform}") - declare -n binaries="${platform}_binaries" - - existing_k8s_spec=$(bosh blobs --column path | grep "${binaries[0]}" | grep -o -E "${package_name}-([0-9]+\.)+[0-9]+") - existing_k8s_version=$(echo "$existing_k8s_spec" | grep -o -E '([0-9]+\.)+[0-9]+') - - if [ "$existing_k8s_version" == "$kubernetes_version" ]; then - echo "Kubernetes version already up-to-date." - else - pushd "packages/${package_name}" - sed -E -i -e "s/([0-9]+\.)+[0-9]+/${kubernetes_version}/" packaging - sed -E -i -e "s/${existing_k8s_spec}/${package_name}-${kubernetes_version}/" spec - popd - - for binary in "${binaries[@]}"; do - download "${binary}" "${staging_dir}" "${kubernetes_version}" "${platform}" - add_blob "${binary}" "${staging_dir}" "${kubernetes_version}" "${package_name}" - done - fi - done + existing_k8s_spec=$(bosh blobs --column path | grep kubelet | grep -o -E 'kubernetes-([0-9]+\.)+[0-9]+') + existing_k8s_version=$(echo "$existing_k8s_spec" | grep -o -E '([0-9]+\.)+[0-9]+') + + if [ "$existing_k8s_version" == "$kubernetes_version" ]; then + echo "Kubernetes version already up-to-date." + else + pushd packages/kubernetes + sed -E -i -e "s/([0-9]+\.)+[0-9]+/${kubernetes_version}/" packaging + sed -E -i -e "s/${existing_k8s_spec}/kubernetes-${kubernetes_version}/" spec + popd + + for binary in "${binaries[@]}"; do + download "${binary}" "${staging_dir}" "${kubernetes_version}" + add_blob "${binary}" "${staging_dir}" "${kubernetes_version}" + done + fi popd } -package_name() { - local platform - platform="$1" - if [ "$platform" == "windows" ]; then - echo -n "kubernetes-windows" - else - echo -n "kubernetes" - fi -} - download() { - local binary_name staging_dir kubernetes_version platform + local binary_name staging_dir kubernetes_version binary_name="$1" staging_dir="$2" kubernetes_version="$3" - platform="$4" - wget -O "${staging_dir}/${binary_name}" "https://storage.googleapis.com/kubernetes-release/release/v${kubernetes_version}/bin/${platform}/amd64/${binary_name}" + wget -O "${staging_dir}/${binary_name}" "https://storage.googleapis.com/kubernetes-release/release/v${kubernetes_version}/bin/linux/amd64/${binary_name}" } add_blob() { - local binary_name blob_name staging_dir kubernetes_version package_name + local binary_name blob_name staging_dir kubernetes_version binary_name="$1" staging_dir="$2" kubernetes_version="$3" - package_name="$4" - blob_name=$(bosh blobs --column path | grep "${binary_name}\s$" | xargs) + blob_name=$(bosh blobs --column path | grep "$binary_name" | xargs) bosh remove-blob "$blob_name" - bosh add-blob "${staging_dir}/${binary_name}" "${package_name}-${kubernetes_version}/$binary_name" + bosh add-blob "${staging_dir}/${binary_name}" "kubernetes-${kubernetes_version}/$binary_name" } main "$@" diff --git a/src/exiter.ps1 b/src/exiter.ps1 deleted file mode 100644 index 9b6fa3eec..000000000 --- a/src/exiter.ps1 +++ /dev/null @@ -1,6 +0,0 @@ -exit 0 -# The reason for needing this file is to make `bosh export-release` happy. -# BOSH doesn't know what job to export, so it tries to export a windows job -# against a linux stemcell. This is a file that will cause a bash script to -# immediately exit (such as when exporting a release) but be ignored when run -# in powershell.