forked from datenlord/datenlord
-
Notifications
You must be signed in to change notification settings - Fork 0
272 lines (266 loc) · 15.8 KB
/
cron.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
name: CRON-CI
on:
pull_request:
branches: [master]
schedule: [cron: "0 */4 * * *"]
env:
RUST_VERSION: 1.47.0
CONFIG_MINIKUBE: scripts/datenlord_minikube.yaml
CONFIG_KIND: scripts/datenlord.yaml
CONFIG_DOCKERHUB: datenlord-deploy.yaml
CONTROLLER_APP_LABEL: csi-controller-datenlord
CONTROLLER_CONTAINER_NAME: datenlord-controller-plugin
DATENLORD_CSI_IMAGE: datenlord/csiplugin:e2e_test
DATENLORD_IMAGE: datenlord/datenlord:e2e_test
DATENLORD_NAMESPACE: csi-datenlord
E2E_TEST_CONFIG: scripts/datenlord-e2e-test.yaml
FUSE_CONTAINER_NAME: datenlord-fuse
FUSE_MOUNT_PATH: /var/opt/datenlord-data
K8S_CONFIG: k8s.e2e.config
K8S_VERSION: v1.19.0
KIND_NODE_VERSION: kindest/node:v1.19.1@sha256:98cf5288864662e37115e362b23e4369c8c4a408f99cbc06e58ac30ddc721600
KIND_VERSION: v0.9.0
NODE_APP_LABEL: csi-nodeplugin-datenlord
NODE_CONTAINER_NAME: datenlord-node-plugin
MINIKUBE_VERSION: v1.13.0
jobs:
#minikube-csi-e2e:
# name: K8S-CSI-E2E
# # How to run K8S CSI e2e test
# # https://www.stackovercloud.com/2020/01/08/blog-testing-of-csi-drivers/
# runs-on: ubuntu-latest
# steps:
# - name: Set SSH config
# run: |
# sudo rm -rf ~/.ssh/
# ssh-keygen -N '' -f ~/.ssh/id_rsa
# cp ~/.ssh/id_rsa.pub ~/.ssh/authorized_keys
# cat ~/.ssh/authorized_keys
# export HOSTNAME=`hostname`
# export WHO=`whoami`
# export HOSTIP=`ifconfig eth0 | grep 'inet ' | awk '{print $2}'`
# cat >>~/.ssh/config <<END
# Host thisrunner
# HostName $HOSTNAME
# User $WHO
# IdentityFile $HOME/.ssh/id_rsa
# StrictHostKeyChecking no
# UserKnownHostsFile /dev/null
# END
# chmod 755 $HOME
# sudo service ssh restart
# sudo service ssh status
# ssh -vvv thisrunner
# exit
# - name: Setup Minikube
# uses: manusa/[email protected]
# with:
# #minikube version: 'v1.13.0'
# #kubernetes version: 'v1.19.0'
# minikube version: ${{ env.MINIKUBE_VERSION }}
# kubernetes version: ${{ env.K8S_VERSION }}
# github token: ${{ secrets.GITHUB_TOKEN }}
# #- name: Deploy minikube
# # uses: opsgang/[email protected]
# # with:
# # minikube-version: 1.12.3
# # k8s-version: 1.18.8
# #- name: Test deploy DatenLord CSI plugin to K8S
# # run: |
# # kubectl cluster-info
# # kubectl get pods -A
# # docker pull quay.io/k8scsi/csi-node-driver-registrar:v1.3.0
# # docker pull quay.io/k8scsi/livenessprobe:v2.0.0
# # docker pull quay.io/k8scsi/csi-attacher:v2.2.0
# # docker pull quay.io/k8scsi/csi-provisioner:v1.6.0
# # docker pull quay.io/k8scsi/csi-snapshotter:v2.1.1
# # docker pull quay.io/k8scsi/csi-resizer:v0.5.0
# # docker pull gcr.io/etcd-development/etcd:latest
# # docker pull datenlord/csiplugin:latest
# # kubectl apply -f csiplugin_k8s.yaml
# # kubectl get csidriver
# # while [[ $(kubectl get pods -l app=csi-controller-datenlord -A -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}') != "True" ]]; do kubectl get pods -A -o wide ; kubectl get nodes -o wide ; kubectl describe pod csi-controller-datenlord-0 -n csi-datenlord ; sleep 6 ; done
# # while [[ $(kubectl get pods -l app=csi-nodeplugin-datenlord -A -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}') != "True" ]]; do kubectl get pods -A -o wide ; kubectl get nodes -o wide ; sleep 6 ; done
# # kubectl get pods -A -o wide
# # sudo netstat -lntp && ls -lsh
# - name: Check out code
# uses: actions/checkout@v2
# - name: Docker build fuse and csi
# run: |
# docker build . --build-arg RUST_IMAGE_VERSION=$RUST_VERSION --file ./Dockerfile --target fuse --tag $DATENLORD_IMAGE
# docker build . --build-arg RUST_IMAGE_VERSION=$RUST_VERSION --file ./Dockerfile --target csi --tag $DATENLORD_CSI_IMAGE
# - name: Deploy DatenLord to K8S
# run: |
# kubectl cluster-info
# kubectl get pods -A
# kubectl apply -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/master/client/config/crd/snapshot.storage.k8s.io_volumesnapshots.yaml
# kubectl apply -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/master/client/config/crd/snapshot.storage.k8s.io_volumesnapshotcontents.yaml
# kubectl apply -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/master/client/config/crd/snapshot.storage.k8s.io_volumesnapshotclasses.yaml
# kubectl apply -f $CONFIG_MINIKUBE
# kubectl wait --for=condition=Ready pod -l app=$CONTROLLER_APP_LABEL -n $DATENLORD_NAMESPACE --timeout=60s
# kubectl wait --for=condition=Ready pod -l app=$NODE_APP_LABEL -n $DATENLORD_NAMESPACE --timeout=60s
# FOUND_PATH=`cat /proc/self/mountinfo | grep fuse | grep $FUSE_MOUNT_PATH | awk '{print $5}'`
# test -n $FOUND_PATH || (echo "FAILED TO FIND MOUNT PATH $FUSE_MOUNT_PATH" && /bin/false)
# kubectl delete -f $CONFIG_MINIKUBE
# NO_PATH=`cat /proc/self/mountinfo | grep fuse | grep $FUSE_MOUNT_PATH | awk '{print $5}'`
# test -z $NO_PATH || (echo "FAILED TO UN-MOUNT PATH $FUSE_MOUNT_PATH" && /bin/false)
# kubectl apply -f $CONFIG_MINIKUBE
# kubectl get csidriver
# kubectl get csinode
# kubectl get storageclass
# kubectl get volumesnapshotclass
# kubectl wait --for=condition=Ready pod -l app=$CONTROLLER_APP_LABEL -n $DATENLORD_NAMESPACE --timeout=60s
# kubectl wait --for=condition=Ready pod -l app=$NODE_APP_LABEL -n $DATENLORD_NAMESPACE --timeout=60s
# wget https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/master/deploy/kubernetes/snapshot-controller/rbac-snapshot-controller.yaml
# wget https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/master/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml
# sed -e 's/namespace\:\ default/namespace\:\ kube\-system/g' rbac-snapshot-controller.yaml > datenlord-rbac-snapshot-controller.yaml
# sed -e 's/namespace\:\ default/namespace\:\ kube\-system/g' setup-snapshot-controller.yaml > datenlord-setup-snapshot-controller.yaml
# kubectl apply -f datenlord-rbac-snapshot-controller.yaml
# kubectl apply -f datenlord-setup-snapshot-controller.yaml
# kubectl wait --for=condition=Ready pod -l app=snapshot-controller -n kube-system --timeout=60s
# kubectl get pods -A -o wide
# sudo netstat -lntp && ls -lsh
# - name: CSI E2E Test
# run: |
# wget --quiet https://dl.k8s.io/$K8S_VERSION/kubernetes-test-linux-amd64.tar.gz
# tar zxvf kubernetes-test-linux-amd64.tar.gz
# kubectl config view --raw > $K8S_CONFIG
# #kubernetes/test/bin/e2e.test -v=5 -ginkgo.failFast -ginkgo.failOnPending -ginkgo.debug -ginkgo.v -ginkgo.focus='External.Storage' -kubectl-path=`which kubectl` -kubeconfig=$K8S_CONFIG -storage.testdriver=`realpath $E2E_TEST_CONFIG`
# kubernetes/test/bin/ginkgo -p -v -failFast -failOnPending -debug -focus='External.Storage' kubernetes/test/bin/e2e.test -- -v=5 -kubectl-path=`which kubectl` -kubeconfig=`realpath $K8S_CONFIG` -storage.testdriver=`realpath $E2E_TEST_CONFIG` -test.parallel=3
# - name: Print DatenLord logs
# if: ${{ failure() }}
# run: |
# CONTROLLER_POD_NAME=`kubectl get pod -l app=$CONTROLLER_APP_LABEL -n $DATENLORD_NAMESPACE -o jsonpath="{.items[0].metadata.name}"`
# echo "SHOW LOGS OF $CONTROLLER_CONTAINER_NAME IN $CONTROLLER_POD_NAME"
# kubectl logs $CONTROLLER_POD_NAME -n $DATENLORD_NAMESPACE -c $CONTROLLER_CONTAINER_NAME
# NODE_POD_NAME=`kubectl get pod -l app=$NODE_APP_LABEL -n $DATENLORD_NAMESPACE -o jsonpath="{.items[0].metadata.name}"`
# echo "SHOW LOGS OF $NODE_CONTAINER_NAME IN $NODE_POD_NAME"
# kubectl logs $NODE_POD_NAME -n $DATENLORD_NAMESPACE -c $NODE_CONTAINER_NAME
# echo "SHOW LOGS OF $FUSE_CONTAINER_NAME IN $NODE_POD_NAME"
# kubectl logs $NODE_POD_NAME -n $DATENLORD_NAMESPACE -c $FUSE_CONTAINER_NAME
# - name: Setup tmate session
# if: ${{ failure() }}
# uses: mxschmitt/action-tmate@v3
kind-csi-e2e:
name: KIND-K8S-CSI-E2E
# How to run K8S CSI e2e test
# https://www.stackovercloud.com/2020/01/08/blog-testing-of-csi-drivers/
runs-on: ubuntu-latest
steps:
- name: Check out code
uses: actions/checkout@v2
- name: Install Kind
run: |
curl -Lo ./kind https://kind.sigs.k8s.io/dl/$KIND_VERSION/kind-linux-amd64
chmod +x ./kind
sudo mv ./kind /usr/local/bin
- name: Create Kind Cluster
run: |
cat >> ./kind-config.yaml <<END
# Kind cluster with 1 control plane node and 3 workers
kind: Cluster
apiVersion: kind.x-k8s.io/v1alpha4
nodes:
# the control plane node config
- role: control-plane
# the three workers
- role: worker
image: $KIND_NODE_VERSION
- role: worker
image: $KIND_NODE_VERSION
- role: worker
image: $KIND_NODE_VERSION
END
kind create cluster --config ./kind-config.yaml
- name: Setup SSH
run: |
cat >>/tmp/setup.sh <<'END'
NODES_IP="$(kubectl get nodes -A -o wide | awk 'FNR > 2 {print $6}')"
NODES="$(kubectl get nodes -A -o wide | awk 'FNR > 2 {print $1}')"
for node in ${NODES}; do
USER="$(whoami)"
docker exec ${node} apt-get update
docker exec ${node} apt-get install -y ssh sudo
docker exec ${node} systemctl start sshd
docker exec ${node} useradd -m ${USER}
docker exec ${node} usermod -aG sudo ${USER}
echo "${USER} ALL=(ALL) NOPASSWD:ALL" > /tmp/${USER}
docker cp /tmp/${USER} ${node}:/etc/sudoers.d/${USER}
docker exec ${node} chown root:root /etc/sudoers.d/${USER}
docker exec ${node} mkdir /home/${USER}/.ssh
docker exec ${node} ls -al /home/${USER}/
docker cp ${HOME}/.ssh/id_rsa.pub ${node}:/home/${USER}/.ssh/authorized_keys
docker exec ${node} chown ${USER}:${USER} /home/${USER}/ -R
done
for ip in ${NODES_IP}; do
ssh-keyscan -H $ip >> ${HOME}/.ssh/known_hosts
done
END
rm -rf $HOME/.ssh/
ssh-keygen -N '' -f ~/.ssh/id_rsa
/bin/bash /tmp/setup.sh
- name: Docker build fuse and csi
run: |
docker build . --build-arg RUST_IMAGE_VERSION=$RUST_VERSION --file ./Dockerfile --target fuse --tag $DATENLORD_IMAGE
docker build . --build-arg RUST_IMAGE_VERSION=$RUST_VERSION --file ./Dockerfile --target csi --tag $DATENLORD_CSI_IMAGE
kind load docker-image $DATENLORD_IMAGE
kind load docker-image $DATENLORD_CSI_IMAGE
- name: Deploy DatenLord to K8S
run: |
kubectl cluster-info
kubectl get pods -A
kubectl apply -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/master/client/config/crd/snapshot.storage.k8s.io_volumesnapshots.yaml
kubectl apply -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/master/client/config/crd/snapshot.storage.k8s.io_volumesnapshotcontents.yaml
kubectl apply -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/master/client/config/crd/snapshot.storage.k8s.io_volumesnapshotclasses.yaml
kubectl apply -f $CONFIG_KIND
kubectl wait --for=condition=Ready pod -l app=$CONTROLLER_APP_LABEL -n $DATENLORD_NAMESPACE --timeout=60s
kubectl wait --for=condition=Ready pod -l app=$NODE_APP_LABEL -n $DATENLORD_NAMESPACE --timeout=60s
FOUND_PATH=`cat /proc/self/mountinfo | grep fuse | grep $FUSE_MOUNT_PATH | awk '{print $5}'`
test -n $FOUND_PATH || (echo "FAILED TO FIND MOUNT PATH $FUSE_MOUNT_PATH" && /bin/false)
kubectl delete -f $CONFIG_KIND
NO_PATH=`cat /proc/self/mountinfo | grep fuse | grep $FUSE_MOUNT_PATH | awk '{print $5}'`
test -z $NO_PATH || (echo "FAILED TO UN-MOUNT PATH $FUSE_MOUNT_PATH" && /bin/false)
kubectl apply -f $CONFIG_KIND
kubectl get csidriver
kubectl get csinode
kubectl get storageclass
kubectl get volumesnapshotclass
kubectl wait --for=condition=Ready pod -l app=$CONTROLLER_APP_LABEL -n $DATENLORD_NAMESPACE --timeout=60s
kubectl wait --for=condition=Ready pod -l app=$NODE_APP_LABEL -n $DATENLORD_NAMESPACE --timeout=60s
wget https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/master/deploy/kubernetes/snapshot-controller/rbac-snapshot-controller.yaml
wget https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/master/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml
sed -e 's/namespace\:\ default/namespace\:\ kube\-system/g' rbac-snapshot-controller.yaml > datenlord-rbac-snapshot-controller.yaml
sed -e 's/namespace\:\ default/namespace\:\ kube\-system/g' setup-snapshot-controller.yaml > datenlord-setup-snapshot-controller.yaml
kubectl apply -f datenlord-rbac-snapshot-controller.yaml
kubectl apply -f datenlord-setup-snapshot-controller.yaml
kubectl wait --for=condition=Ready pod -l app=snapshot-controller -n kube-system --timeout=60s
kubectl get pods -A -o wide
# sudo netstat -lntp && ls -lsh
- name: CSI E2E Test
run: |
wget --quiet https://dl.k8s.io/$K8S_VERSION/kubernetes-test-linux-amd64.tar.gz
tar zxvf kubernetes-test-linux-amd64.tar.gz
kubectl config view --raw > $K8S_CONFIG
#kubernetes/test/bin/e2e.test -v=5 -ginkgo.failFast -ginkgo.failOnPending -ginkgo.debug -ginkgo.v -ginkgo.focus='External.Storage' -kubectl-path=`which kubectl` -kubeconfig=$K8S_CONFIG -storage.testdriver=`realpath $E2E_TEST_CONFIG`
kubernetes/test/bin/ginkgo -p -v -failFast -failOnPending -debug -focus='External.Storage' -skip='.*Dynamic PV.*filesystem.*should access to two volumes with the same volume mode and retain data across pod recreation on .* node' kubernetes/test/bin/e2e.test -- -v=5 -kubectl-path=`which kubectl` -kubeconfig=`realpath $K8S_CONFIG` -storage.testdriver=`realpath $E2E_TEST_CONFIG` -test.parallel=3
- name: Quick Start Test
run: |
kubectl delete -f $CONFIG_KIND
sed -e 's/e2e_test/latest/g' $CONFIG_KIND > $CONFIG_DOCKERHUB
kubectl apply -f $CONFIG_DOCKERHUB
kubectl wait --for=condition=Ready pod -l app=$CONTROLLER_APP_LABEL -n $DATENLORD_NAMESPACE --timeout=60s
kubectl wait --for=condition=Ready pod -l app=$NODE_APP_LABEL -n $DATENLORD_NAMESPACE --timeout=60s
- name: Print DatenLord logs
if: ${{ failure() }}
run: |
CONTROLLER_POD_NAME=`kubectl get pod -l app=$CONTROLLER_APP_LABEL -n $DATENLORD_NAMESPACE -o jsonpath="{.items[0].metadata.name}"`
echo "SHOW LOGS OF $CONTROLLER_CONTAINER_NAME IN $CONTROLLER_POD_NAME"
kubectl logs $CONTROLLER_POD_NAME -n $DATENLORD_NAMESPACE -c $CONTROLLER_CONTAINER_NAME
NODE_POD_NAME=`kubectl get pod -l app=$NODE_APP_LABEL -n $DATENLORD_NAMESPACE -o jsonpath="{.items[0].metadata.name}"`
echo "SHOW LOGS OF $NODE_CONTAINER_NAME IN $NODE_POD_NAME"
kubectl logs $NODE_POD_NAME -n $DATENLORD_NAMESPACE -c $NODE_CONTAINER_NAME
echo "SHOW LOGS OF $FUSE_CONTAINER_NAME IN $NODE_POD_NAME"
kubectl logs $NODE_POD_NAME -n $DATENLORD_NAMESPACE -c $FUSE_CONTAINER_NAME
- name: Setup tmate session
if: ${{ failure() }}
uses: mxschmitt/action-tmate@v3