Skip to content

Commit

Permalink
kubectl-ko: fix subnet diagnose failure (kubeovn#3808)
Browse files Browse the repository at this point in the history 
Signed-off-by: zhangzujian <[email protected]>
  • Loading branch information
@zhangzujian
zhangzujian committed Mar 11, 2024
1 parent efe0809 commit 4c19787
Show file tree
Hide file tree
Showing 6 changed files with 65 additions and 50 deletions.
22 changes: 9 additions & 13 deletions cmd/pinger/pinger.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,19 +41,15 @@ func CmdMain() {
}

if config.EnableVerboseConnCheck {
go func() {
addr := util.JoinHostPort("0.0.0.0", config.TCPConnCheckPort)
if err := util.TCPConnectivityListen(addr); err != nil {
util.LogFatalAndExit(err, "failed to start TCP listen on addr %s ", addr)
}
}()

go func() {
addr := util.JoinHostPort("0.0.0.0", config.UDPConnCheckPort)
if err := util.UDPConnectivityListen(addr); err != nil {
util.LogFatalAndExit(err, "failed to start UDP listen on addr %s ", addr)
}
}()
addr := util.JoinHostPort("0.0.0.0", config.UDPConnCheckPort)
if err = util.UDPConnectivityListen(addr); err != nil {
util.LogFatalAndExit(err, "failed to start UDP listen on addr %s", addr)
}

addr = util.JoinHostPort("0.0.0.0", config.TCPConnCheckPort)
if err = util.TCPConnectivityListen(addr); err != nil {
util.LogFatalAndExit(err, "failed to start TCP listen on addr %s", addr)
}
}
}
pinger.StartPinger(config)
Expand Down
35 changes: 23 additions & 12 deletions dist/images/kubectl-ko
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@ CONN_CHECK_SERVER="conn-check-server"
PERF_GC_COMMAND=()
LAST_PERF_FAILED_LOG=""
OVN_TRACE=0
TCP_CONN_CHECK_PORT=${TCP_CONN_CHECK_PORT:-8100}
UDP_CONN_CHECK_PORT=${UDP_CONN_CHECK_PORT:-8101}

showHelp(){
echo "kubectl ko {subcommand} [option...]"
Expand Down Expand Up @@ -543,27 +545,31 @@ spec:
command:
- /kube-ovn/kube-ovn-pinger
args:
- --ds-namespace=$KUBE_OVN_NS
- --ds-name=$subnetName-$CONN_CHECK_SERVER
- --enable-verbose-conn-check=true
- --tcp-conn-check-port=$TCP_CONN_CHECK_PORT
- --udp-conn-check-port=$UDP_CONN_CHECK_PORT
env:
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
readinessProbe:
failureThreshold: 1
initialDelaySeconds: 3
periodSeconds: 5
successThreshold: 1
tcpSocket:
port: $TCP_CONN_CHECK_PORT
timeoutSeconds: 1
EOF

isfailed=true
for i in {0..59}
do
if kubectl wait pod --for=condition=Ready -l app=$CONN_CHECK_LABEL -n $KUBE_OVN_NS 2> /dev/null; then
isfailed=false
break
fi
sleep 1; \
done

if $isfailed; then
if kubectl -n $KUBE_OVN_NS rollout status ds $subnetName-$CONN_CHECK_SERVER --timeout=30s; then
isfailed=false
else
echo "Error ds $subnetName-$CONN_CHECK_SERVER pod not ready"
return
fi
}

Expand Down Expand Up @@ -721,10 +727,15 @@ diagnose(){
fi

pingers=$(kubectl -n $KUBE_OVN_NS get po --no-headers -o custom-columns=NAME:.metadata.name -l app=kube-ovn-pinger)
cmd=(
/kube-ovn/kube-ovn-pinger --mode=job --ds-name=$subnetName-$CONN_CHECK_SERVER --ds-namespace=$KUBE_OVN_NS
--tcp-conn-check-port=$TCP_CONN_CHECK_PORT --udp-conn-check-port=$UDP_CONN_CHECK_PORT
--external-address=1.1.1.1,2606:4700:4700::1111 --target-ip-ports=$targetIPPorts
)
for pinger in $pingers
do
echo "#### pinger $pinger on namespace $KUBE_OVN_NS diagnose results:"
kubectl exec -n $KUBE_OVN_NS "$pinger" -- /kube-ovn/kube-ovn-pinger --mode=job --ds-name=$subnetName-$CONN_CHECK_SERVER --ds-namespace=$KUBE_OVN_NS --enable-verbose-conn-check=true --external-address=114.114.114.114,2400:3200::1 --target-ip-ports=$targetIPPorts
kubectl exec -n $KUBE_OVN_NS "$pinger" -- ${cmd[*]}
echo ""
done
;;
Expand Down
38 changes: 23 additions & 15 deletions pkg/util/net.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -555,13 +555,17 @@ func TCPConnectivityListen(address string) error {
return fmt.Errorf("listen failed with err %v", err)
}

for {
conn, err := listener.Accept()
if err != nil {
continue
go func() {
for {
conn, err := listener.Accept()
if err != nil {
continue
}
_ = conn.Close()
}
_ = conn.Close()
}
}()

return nil
}

func UDPConnectivityCheck(address string) error {
Expand Down Expand Up @@ -609,17 +613,21 @@ func UDPConnectivityListen(address string) error {

buffer := make([]byte, 1024)

for {
_, clientAddr, err := conn.ReadFromUDP(buffer)
if err != nil {
continue
}
go func() {
for {
_, clientAddr, err := conn.ReadFromUDP(buffer)
if err != nil {
continue
}

_, err = conn.WriteToUDP([]byte("health check"), clientAddr)
if err != nil {
continue
_, err = conn.WriteToUDP([]byte("health check"), clientAddr)
if err != nil {
continue
}
}
}
}()

return nil
}

func GetDefaultListenAddr() string {
Expand Down
14 changes: 7 additions & 7 deletions test/e2e/kube-ovn/kubectl-ko/kubectl-ko.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -130,10 +130,10 @@ var _ = framework.Describe("[group:kubectl-ko]", func() {
supportARP := !f.VersionPriorTo(1, 11)
supportDstMAC := !f.VersionPriorTo(1, 10)
if !supportARP {
framework.Logf("Support for ARP was introduce in v1.11")
framework.Logf("Support for ARP was introduced in v1.11")
}
if !supportDstMAC {
framework.Logf("Support for destination MAC was introduce in v1.10")
framework.Logf("Support for destination MAC was introduced in v1.10")
}

for _, ip := range pod.Status.PodIPs {
Expand Down Expand Up @@ -165,7 +165,7 @@ var _ = framework.Describe("[group:kubectl-ko]", func() {
})

framework.ConformanceIt(`should support "kubectl ko trace <pod> <args...>" for pod with host network`, func() {
f.SkipVersionPriorTo(1, 12, "This feature was introduce in v1.12")
f.SkipVersionPriorTo(1, 12, "This feature was introduced in v1.12")

ginkgo.By("Creating pod " + podName + " with host network")
pod := framework.MakePod(namespaceName, podName, nil, nil, "", nil, nil)
Expand Down Expand Up @@ -198,7 +198,7 @@ var _ = framework.Describe("[group:kubectl-ko]", func() {
})

framework.ConformanceIt(`should support "kubectl ko trace <node> <args...>"`, func() {
f.SkipVersionPriorTo(1, 12, "This feature was introduce in v1.12")
f.SkipVersionPriorTo(1, 12, "This feature was introduced in v1.12")

ginkgo.By("Getting nodes")
nodeList, err := e2enode.GetReadySchedulableNodes(context.Background(), cs)
Expand Down Expand Up @@ -237,15 +237,15 @@ var _ = framework.Describe("[group:kubectl-ko]", func() {
})

framework.ConformanceIt(`should support "kubectl ko log kube-ovn all"`, func() {
f.SkipVersionPriorTo(1, 12, "This feature was introduce in v1.12")
f.SkipVersionPriorTo(1, 12, "This feature was introduced in v1.12")
components := [...]string{"kube-ovn", "ovn", "ovs", "linux", "all"}
for _, component := range components {
execOrDie(fmt.Sprintf("ko log %s", component))
}
})

framework.ConformanceIt(`should support "kubectl ko diagnose subnet IPPorts "`, func() {
f.SkipVersionPriorTo(1, 12, "This feature was introduce in v1.12")
framework.ConformanceIt(`should support "kubectl ko diagnose subnet IPPorts <IPPorts>"`, func() {
f.SkipVersionPriorTo(1, 12, "This feature was introduced in v1.12")
execOrDie("ko diagnose subnet ovn-default")
execOrDie("ko diagnose IPPorts tcp-114.114.114.114-53,udp-114.114.114.114-53")
})
Expand Down
4 changes: 2 additions & 2 deletions test/e2e/kube-ovn/node/node.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@ var _ = framework.OrderedDescribe("[group:node]", func() {
})

framework.ConformanceIt("should access overlay pods using node ip", func() {
f.SkipVersionPriorTo(1, 12, "This feature was introduce in v1.12")
f.SkipVersionPriorTo(1, 12, "This feature was introduced in v1.12")

ginkgo.By("Creating subnet " + subnetName)
subnet = framework.MakeSubnet(subnetName, "", cidr, "", "", "", nil, nil, nil)
Expand Down Expand Up @@ -145,7 +145,7 @@ var _ = framework.OrderedDescribe("[group:node]", func() {
})

framework.ConformanceIt("should access overlay services using node ip", func() {
f.SkipVersionPriorTo(1, 12, "This feature was introduce in v1.12")
f.SkipVersionPriorTo(1, 12, "This feature was introduced in v1.12")

ginkgo.By("Creating subnet " + subnetName)
subnet = framework.MakeSubnet(subnetName, "", cidr, "", "", "", nil, nil, nil)
Expand Down
2 changes: 1 addition & 1 deletion test/e2e/kube-ovn/switch_lb_rule/switch_lb_rule.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@ var _ = framework.Describe("[group:slr]", func() {
})

framework.ConformanceIt("should access sts and slr svc ok", func() {
f.SkipVersionPriorTo(1, 12, "This feature was introduce in v1.12")
f.SkipVersionPriorTo(1, 12, "This feature was introduced in v1.12")
ginkgo.By("1. Creating sts svc with slr")
var (
clientPod *corev1.Pod
Expand Down

0 comments on commit 4c19787

Please sign in to comment.