perf: use block syscall enter for epoll_wait

poll_default_linux.go

@@ -97,13 +97,23 @@ func (p *defaultPoll) Wait() (err error) {
 		if n == p.size && p.size < 128*1024 {
 			p.Reset(p.size<<1, caps)
 		}
-		n, err = EpollWait(p.fd, p.events, msec)
+		// msec: 0(raw) => 1ms(sched,raw) => -1(block_syscall)
+		// poller's G will hold P at most 1ms
+		if msec >= 0 {
+			n, err = EpollWaitRaw(p.fd, p.events, msec)
+		} else { // < 0
+			n, err = EpollWaitBlock(p.fd, p.events, msec)
+		}
 		if err != nil && err != syscall.EINTR {
 			return err
 		}
 		if n <= 0 {
-			msec = -1
-			runtime.Gosched()
+			if msec > 0 {
+				msec = -1 // no need to sched, because we will use block syscall
+			} else if msec == 0 {
+				msec = 1
+				runtime.Gosched()
+			}
 			continue
 		}
 		msec = 0

sys_epoll_linux.go

@@ -12,8 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-//go:build !arm64 && !loong64
-// +build !arm64,!loong64
+//go:build linux
+// +build linux
 
 package netpoll
 
@@ -22,13 +22,25 @@ import (
 	"unsafe"
 )
 
-const EPOLLET = -syscall.EPOLLET
+//go:linkname entersyscallblock runtime.entersyscallblock
+func entersyscallblock()
 
-type epollevent struct {
-	events uint32
-	data   [8]byte // unaligned uintptr
+//go:linkname exitsyscall runtime.exitsyscall
+func exitsyscall()
+
+//go:nosplit
+func callEntersyscallblock() {
+	entersyscallblock()
+}
+
+//go:nosplit
+func callExitsyscall() {
+	exitsyscall()
 }
 
+//go:noescape
+func BlockSyscall6(num, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2, errno uintptr)
+
 // EpollCreate implements epoll_create1.
 func EpollCreate(flag int) (fd int, err error) {
 	var r0 uintptr
@@ -51,14 +63,28 @@ func EpollCtl(epfd int, op int, fd int, event *epollevent) (err error) {
 // EpollWait implements epoll_wait.
 func EpollWait(epfd int, events []epollevent, msec int) (n int, err error) {
 	var r0 uintptr
-	var _p0 = unsafe.Pointer(&events[0])
-	if msec == 0 {
-		r0, _, err = syscall.RawSyscall6(syscall.SYS_EPOLL_WAIT, uintptr(epfd), uintptr(_p0), uintptr(len(events)), 0, 0, 0)
-	} else {
-		r0, _, err = syscall.Syscall6(syscall.SYS_EPOLL_WAIT, uintptr(epfd), uintptr(_p0), uintptr(len(events)), uintptr(msec), 0, 0)
+	r0, _, err = syscall.Syscall6(SYS_EPOLL_WAIT, uintptr(epfd), uintptr(unsafe.Pointer(&events[0])), uintptr(len(events)), uintptr(msec), 0, 0)
+	if err == syscall.Errno(0) {
+		err = nil
 	}
+	return int(r0), err
+}
+
+func EpollWaitRaw(epfd int, events []epollevent, msec int) (n int, err error) {
+	var r0 uintptr
+	r0, _, err = syscall.RawSyscall6(SYS_EPOLL_WAIT, uintptr(epfd), uintptr(unsafe.Pointer(&events[0])), uintptr(len(events)), uintptr(msec), 0, 0)
 	if err == syscall.Errno(0) {
 		err = nil
 	}
 	return int(r0), err
 }
+
+func EpollWaitBlock(epfd int, events []epollevent, msec int) (n int, err error) {
+	r0, _, errno := BlockSyscall6(SYS_EPOLL_WAIT, uintptr(epfd), uintptr(unsafe.Pointer(&events[0])), uintptr(len(events)), uintptr(msec), 0, 0)
+	if errno == 0 {
+		err = nil
+	} else {
+		err = syscall.Errno(errno)
+	}
+	return int(r0), err
+}

sys_epoll_linux_amd64.go

@@ -0,0 +1,30 @@
+// Copyright 2022 CloudWeGo Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build !arm64 && !loong64
+// +build !arm64,!loong64
+
+package netpoll
+
+import (
+	"syscall"
+)
+
+const EPOLLET = -syscall.EPOLLET
+const SYS_EPOLL_WAIT = syscall.SYS_EPOLL_WAIT
+
+type epollevent struct {
+	events uint32
+	data   [8]byte // unaligned uintptr
+}

sys_epoll_linux_amd64.s

@@ -0,0 +1,41 @@
+// Copyright 2024 CloudWeGo Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "textflag.h"
+
+// func BlockSyscall6(num, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2, errno uintptr)
+TEXT ·BlockSyscall6(SB),NOSPLIT,$0-80
+	CALL	·callEntersyscallblock(SB)
+	MOVQ	num+0(FP), AX	// syscall entry
+	MOVQ	a1+8(FP), DI
+	MOVQ	a2+16(FP), SI
+	MOVQ	a3+24(FP), DX
+	MOVQ	a4+32(FP), R10
+	MOVQ	a5+40(FP), R8
+	MOVQ	a6+48(FP), R9
+	SYSCALL
+	CMPQ	AX, $0xfffffffffffff001
+	JLS	ok
+	MOVQ	$-1, r1+56(FP)
+	MOVQ	$0, r2+64(FP)
+	NEGQ	AX
+	MOVQ	AX, errno+72(FP)
+	CALL	·callExitsyscall(SB)
+	RET
+ok:
+	MOVQ	AX, r1+56(FP)
+	MOVQ	DX, r2+64(FP)
+	MOVQ	$0, errno+72(FP)
+	CALL	·callExitsyscall(SB)
+	RET

sys_epoll_linux_arm64.go

@@ -12,51 +12,20 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build arm64
+// +build arm64
+
 package netpoll
 
 import (
 	"syscall"
-	"unsafe"
 )
 
 const EPOLLET = syscall.EPOLLET
+const SYS_EPOLL_WAIT = syscall.SYS_EPOLL_PWAIT
 
 type epollevent struct {
 	events uint32
 	_      int32
 	data   [8]byte // unaligned uintptr
 }
-
-// EpollCreate implements epoll_create1.
-func EpollCreate(flag int) (fd int, err error) {
-	var r0 uintptr
-	r0, _, err = syscall.RawSyscall(syscall.SYS_EPOLL_CREATE1, uintptr(flag), 0, 0)
-	if err == syscall.Errno(0) {
-		err = nil
-	}
-	return int(r0), err
-}
-
-// EpollCtl implements epoll_ctl.
-func EpollCtl(epfd int, op int, fd int, event *epollevent) (err error) {
-	_, _, err = syscall.RawSyscall6(syscall.SYS_EPOLL_CTL, uintptr(epfd), uintptr(op), uintptr(fd), uintptr(unsafe.Pointer(event)), 0, 0)
-	if err == syscall.Errno(0) {
-		err = nil
-	}
-	return err
-}
-
-// EpollWait implements epoll_wait.
-func EpollWait(epfd int, events []epollevent, msec int) (n int, err error) {
-	var r0 uintptr
-	var _p0 = unsafe.Pointer(&events[0])
-	if msec == 0 {
-		r0, _, err = syscall.RawSyscall6(syscall.SYS_EPOLL_PWAIT, uintptr(epfd), uintptr(_p0), uintptr(len(events)), 0, 0, 0)
-	} else {
-		r0, _, err = syscall.Syscall6(syscall.SYS_EPOLL_PWAIT, uintptr(epfd), uintptr(_p0), uintptr(len(events)), uintptr(msec), 0, 0)
-	}
-	if err == syscall.Errno(0) {
-		err = nil
-	}
-	return int(r0), err
-}

sys_epoll_linux_arm64.s

@@ -0,0 +1,42 @@
+// Copyright 2024 CloudWeGo Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "textflag.h"
+
+// func BlockSyscall6(num, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2, errno uintptr)
+TEXT ·BlockSyscall6(SB),NOSPLIT,$0-80
+	BL	·callEntersyscallblock(SB)
+	MOVD	num+0(FP), R8	// syscall entry
+	MOVD	a1+8(FP), R0
+	MOVD	a2+16(FP), R1
+	MOVD	a3+24(FP), R2
+	MOVD	a4+32(FP), R3
+	MOVD	a5+40(FP), R4
+	MOVD	a6+48(FP), R5
+	SVC
+	CMN	$4095, R0
+	BCC	ok
+	MOVD	$-1, R4
+	MOVD	R4, r1+56(FP)
+	MOVD	ZR, r2+64(FP)
+	NEG	R0, R0
+	MOVD	R0, errno+72(FP)
+	BL	·callExitsyscall(SB)
+	RET
+ok:
+	MOVD	R0, r1+56(FP)
+	MOVD	R1, r2+64(FP)
+	MOVD	ZR, errno+72(FP)
+	BL	·callExitsyscall(SB)
+	RET

sys_epoll_linux_loong64.go

@@ -12,54 +12,20 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-//go:build linux && loong64
-// +build linux,loong64
+//go:build loong64
+// +build loong64
 
 package netpoll
 
 import (
 	"syscall"
-	"unsafe"
 )
 
 const EPOLLET = syscall.EPOLLET
+const SYS_EPOLL_WAIT = syscall.SYS_EPOLL_PWAIT
 
 type epollevent struct {
 	events uint32
 	_      int32
 	data   [8]byte // unaligned uintptr
 }
-
-// EpollCreate implements epoll_create1.
-func EpollCreate(flag int) (fd int, err error) {
-	var r0 uintptr
-	r0, _, err = syscall.RawSyscall(syscall.SYS_EPOLL_CREATE1, uintptr(flag), 0, 0)
-	if err == syscall.Errno(0) {
-		err = nil
-	}
-	return int(r0), err
-}
-
-// EpollCtl implements epoll_ctl.
-func EpollCtl(epfd int, op int, fd int, event *epollevent) (err error) {
-	_, _, err = syscall.RawSyscall6(syscall.SYS_EPOLL_CTL, uintptr(epfd), uintptr(op), uintptr(fd), uintptr(unsafe.Pointer(event)), 0, 0)
-	if err == syscall.Errno(0) {
-		err = nil
-	}
-	return err
-}
-
-// EpollWait implements epoll_wait.
-func EpollWait(epfd int, events []epollevent, msec int) (n int, err error) {
-	var r0 uintptr
-	var _p0 = unsafe.Pointer(&events[0])
-	if msec == 0 {
-		r0, _, err = syscall.RawSyscall6(syscall.SYS_EPOLL_PWAIT, uintptr(epfd), uintptr(_p0), uintptr(len(events)), 0, 0, 0)
-	} else {
-		r0, _, err = syscall.Syscall6(syscall.SYS_EPOLL_PWAIT, uintptr(epfd), uintptr(_p0), uintptr(len(events)), uintptr(msec), 0, 0)
-	}
-	if err == syscall.Errno(0) {
-		err = nil
-	}
-	return int(r0), err
-}