Add tests for configurable TIME-WAIT reuse delay

This set of tests accompanies the kernel patch set which adds a knob for
setting the TIME-WAIT reuse delay [1].

Tests cover connection reincarnation through TIME-WAIT reuse with the
default reuse delay (1 second) as well as with a reuse delay radically
shortened down to 1 millisecond.

We exercise both the happy and failure scenarios, where TW reuse is not
possible because the required delay period has not elapsed yet.

[1] https://lore.kernel.org/all/20241204-jakub-krn-909-poc-msec-tw-tstamp-v1-0-8b54467a0f34@cloudflare.com/

gtests/net/packetdrill/symbols_linux.c

@@ -168,6 +168,8 @@ struct int_symbol platform_symbols_table[] = {
 #endif
 	{ IPV6_TCLASS,                      "IPV6_TCLASS"                     },
 	{ IPV6_HOPLIMIT,                    "IPV6_HOPLIMIT"                   },
+	{ IP_BIND_ADDRESS_NO_PORT,          "IP_BIND_ADDRESS_NO_PORT"         },
+	{ IP_LOCAL_PORT_RANGE,              "IP_LOCAL_PORT_RANGE"             },
 
 	{ TCP_NODELAY,                      "TCP_NODELAY"                     },
 	{ TCP_MAXSEG,                       "TCP_MAXSEG"                      },

gtests/net/tcp/ts_recent/tw_reuse/fast_tw_reuse_fail_after_500usec.pkt

@@ -0,0 +1,52 @@
+// Check that TIME-WAIT 4-tuple can't be reused after less than a millisecond
+
+--ip_version=ipv4
+--tcp_ts_tick_usecs=1000
+--tolerance_percent=1
+
+`
+../../common/defaults.sh
+../../../tcp/common/set_sysctls.py  /proc/sys/net/ipv4/tcp_tw_reuse=1 # global enable
+../../../tcp/common/set_sysctls.py  /proc/sys/net/ipv4/tcp_tw_reuse_delay=1 # msec
+`
+
+//
+// Prime TIME-WAIT - open first connection and close it.
+// Use source port 0xdead (57005).
+//
+
+    0   socket(..., SOCK_STREAM | SOCK_NONBLOCK, IPPROTO_TCP) = 3
+
+   +0   setsockopt(3, SOL_IP, IP_BIND_ADDRESS_NO_PORT, [1], 4) = 0
+   +0   setsockopt(3, SOL_IP, IP_LOCAL_PORT_RANGE, [0xdeaddead], 4) = 0
+
+   +0   connect(3, ..., ...) = -1 EINPROGRESS (Operation now in progress)
+
+   +0   > S  0:0(0)                 <mss 1460,sackOK,TS val 1001 ecr 0,nop,wscale 8>
++.001   < S. 0:0(0) ack 1 win 65535 <mss 1460,sackOK,TS val 3001 ecr 1001,nop,wscale 8>
+   +0   > .  1:1(0) ack 1           <nop,nop,TS val 1002 ecr 3001>
+
+   +0   close(3) = 0
+
+   +0   > F. 1:1(0) ack 1       <nop,nop,TS val 1002 ecr 3001>
++.001   < F. 1:1(0) ack 2 win 8 <nop,nop,TS val 3002 ecr 1002>
+   +0   > .  2:2(0) ack 2       <nop,nop,TS val 1003 ecr 3002>
+
+//
+// Reincarnation after less than 1 msec - open and close second connection.
+// Use the same source port.
+//
+
+   +0   socket(..., SOCK_STREAM | SOCK_NONBLOCK, IPPROTO_TCP) = 3
+
+   +0   setsockopt(3, SOL_IP, IP_BIND_ADDRESS_NO_PORT, [1], 4) = 0
+   +0   setsockopt(3, SOL_IP, IP_LOCAL_PORT_RANGE, [0xdeaddead], 4) = 0
+
+// Delay for 500 usec and assume it takes packetdrill less than 500 usec to
+// reincarnate a connection (the interval from last ACK to next connect()), so
+// the total delay is less than 1 msec. On a consumer laptop connection
+// reincarnation takes ~200 usec.
++.0005  connect(3, ..., ...) = -1 EADDRNOTAVAIL (Cannot assign requested address)
+
+   +0   close(3) = 0
+

gtests/net/tcp/ts_recent/tw_reuse/fast_tw_reuse_ok_after_1msec.pkt

@@ -0,0 +1,61 @@
+// Check that TIME-WAIT 4-tuple can be reused after a 1 millisecond period
+
+--ip_version=ipv4
+--tcp_ts_tick_usecs=1000
+--tolerance_percent=1
+
+`
+../../common/defaults.sh
+../../../tcp/common/set_sysctls.py  /proc/sys/net/ipv4/tcp_tw_reuse=1 # global enable
+../../../tcp/common/set_sysctls.py  /proc/sys/net/ipv4/tcp_tw_reuse_delay=1 # msec
+`
+
+//
+// Prime TIME-WAIT - open first connection and close it.
+// Use source port 0xdead (57005).
+//
+
+    0   socket(..., SOCK_STREAM | SOCK_NONBLOCK, IPPROTO_TCP) = 3
+
+   +0   setsockopt(3, SOL_IP, IP_BIND_ADDRESS_NO_PORT, [1], 4) = 0
+   +0   setsockopt(3, SOL_IP, IP_LOCAL_PORT_RANGE, [0xdeaddead], 4) = 0
+
+   +0   connect(3, ..., ...) = -1 EINPROGRESS (Operation now in progress)
+
+   +0   > S  0:0(0)                 <mss 1460,sackOK,TS val 1001 ecr 0,nop,wscale 8>
++.001   < S. 0:0(0) ack 1 win 65535 <mss 1460,sackOK,TS val 3001 ecr 1001,nop,wscale 8>
+   +0   > .  1:1(0) ack 1           <...>
+
+   +0   close(3) = 0
+
+   +0   > F. 1:1(0) ack 1       <nop,nop,TS val 1002 ecr 3001>
++.001   < F. 1:1(0) ack 2 win 8 <nop,nop,TS val 3002 ecr 1002>
+   +0   > .  2:2(0) ack 2       <nop,nop,TS val 1003 ecr 3002>
+
+//
+// Reincarnation after 1 millisecond - open and close second connection.
+// Use the same source port.
+//
+
+   +0   socket(..., SOCK_STREAM | SOCK_NONBLOCK, IPPROTO_TCP) = 3
+
+   +0   setsockopt(3, SOL_IP, IP_BIND_ADDRESS_NO_PORT, [1], 4) = 0
+   +0   setsockopt(3, SOL_IP, IP_LOCAL_PORT_RANGE, [0xdeaddead], 4) = 0
+
+
+// Delay for 1 millisecond, plus an extra millisecond to avoid test flakiness.
+// Kernel TIME-WAIT reuse timer has millisecond resolution. Actual reuse delay
+// can be up to 1 msec longer than the setting due to timestamp rounding.
++.002   connect(3, ..., ...) = -1 EINPROGRESS (Operation now in progress)
+
+// NOTE: Packetdrill doesn't actually track timestamps across connection
+// reincarnations despite the fact that the random offset doesn't change.
+   +0   > S  0:0(0)                 <mss 1460,sackOK,TS val 1005 ecr 3002,nop,wscale 8>
++.001   < S. 0:0(0) ack 1 win 65535 <mss 1460,sackOK,TS val 3003 ecr 1005,nop,wscale 8>
+   +0   > .  1:1(0) ack 1           <nop,nop,TS val 1006 ecr 3003>
+
+   +0   close(3) = 0
+
+   +0   > F. 1:1(0) ack 1         <nop,nop,TS val 1006 ecr 3003>
++.001   < F. 1:1(0) ack 2 win 8   <nop,nop,TS val 3004 ecr 1006>
+   +0   > .  2:2(0) ack 2         <nop,nop,TS val 1007 ecr 3004>

gtests/net/tcp/ts_recent/tw_reuse/tw_reuse_fail_after_0sec.pkt

@@ -0,0 +1,47 @@
+// Check that TIME-WAIT 4-tuple can't be reused immediately
+
+--ip_version=ipv4
+--tcp_ts_tick_usecs=1000
+--tolerance_percent=1
+
+`
+../../common/defaults.sh
+../../../tcp/common/set_sysctls.py  /proc/sys/net/ipv4/tcp_tw_reuse=1 # global enable
+../../../tcp/common/set_sysctls.py  /proc/sys/net/ipv4/tcp_tw_reuse_delay=1000 # msec
+`
+
+//
+// Prime TIME-WAIT - open first connection and close it.
+// Use source port 0xdead (57005).
+//
+
+    0   socket(..., SOCK_STREAM | SOCK_NONBLOCK, IPPROTO_TCP) = 3
+
+   +0   setsockopt(3, SOL_IP, IP_BIND_ADDRESS_NO_PORT, [1], 4) = 0
+   +0   setsockopt(3, SOL_IP, IP_LOCAL_PORT_RANGE, [0xdeaddead], 4) = 0
+
+   +0   connect(3, ..., ...) = -1 EINPROGRESS (Operation now in progress)
+
+   +0   > S  0:0(0)                 <mss 1460,sackOK,TS val 1001 ecr 0,nop,wscale 8>
++.001   < S. 0:0(0) ack 1 win 65535 <mss 1460,sackOK,TS val 3001 ecr 1001,nop,wscale 8>
+   +0   > .  1:1(0) ack 1           <nop,nop,TS val 1002 ecr 3001>
+
+   +0   close(3) = 0
+
+   +0   > F. 1:1(0) ack 1       <nop,nop,TS val 1002 ecr 3001>
++.001   < F. 1:1(0) ack 2 win 8 <nop,nop,TS val 3002 ecr 1002>
+   +0   > .  2:2(0) ack 2       <nop,nop,TS val 1003 ecr 3002>
+
+//
+// Reincarnation without delay - open and close second connection.
+// Use the same source port.
+//
+
+   +0   socket(..., SOCK_STREAM | SOCK_NONBLOCK, IPPROTO_TCP) = 3
+
+   +0   setsockopt(3, SOL_IP, IP_BIND_ADDRESS_NO_PORT, [1], 4) = 0
+   +0   setsockopt(3, SOL_IP, IP_LOCAL_PORT_RANGE, [0xdeaddead], 4) = 0
+
++0.00   connect(3, ..., ...) = -1 EADDRNOTAVAIL (Cannot assign requested address)
+
+   +0   close(3) = 0

gtests/net/tcp/ts_recent/tw_reuse/tw_reuse_fail_after_1sec_without_tsopt.pkt

@@ -0,0 +1,48 @@
+// Check that TIME-WAIT 4-tuple can't be reused after the TIME-WAIT reuse delay
+// when peer doesn't support TSopt
+
+--ip_version=ipv4
+--tcp_ts_tick_usecs=1000
+--tolerance_percent=1
+
+`
+../../common/defaults.sh
+../../../tcp/common/set_sysctls.py  /proc/sys/net/ipv4/tcp_tw_reuse=1 # global enable
+../../../tcp/common/set_sysctls.py  /proc/sys/net/ipv4/tcp_tw_reuse_delay=1000 # msec
+`
+
+//
+// Prime TIME-WAIT - open first connection and close it.
+// Use source port 0xdead (57005).
+//
+
+    0   socket(..., SOCK_STREAM | SOCK_NONBLOCK, IPPROTO_TCP) = 3
+
+   +0   setsockopt(3, SOL_IP, IP_BIND_ADDRESS_NO_PORT, [1], 4) = 0
+   +0   setsockopt(3, SOL_IP, IP_LOCAL_PORT_RANGE, [0xdeaddead], 4) = 0
+
+   +0   connect(3, ..., ...) = -1 EINPROGRESS (Operation now in progress)
+
+   +0   > S  0:0(0)                 <mss 1460,sackOK,TS val 1001 ecr 0,nop,wscale 8>
++.001   < S. 0:0(0) ack 1 win 65535 <mss 1460,sackOK,nop,nop,nop,wscale 8>
+   +0   > .  1:1(0) ack 1
+
+   +0   close(3) = 0
+
+   +0   > F. 1:1(0) ack 1
++.001   < F. 1:1(0) ack 2 win 8
+   +0   > .  2:2(0) ack 2
+
+//
+// Reincarnation after 1 second - open and close second connection.
+// Use same source port.
+//
+
+   +0   socket(..., SOCK_STREAM | SOCK_NONBLOCK, IPPROTO_TCP) = 3
+
+   +0   setsockopt(3, SOL_IP, IP_BIND_ADDRESS_NO_PORT, [1], 4) = 0
+   +0   setsockopt(3, SOL_IP, IP_LOCAL_PORT_RANGE, [0xdeaddead], 4) = 0
+
++1.00   connect(3, ..., ...) = -1 EADDRNOTAVAIL (Cannot assign requested address)
+
+   +0   close(3) = 0

gtests/net/tcp/ts_recent/tw_reuse/tw_reuse_fail_after_990msec.pkt

@@ -0,0 +1,48 @@
+// Check that TCP 4-tuple can't be reincarnated when we are close to TIME-WAIT
+// reuse threshold (990 msec have elapsed).
+
+--ip_version=ipv4
+--tcp_ts_tick_usecs=1000
+--tolerance_percent=1
+
+`
+../../common/defaults.sh
+../../../tcp/common/set_sysctls.py  /proc/sys/net/ipv4/tcp_tw_reuse=1 # global enable
+../../../tcp/common/set_sysctls.py  /proc/sys/net/ipv4/tcp_tw_reuse_delay=1000 # msec
+`
+
+//
+// Prime TIME-WAIT - open first connection and close it.
+// Use source port 0xdead (57005).
+//
+
+    0   socket(..., SOCK_STREAM | SOCK_NONBLOCK, IPPROTO_TCP) = 3
+
+   +0   setsockopt(3, SOL_IP, IP_BIND_ADDRESS_NO_PORT, [1], 4) = 0
+   +0   setsockopt(3, SOL_IP, IP_LOCAL_PORT_RANGE, [0xdeaddead], 4) = 0
+
+   +0   connect(3, ..., ...) = -1 EINPROGRESS (Operation now in progress)
+
+   +0   > S  0:0(0)                 <mss 1460,sackOK,TS val 1001 ecr 0,nop,wscale 8>
++.001   < S. 0:0(0) ack 1 win 65535 <mss 1460,sackOK,TS val 3001 ecr 1001,nop,wscale 8>
+   +0   > .  1:1(0) ack 1           <nop,nop,TS val 1002 ecr 3001>
+
+   +0   close(3) = 0
+
+   +0   > F. 1:1(0) ack 1       <nop,nop,TS val 1002 ecr 3001>
++.001   < F. 1:1(0) ack 2 win 8 <nop,nop,TS val 3002 ecr 1002>
+   +0   > .  2:2(0) ack 2       <nop,nop,TS val 1003 ecr 3002>
+
+//
+// Reincarnation after almost 1 second - open and close second connection.
+// Use the same source port.
+//
+
+   +0   socket(..., SOCK_STREAM | SOCK_NONBLOCK, IPPROTO_TCP) = 3
+
+   +0   setsockopt(3, SOL_IP, IP_BIND_ADDRESS_NO_PORT, [1], 4) = 0
+   +0   setsockopt(3, SOL_IP, IP_LOCAL_PORT_RANGE, [0xdeaddead], 4) = 0
+
++0.99   connect(3, ..., ...) = -1 EADDRNOTAVAIL (Cannot assign requested address)
+
+   +0   close(3) = 0

gtests/net/tcp/ts_recent/tw_reuse/tw_reuse_ok_after_0sec_with_portlock.pkt

@@ -0,0 +1,59 @@
+// Check that TIME-WAIT 4-tuple can be reused immediately if bind() was used to
+// reserve the source port.
+
+--ip_version=ipv4
+--bind_port=57005
+--tcp_ts_tick_usecs=1000
+--tolerance_percent=1
+
+`
+../../common/defaults.sh
+../../../tcp/common/set_sysctls.py  /proc/sys/net/ipv4/tcp_tw_reuse=1 # global enable
+../../../tcp/common/set_sysctls.py  /proc/sys/net/ipv4/tcp_tw_reuse_delay=1000 # msec
+`
+
+//
+// Prime TIME-WAIT - open first connection and close it.
+// Use source port 57005. Reserve it with bind().
+//
+
+    0   socket(..., SOCK_STREAM | SOCK_NONBLOCK, IPPROTO_TCP) = 3
+
+   +0   setsockopt(3, SOL_SOCKET, SO_REUSEADDR, [1], 4) = 0
+   +0   bind(3, ..., ...) = 0
+
+   +0   connect(3, ..., ...) = -1 EINPROGRESS (Operation now in progress)
+
+   +0   > S  0:0(0)                 <mss 1460,sackOK,TS val 1001 ecr 0,nop,wscale 8>
++.001   < S. 0:0(0) ack 1 win 65535 <mss 1460,sackOK,TS val 3001 ecr 1001,nop,wscale 8>
+   +0   > .  1:1(0) ack 1           <nop,nop,TS val 1002 ecr 3001>
+
+   +0   close(3) = 0
+
+   +0   > F. 1:1(0) ack 1       <nop,nop,TS val 1002 ecr 3001>
++.001   < F. 1:1(0) ack 2 win 8 <nop,nop,TS val 3002 ecr 1002>
+   +0   > .  2:2(0) ack 2       <nop,nop,TS val 1003 ecr 3002>
+
+//
+// Reincarnation without delay - open and close second connection.
+// Use same source port.
+//
+
+   +0   socket(..., SOCK_STREAM | SOCK_NONBLOCK, IPPROTO_TCP) = 3
+
+   +0   setsockopt(3, SOL_SOCKET, SO_REUSEADDR, [1], 4) = 0
+   +0   bind(3, ..., ...) = 0
+
++0.00   connect(3, ..., ...) = -1 EINPROGRESS (Operation now in progress)
+
+// NOTE: Packetdrill doesn't actually track timestamps across connection
+// reincarnations despite the fact that the random offset doesn't change.
+   +0   > S  0:0(0)                 <mss 1460,sackOK,TS val 1003 ecr 0,nop,wscale 8>
++.001   < S. 0:0(0) ack 1 win 65535 <mss 1460,sackOK,TS val 3002 ecr 1003,nop,wscale 8>
+   +0   > .  1:1(0) ack 1           <nop,nop,TS val 1004 ecr 3002>
+
+   +0   close(3) = 0
+
+   +0   > F. 1:1(0) ack 1       <nop,nop,TS val 1004 ecr 3002>
++.001   < F. 1:1(0) ack 2 win 8 <nop,nop,TS val 3003 ecr 1004>
+   +0   > .  2:2(0) ack 2       <nop,nop,TS val 1005 ecr 3003>

gtests/net/tcp/ts_recent/tw_reuse/tw_reuse_ok_after_1sec.pkt

@@ -0,0 +1,60 @@
+// Check that TIME-WAIT 4-tuple can be reused after a 1 second period.
+
+--ip_version=ipv4
+--tcp_ts_tick_usecs=1000
+--tolerance_percent=1
+
+`
+../../common/defaults.sh
+../../../tcp/common/set_sysctls.py  /proc/sys/net/ipv4/tcp_tw_reuse=1 # global enable
+../../../tcp/common/set_sysctls.py  /proc/sys/net/ipv4/tcp_tw_reuse_delay=1000 # msec
+`
+
+//
+// Prime TIME-WAIT - open first connection and close it.
+// Use source port 0xdead (57005).
+//
+
+    0   socket(..., SOCK_STREAM | SOCK_NONBLOCK, IPPROTO_TCP) = 3
+
+   +0   setsockopt(3, SOL_IP, IP_BIND_ADDRESS_NO_PORT, [1], 4) = 0
+   +0   setsockopt(3, SOL_IP, IP_LOCAL_PORT_RANGE, [0xdeaddead], 4) = 0
+
+   +0   connect(3, ..., ...) = -1 EINPROGRESS (Operation now in progress)
+
+   +0   > S  0:0(0)                 <mss 1460,sackOK,TS val 1001 ecr 0,nop,wscale 8>
++.001   < S. 0:0(0) ack 1 win 65535 <mss 1460,sackOK,TS val 3001 ecr 1001,nop,wscale 8>
+   +0   > .  1:1(0) ack 1           <...>
+
+   +0   close(3) = 0
+
+   +0   > F. 1:1(0) ack 1       <nop,nop,TS val 1002 ecr 3001>
++.001   < F. 1:1(0) ack 2 win 8 <nop,nop,TS val 3002 ecr 1002>
+   +0   > .  2:2(0) ack 2       <nop,nop,TS val 1003 ecr 3002>
+
+//
+// Reincarnation after 1 second - open and close second connection.
+// Use the same source port.
+//
+
+   +0   socket(..., SOCK_STREAM | SOCK_NONBLOCK, IPPROTO_TCP) = 3
+
+   +0   setsockopt(3, SOL_IP, IP_BIND_ADDRESS_NO_PORT, [1], 4) = 0
+   +0   setsockopt(3, SOL_IP, IP_LOCAL_PORT_RANGE, [0xdeaddead], 4) = 0
+
+// Avoid spurious failures with an extra 1 millisecond delay.
+// Kernel TIME-WAIT reuse timer has millisecond resolution. Actual reuse delay
+// can be up to 1 msec longer than the setting due to timestamp rounding.
++1.01   connect(3, ..., ...) = -1 EINPROGRESS (Operation now in progress)
+
+// NOTE: Packetdrill doesn't actually track timestamps across connection
+// reincarnations despite the fact that the random offset doesn't change.
+   +0   > S  0:0(0)                 <mss 1460,sackOK,TS val 2004 ecr 3002,nop,wscale 8>
++.001   < S. 0:0(0) ack 1 win 65535 <mss 1460,sackOK,TS val 4003 ecr 2004,nop,wscale 8>
+   +0   > .  1:1(0) ack 1           <nop,nop,TS val 2005 ecr 4003>
+
+   +0   close(3) = 0
+
+   +0   > F. 1:1(0) ack 1         <nop,nop,TS val 2005 ecr 4003>
++.001   < F. 1:1(0) ack 2 win 8   <nop,nop,TS val 4004 ecr 2005>
+   +0   > .  2:2(0) ack 2         <nop,nop,TS val 2006 ecr 4004>