zinject: add "probe" device injection type

Injecting a device probe failure is not possible by matching IO types,
because probe IO goes to the label regions, which is explicitly excluded
from injection. Even if it were possible, it would be awkward to do,
because a probe is sequence of reads and writes.

This commit adds a new IO "type" to match for injection, which looks for
the ZIO_FLAG_PROBE flag instead. Any probe IO will be match the
injection record and recieve the wanted error.

Sponsored-by: Klara, Inc.
Sponsored-by: Wasabi Technology, Inc.
Signed-off-by: Rob Norris <[email protected]>

cmd/zinject/zinject.c

@@ -251,6 +251,7 @@ static const char *const iotypestrtable[ZINJECT_IOTYPES] = {
 	[ZINJECT_IOTYPE_FLUSH]	= "flush",
 	[ZINJECT_IOTYPE_TRIM]	= "trim",
 	[ZINJECT_IOTYPE_ALL]	= "all",
+	[ZINJECT_IOTYPE_PROBE]	= "probe",
 };
 
 static zinject_iotype_t

include/sys/zfs_ioctl.h

@@ -471,7 +471,8 @@ typedef enum zinject_iotype {
 	ZINJECT_IOTYPE_TRIM	= ZIO_TYPE_TRIM,
 	ZINJECT_IOTYPE_ALL	= ZIO_TYPES,
 	/* Room for future expansion for ZIO_TYPE_* */
-	ZINJECT_IOTYPES		= 16,
+	ZINJECT_IOTYPE_PROBE	= 16,
+	ZINJECT_IOTYPES,
 } zinject_iotype_t;
 
 typedef struct zfs_share {

man/man8/zinject.8

@@ -19,11 +19,11 @@
 .\" CDDL HEADER END
 .\"
 .\" Copyright 2013 Darik Horn <[email protected]>. All rights reserved.
-.\" Copyright (c) 2024, Klara Inc.
+.\" Copyright (c) 2024, 2025, Klara, Inc.
 .\"
 .\" lint-ok: WARNING: sections out of conventional order: Sh SYNOPSIS
 .\"
-.Dd December 2, 2024
+.Dd January 14, 2025
 .Dt ZINJECT 8
 .Os
 .
@@ -265,15 +265,16 @@ will be translated to the appropriate blkid range according to the
 object's properties.
 .It Fl s Ar seconds
 Run for this many seconds before reporting failure.
-.It Fl T Ar failure
-Set the failure type to one of
-.Sy all ,
-.Sy flush ,
-.Sy claim ,
-.Sy free ,
-.Sy read ,
-or
-.Sy write .
+.It Fl T Ar type
+Inject the error into I/O of this type.
+.Bl -tag -compact -width "read, write, flush, claim, free"
+.It Sy read , Sy write , Sy flush , Sy claim , Sy free
+Fundamental I/O types
+.It Sy all
+All fundamental I/O types
+.It Sy probe
+Device probe I/O
+.El
 .It Fl t Ar mos_type
 Set this to
 .Bl -tag -compact -width "spacemap"

module/zfs/zio_inject.c

@@ -386,6 +386,10 @@ zio_match_iotype(zio_t *zio, uint32_t iotype)
 	if (iotype >= ZINJECT_IOTYPES)
 		return (B_FALSE);
 
+	/* Probe IOs only match IOTYPE_PROBE, regardless of their type. */
+	if (zio->io_flags & ZIO_FLAG_PROBE)
+		return (iotype == ZINJECT_IOTYPE_PROBE);
+
 	/* Standard IO types, match against ZIO type. */
 	if (iotype < ZINJECT_IOTYPE_ALL)
 		return (iotype == zio->io_type);
@@ -405,9 +409,11 @@ zio_handle_device_injection_impl(vdev_t *vd, zio_t *zio, int err1, int err2)
 
 	/*
 	 * We skip over faults in the labels unless it's during device open
-	 * (i.e. zio == NULL) or a device flush (offset is meaningless)
+	 * (i.e. zio == NULL) or a device flush (offset is meaningless). We let
+	 * probe IOs through so we can match them to probe inject records.
 	 */
-	if (zio != NULL && zio->io_type != ZIO_TYPE_FLUSH) {
+	if (zio != NULL && zio->io_type != ZIO_TYPE_FLUSH &&
+	    !(zio->io_flags & ZIO_FLAG_PROBE)) {
 		uint64_t offset = zio->io_offset;
 
 		if (offset < VDEV_LABEL_START_SIZE ||

tests/runfiles/common.run

@@ -159,7 +159,7 @@ tests = ['json_sanity']
 tags = ['functional', 'cli_root', 'json']
 
 [tests/functional/cli_root/zinject]
-tests = ['zinject_args', 'zinject_counts']
+tests = ['zinject_args', 'zinject_counts', 'zinject_probe']
 pre =
 post =
 tags = ['functional', 'cli_root', 'zinject']

tests/zfs-tests/tests/Makefile.am

@@ -616,6 +616,7 @@ nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \
 	functional/cli_root/json/json_sanity.ksh \
 	functional/cli_root/zinject/zinject_args.ksh \
 	functional/cli_root/zinject/zinject_counts.ksh \
+	functional/cli_root/zinject/zinject_probe.ksh \
 	functional/cli_root/zdb/zdb_002_pos.ksh \
 	functional/cli_root/zdb/zdb_003_pos.ksh \
 	functional/cli_root/zdb/zdb_004_pos.ksh \

tests/zfs-tests/tests/functional/cli_root/zinject/zinject_probe.ksh

@@ -0,0 +1,63 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or https://opensource.org/licenses/CDDL-1.0.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2025, Klara, Inc.
+#
+
+. $STF_SUITE/include/libtest.shlib
+
+verify_runnable "global"
+
+log_assert "Check zinject can correctly inject a probe failure."
+
+DISK1=${DISKS%% *}
+
+function cleanup
+{
+	zinject -c all
+	default_cleanup_noexit
+}
+
+log_onexit cleanup
+
+zpool create $TESTPOOL $DISK1
+
+log_must zinject -d $DISK1 -e io -T probe $TESTPOOL
+log_must zinject -d $DISK1 -e io -T write $TESTPOOL
+
+log_must dd if=/dev/urandom of=/$TESTPOOL/file bs=1M count=1
+
+log_note "waiting for pool to suspend"
+typeset -i tries=30
+until [[ $(kstat_pool $TESTPOOL state) == "SUSPENDED" ]] ; do
+	if ((tries-- == 0)); then
+		log_fail "pool didn't suspend"
+	fi
+	sleep 1
+done
+
+log_must zinject -c all
+log_must zpool clear $TESTPOOL
+log_must zpool destroy -f $TESTPOOL
+
+log_pass "zinject can correctly inject a probe failure."