From fe45a5f67bf84c59e51460f87673f58bf2fe4b7c Mon Sep 17 00:00:00 2001 From: Rob Norris Date: Wed, 8 Jan 2025 17:14:33 +1100 Subject: [PATCH] zinject: add "probe" device injection type Injecting a device probe failure is not possible by matching IO types, because probe IO goes to the label regions, which is explicitly excluded from injection. Even if it were possible, it would be awkward to do, because a probe is sequence of reads and writes. This commit adds a new IO "type" to match for injection, which looks for the ZIO_FLAG_PROBE flag instead. Any probe IO will be match the injection record and recieve the wanted error. Sponsored-by: Klara, Inc. Sponsored-by: Wasabi Technology, Inc. Signed-off-by: Rob Norris --- cmd/zinject/zinject.c | 1 + include/sys/zfs_ioctl.h | 3 +- man/man8/zinject.8 | 23 +++--- module/zfs/zio_inject.c | 10 ++- tests/runfiles/common.run | 2 +- tests/zfs-tests/tests/Makefile.am | 1 + .../cli_root/zinject/zinject_probe.ksh | 75 +++++++++++++++++++ 7 files changed, 100 insertions(+), 15 deletions(-) create mode 100755 tests/zfs-tests/tests/functional/cli_root/zinject/zinject_probe.ksh diff --git a/cmd/zinject/zinject.c b/cmd/zinject/zinject.c index d66b0d986f2d..4374e69a7f94 100644 --- a/cmd/zinject/zinject.c +++ b/cmd/zinject/zinject.c @@ -251,6 +251,7 @@ static const char *const iotypestrtable[ZINJECT_IOTYPES] = { [ZINJECT_IOTYPE_FLUSH] = "flush", [ZINJECT_IOTYPE_TRIM] = "trim", [ZINJECT_IOTYPE_ALL] = "all", + [ZINJECT_IOTYPE_PROBE] = "probe", }; static zinject_iotype_t diff --git a/include/sys/zfs_ioctl.h b/include/sys/zfs_ioctl.h index 9afe984e1749..a8c3ffc76455 100644 --- a/include/sys/zfs_ioctl.h +++ b/include/sys/zfs_ioctl.h @@ -471,7 +471,8 @@ typedef enum zinject_iotype { ZINJECT_IOTYPE_TRIM = ZIO_TYPE_TRIM, ZINJECT_IOTYPE_ALL = ZIO_TYPES, /* Room for future expansion for ZIO_TYPE_* */ - ZINJECT_IOTYPES = 16, + ZINJECT_IOTYPE_PROBE = 16, + ZINJECT_IOTYPES, } zinject_iotype_t; typedef struct zfs_share { diff --git a/man/man8/zinject.8 b/man/man8/zinject.8 index abccc4d086e0..53461681bb1d 100644 --- a/man/man8/zinject.8 +++ b/man/man8/zinject.8 @@ -19,11 +19,11 @@ .\" CDDL HEADER END .\" .\" Copyright 2013 Darik Horn . All rights reserved. -.\" Copyright (c) 2024, Klara Inc. +.\" Copyright (c) 2024, 2025, Klara, Inc. .\" .\" lint-ok: WARNING: sections out of conventional order: Sh SYNOPSIS .\" -.Dd December 2, 2024 +.Dd January 14, 2025 .Dt ZINJECT 8 .Os . @@ -265,15 +265,16 @@ will be translated to the appropriate blkid range according to the object's properties. .It Fl s Ar seconds Run for this many seconds before reporting failure. -.It Fl T Ar failure -Set the failure type to one of -.Sy all , -.Sy flush , -.Sy claim , -.Sy free , -.Sy read , -or -.Sy write . +.It Fl T Ar type +Inject the error into I/O of this type. +.Bl -tag -compact -width "read, write, flush, claim, free" +.It Sy read , Sy write , Sy flush , Sy claim , Sy free +Fundamental I/O types +.It Sy all +All fundamental I/O types +.It Sy probe +Device probe I/O +.El .It Fl t Ar mos_type Set this to .Bl -tag -compact -width "spacemap" diff --git a/module/zfs/zio_inject.c b/module/zfs/zio_inject.c index 6848d46f73e7..f90044299cef 100644 --- a/module/zfs/zio_inject.c +++ b/module/zfs/zio_inject.c @@ -386,6 +386,10 @@ zio_match_iotype(zio_t *zio, uint32_t iotype) if (iotype >= ZINJECT_IOTYPES) return (B_FALSE); + /* Probe IOs only match IOTYPE_PROBE, regardless of their type. */ + if (zio->io_flags & ZIO_FLAG_PROBE) + return (iotype == ZINJECT_IOTYPE_PROBE); + /* Standard IO types, match against ZIO type. */ if (iotype < ZINJECT_IOTYPE_ALL) return (iotype == zio->io_type); @@ -405,9 +409,11 @@ zio_handle_device_injection_impl(vdev_t *vd, zio_t *zio, int err1, int err2) /* * We skip over faults in the labels unless it's during device open - * (i.e. zio == NULL) or a device flush (offset is meaningless) + * (i.e. zio == NULL) or a device flush (offset is meaningless). We let + * probe IOs through so we can match them to probe inject records. */ - if (zio != NULL && zio->io_type != ZIO_TYPE_FLUSH) { + if (zio != NULL && zio->io_type != ZIO_TYPE_FLUSH && + !(zio->io_flags & ZIO_FLAG_PROBE)) { uint64_t offset = zio->io_offset; if (offset < VDEV_LABEL_START_SIZE || diff --git a/tests/runfiles/common.run b/tests/runfiles/common.run index 9e186de37369..e2edfc9ebbb5 100644 --- a/tests/runfiles/common.run +++ b/tests/runfiles/common.run @@ -159,7 +159,7 @@ tests = ['json_sanity'] tags = ['functional', 'cli_root', 'json'] [tests/functional/cli_root/zinject] -tests = ['zinject_args', 'zinject_counts'] +tests = ['zinject_args', 'zinject_counts', 'zinject_probe'] pre = post = tags = ['functional', 'cli_root', 'zinject'] diff --git a/tests/zfs-tests/tests/Makefile.am b/tests/zfs-tests/tests/Makefile.am index 044a70d1998b..d0eb4c30db48 100644 --- a/tests/zfs-tests/tests/Makefile.am +++ b/tests/zfs-tests/tests/Makefile.am @@ -616,6 +616,7 @@ nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \ functional/cli_root/json/json_sanity.ksh \ functional/cli_root/zinject/zinject_args.ksh \ functional/cli_root/zinject/zinject_counts.ksh \ + functional/cli_root/zinject/zinject_probe.ksh \ functional/cli_root/zdb/zdb_002_pos.ksh \ functional/cli_root/zdb/zdb_003_pos.ksh \ functional/cli_root/zdb/zdb_004_pos.ksh \ diff --git a/tests/zfs-tests/tests/functional/cli_root/zinject/zinject_probe.ksh b/tests/zfs-tests/tests/functional/cli_root/zinject/zinject_probe.ksh new file mode 100755 index 000000000000..22537a54db73 --- /dev/null +++ b/tests/zfs-tests/tests/functional/cli_root/zinject/zinject_probe.ksh @@ -0,0 +1,75 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or https://opensource.org/licenses/CDDL-1.0. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright (c) 2025, Klara, Inc. +# + +. $STF_SUITE/include/libtest.shlib + +verify_runnable "global" + +log_assert "Check zinject can correctly inject a probe failure." + +DISK1=${DISKS%% *} + +function cleanup +{ + log_pos zinject -c all + log_pos zpool clear $TESTPOOL + log_pos zpool destroy -f $TESTPOOL + log_pos restore_tunable TXG_TIMEOUT +} + +log_onexit cleanup + +log_must zpool create $TESTPOOL $DISK1 + +# set the txg timeout a long way out, to try and avoid the pool syncing +# between error injection and writing +save_tunable TXG_TIMEOUT +log_must set_tunable32 TXG_TIMEOUT 600 + +# force a sync now +log_must zpool sync -f + +# write stuff. this should go into memory, not written yet +log_must dd if=/dev/urandom of=/$TESTPOOL/file bs=1M count=1 + +# inject faults +log_must zinject -d $DISK1 -e io -T probe $TESTPOOL +log_must zinject -d $DISK1 -e io -T write $TESTPOOL + +# force the sync now. backgrounded, because the pool will suspend and we don't +# want to block. +log_pos zpool sync & + +log_note "waiting for pool to suspend" +typeset -i tries=30 +until [[ $(kstat_pool $TESTPOOL state) == "SUSPENDED" ]] ; do + if ((tries-- == 0)); then + log_fail "pool didn't suspend" + fi + sleep 1 +done + +log_pass "zinject can correctly inject a probe failure."