diff --git a/cmd/zinject/zinject.c b/cmd/zinject/zinject.c index 6c856763c958..4374e69a7f94 100644 --- a/cmd/zinject/zinject.c +++ b/cmd/zinject/zinject.c @@ -242,6 +242,36 @@ err_to_str(int err) return ("[unknown]"); } +static const char *const iotypestrtable[ZINJECT_IOTYPES] = { + [ZINJECT_IOTYPE_NULL] = "null", + [ZINJECT_IOTYPE_READ] = "read", + [ZINJECT_IOTYPE_WRITE] = "write", + [ZINJECT_IOTYPE_FREE] = "free", + [ZINJECT_IOTYPE_CLAIM] = "claim", + [ZINJECT_IOTYPE_FLUSH] = "flush", + [ZINJECT_IOTYPE_TRIM] = "trim", + [ZINJECT_IOTYPE_ALL] = "all", + [ZINJECT_IOTYPE_PROBE] = "probe", +}; + +static zinject_iotype_t +str_to_iotype(const char *arg) +{ + for (uint_t iotype = 0; iotype < ZINJECT_IOTYPES; iotype++) + if (iotypestrtable[iotype] != NULL && + strcasecmp(iotypestrtable[iotype], arg) == 0) + return (iotype); + return (ZINJECT_IOTYPES); +} + +static const char * +iotype_to_str(zinject_iotype_t iotype) +{ + if (iotype >= ZINJECT_IOTYPES || iotypestrtable[iotype] == NULL) + return ("[unknown]"); + return (iotypestrtable[iotype]); +} + /* * Print usage message. */ @@ -435,10 +465,6 @@ static int print_device_handler(int id, const char *pool, zinject_record_t *record, void *data) { - static const char *iotypestr[] = { - "null", "read", "write", "free", "claim", "flush", "trim", "all", - }; - int *count = data; if (record->zi_guid == 0 || record->zi_func[0] != '\0') @@ -465,7 +491,7 @@ print_device_handler(int id, const char *pool, zinject_record_t *record, (void) printf("%3d %-15s %llx %-5s %-10s %8.4f%% " "%6lu %6lu\n", id, pool, (u_longlong_t)record->zi_guid, - iotypestr[record->zi_iotype], err_to_str(record->zi_error), + iotype_to_str(record->zi_iotype), err_to_str(record->zi_error), freq, record->zi_match_count, record->zi_inject_count); return (0); @@ -866,7 +892,7 @@ main(int argc, char **argv) int quiet = 0; int error = 0; int domount = 0; - int io_type = ZIO_TYPES; + int io_type = ZINJECT_IOTYPE_ALL; int action = VDEV_STATE_UNKNOWN; err_type_t type = TYPE_INVAL; err_type_t label = TYPE_INVAL; @@ -1060,19 +1086,8 @@ main(int argc, char **argv) } break; case 'T': - if (strcasecmp(optarg, "read") == 0) { - io_type = ZIO_TYPE_READ; - } else if (strcasecmp(optarg, "write") == 0) { - io_type = ZIO_TYPE_WRITE; - } else if (strcasecmp(optarg, "free") == 0) { - io_type = ZIO_TYPE_FREE; - } else if (strcasecmp(optarg, "claim") == 0) { - io_type = ZIO_TYPE_CLAIM; - } else if (strcasecmp(optarg, "flush") == 0) { - io_type = ZIO_TYPE_FLUSH; - } else if (strcasecmp(optarg, "all") == 0) { - io_type = ZIO_TYPES; - } else { + io_type = str_to_iotype(optarg); + if (io_type == ZINJECT_IOTYPES) { (void) fprintf(stderr, "invalid I/O type " "'%s': must be 'read', 'write', 'free', " "'claim', 'flush' or 'all'\n", optarg); @@ -1194,7 +1209,7 @@ main(int argc, char **argv) } if (error == EILSEQ && - (record.zi_freq == 0 || io_type != ZIO_TYPE_READ)) { + (record.zi_freq == 0 || io_type != ZINJECT_IOTYPE_READ)) { (void) fprintf(stderr, "device corrupt errors require " "io type read and a frequency value\n"); libzfs_fini(g_zfs); @@ -1209,9 +1224,9 @@ main(int argc, char **argv) if (record.zi_nlanes) { switch (io_type) { - case ZIO_TYPE_READ: - case ZIO_TYPE_WRITE: - case ZIO_TYPES: + case ZINJECT_IOTYPE_READ: + case ZINJECT_IOTYPE_WRITE: + case ZINJECT_IOTYPE_ALL: break; default: (void) fprintf(stderr, "I/O type for a delay " diff --git a/include/sys/zfs_ioctl.h b/include/sys/zfs_ioctl.h index e61d7644764e..a8c3ffc76455 100644 --- a/include/sys/zfs_ioctl.h +++ b/include/sys/zfs_ioctl.h @@ -456,6 +456,25 @@ typedef enum zinject_type { ZINJECT_DELAY_EXPORT, } zinject_type_t; +typedef enum zinject_iotype { + /* + * Compatibility: zi_iotype used to be set to ZIO_TYPE_, so make sure + * the corresponding ZINJECT_IOTYPE_ matches. Note that existing here + * does not mean that injections are possible for all these types. + */ + ZINJECT_IOTYPE_NULL = ZIO_TYPE_NULL, + ZINJECT_IOTYPE_READ = ZIO_TYPE_READ, + ZINJECT_IOTYPE_WRITE = ZIO_TYPE_WRITE, + ZINJECT_IOTYPE_FREE = ZIO_TYPE_FREE, + ZINJECT_IOTYPE_CLAIM = ZIO_TYPE_CLAIM, + ZINJECT_IOTYPE_FLUSH = ZIO_TYPE_FLUSH, + ZINJECT_IOTYPE_TRIM = ZIO_TYPE_TRIM, + ZINJECT_IOTYPE_ALL = ZIO_TYPES, + /* Room for future expansion for ZIO_TYPE_* */ + ZINJECT_IOTYPE_PROBE = 16, + ZINJECT_IOTYPES, +} zinject_iotype_t; + typedef struct zfs_share { uint64_t z_exportdata; uint64_t z_sharedata; diff --git a/man/man8/zinject.8 b/man/man8/zinject.8 index abccc4d086e0..53461681bb1d 100644 --- a/man/man8/zinject.8 +++ b/man/man8/zinject.8 @@ -19,11 +19,11 @@ .\" CDDL HEADER END .\" .\" Copyright 2013 Darik Horn . All rights reserved. -.\" Copyright (c) 2024, Klara Inc. +.\" Copyright (c) 2024, 2025, Klara, Inc. .\" .\" lint-ok: WARNING: sections out of conventional order: Sh SYNOPSIS .\" -.Dd December 2, 2024 +.Dd January 14, 2025 .Dt ZINJECT 8 .Os . @@ -265,15 +265,16 @@ will be translated to the appropriate blkid range according to the object's properties. .It Fl s Ar seconds Run for this many seconds before reporting failure. -.It Fl T Ar failure -Set the failure type to one of -.Sy all , -.Sy flush , -.Sy claim , -.Sy free , -.Sy read , -or -.Sy write . +.It Fl T Ar type +Inject the error into I/O of this type. +.Bl -tag -compact -width "read, write, flush, claim, free" +.It Sy read , Sy write , Sy flush , Sy claim , Sy free +Fundamental I/O types +.It Sy all +All fundamental I/O types +.It Sy probe +Device probe I/O +.El .It Fl t Ar mos_type Set this to .Bl -tag -compact -width "spacemap" diff --git a/module/zfs/zio_inject.c b/module/zfs/zio_inject.c index f972522b6454..f90044299cef 100644 --- a/module/zfs/zio_inject.c +++ b/module/zfs/zio_inject.c @@ -376,6 +376,31 @@ zio_inject_bitflip_cb(void *data, size_t len, void *private) return (1); /* stop after first flip */ } +/* Test if this zio matches the iotype from the injection record. */ +static boolean_t +zio_match_iotype(zio_t *zio, uint32_t iotype) +{ + ASSERT3P(zio, !=, NULL); + + /* Unknown iotype, maybe from a newer version of zinject. Reject it. */ + if (iotype >= ZINJECT_IOTYPES) + return (B_FALSE); + + /* Probe IOs only match IOTYPE_PROBE, regardless of their type. */ + if (zio->io_flags & ZIO_FLAG_PROBE) + return (iotype == ZINJECT_IOTYPE_PROBE); + + /* Standard IO types, match against ZIO type. */ + if (iotype < ZINJECT_IOTYPE_ALL) + return (iotype == zio->io_type); + + /* Match any standard IO type. */ + if (iotype == ZINJECT_IOTYPE_ALL) + return (B_TRUE); + + return (B_FALSE); +} + static int zio_handle_device_injection_impl(vdev_t *vd, zio_t *zio, int err1, int err2) { @@ -384,9 +409,11 @@ zio_handle_device_injection_impl(vdev_t *vd, zio_t *zio, int err1, int err2) /* * We skip over faults in the labels unless it's during device open - * (i.e. zio == NULL) or a device flush (offset is meaningless) + * (i.e. zio == NULL) or a device flush (offset is meaningless). We let + * probe IOs through so we can match them to probe inject records. */ - if (zio != NULL && zio->io_type != ZIO_TYPE_FLUSH) { + if (zio != NULL && zio->io_type != ZIO_TYPE_FLUSH && + !(zio->io_flags & ZIO_FLAG_PROBE)) { uint64_t offset = zio->io_offset; if (offset < VDEV_LABEL_START_SIZE || @@ -410,9 +437,8 @@ zio_handle_device_injection_impl(vdev_t *vd, zio_t *zio, int err1, int err2) } /* Handle type specific I/O failures */ - if (zio != NULL && - handler->zi_record.zi_iotype != ZIO_TYPES && - handler->zi_record.zi_iotype != zio->io_type) + if (zio != NULL && !zio_match_iotype(zio, + handler->zi_record.zi_iotype)) continue; if (handler->zi_record.zi_error == err1 || @@ -635,10 +661,8 @@ zio_handle_io_delay(zio_t *zio) continue; /* also match on I/O type (e.g., -T read) */ - if (handler->zi_record.zi_iotype != ZIO_TYPES && - handler->zi_record.zi_iotype != zio->io_type) { + if (!zio_match_iotype(zio, handler->zi_record.zi_iotype)) continue; - } /* * Defensive; should never happen as the array allocation diff --git a/tests/runfiles/common.run b/tests/runfiles/common.run index 9e186de37369..e2edfc9ebbb5 100644 --- a/tests/runfiles/common.run +++ b/tests/runfiles/common.run @@ -159,7 +159,7 @@ tests = ['json_sanity'] tags = ['functional', 'cli_root', 'json'] [tests/functional/cli_root/zinject] -tests = ['zinject_args', 'zinject_counts'] +tests = ['zinject_args', 'zinject_counts', 'zinject_probe'] pre = post = tags = ['functional', 'cli_root', 'zinject'] diff --git a/tests/zfs-tests/tests/Makefile.am b/tests/zfs-tests/tests/Makefile.am index 044a70d1998b..d0eb4c30db48 100644 --- a/tests/zfs-tests/tests/Makefile.am +++ b/tests/zfs-tests/tests/Makefile.am @@ -616,6 +616,7 @@ nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \ functional/cli_root/json/json_sanity.ksh \ functional/cli_root/zinject/zinject_args.ksh \ functional/cli_root/zinject/zinject_counts.ksh \ + functional/cli_root/zinject/zinject_probe.ksh \ functional/cli_root/zdb/zdb_002_pos.ksh \ functional/cli_root/zdb/zdb_003_pos.ksh \ functional/cli_root/zdb/zdb_004_pos.ksh \ diff --git a/tests/zfs-tests/tests/functional/cli_root/zinject/zinject_probe.ksh b/tests/zfs-tests/tests/functional/cli_root/zinject/zinject_probe.ksh new file mode 100755 index 000000000000..22537a54db73 --- /dev/null +++ b/tests/zfs-tests/tests/functional/cli_root/zinject/zinject_probe.ksh @@ -0,0 +1,75 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or https://opensource.org/licenses/CDDL-1.0. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright (c) 2025, Klara, Inc. +# + +. $STF_SUITE/include/libtest.shlib + +verify_runnable "global" + +log_assert "Check zinject can correctly inject a probe failure." + +DISK1=${DISKS%% *} + +function cleanup +{ + log_pos zinject -c all + log_pos zpool clear $TESTPOOL + log_pos zpool destroy -f $TESTPOOL + log_pos restore_tunable TXG_TIMEOUT +} + +log_onexit cleanup + +log_must zpool create $TESTPOOL $DISK1 + +# set the txg timeout a long way out, to try and avoid the pool syncing +# between error injection and writing +save_tunable TXG_TIMEOUT +log_must set_tunable32 TXG_TIMEOUT 600 + +# force a sync now +log_must zpool sync -f + +# write stuff. this should go into memory, not written yet +log_must dd if=/dev/urandom of=/$TESTPOOL/file bs=1M count=1 + +# inject faults +log_must zinject -d $DISK1 -e io -T probe $TESTPOOL +log_must zinject -d $DISK1 -e io -T write $TESTPOOL + +# force the sync now. backgrounded, because the pool will suspend and we don't +# want to block. +log_pos zpool sync & + +log_note "waiting for pool to suspend" +typeset -i tries=30 +until [[ $(kstat_pool $TESTPOOL state) == "SUSPENDED" ]] ; do + if ((tries-- == 0)); then + log_fail "pool didn't suspend" + fi + sleep 1 +done + +log_pass "zinject can correctly inject a probe failure."