zinject: add "probe" device injection type

Injecting a device probe failure is not possible by matching IO types,
because probe IO goes to the label regions, which is explicitly excluded
from injection. Even if it were possible, it would be awkward to do,
because a probe is sequence of reads and writes.

This commit adds a new IO "type" to match for injection, which looks for
the ZIO_FLAG_PROBE flag instead. Any probe IO will be match the
injection record and recieve the wanted error.

Sponsored-by: Klara, Inc.
Sponsored-by: Wasabi Technology, Inc.
Reviewed-by: Alexander Motin <mav@FreeBSD.org>
Reviewed-by: Tony Hutter <hutter2@llnl.gov>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: Rob Norris <rob.norris@klarasystems.com>
Closes #16947

cmd/zinject/zinject.c

@@ -251,6 +251,7 @@ static const char *const iotypestrtable[ZINJECT_IOTYPES] = {
 	[ZINJECT_IOTYPE_FLUSH]	= "flush",
 	[ZINJECT_IOTYPE_TRIM]	= "trim",
 	[ZINJECT_IOTYPE_ALL]	= "all",
+	[ZINJECT_IOTYPE_PROBE]	= "probe",
 };
 
 static zinject_iotype_t

include/sys/zfs_ioctl.h

@@ -471,7 +471,8 @@ typedef enum zinject_iotype {
 	ZINJECT_IOTYPE_TRIM	= ZIO_TYPE_TRIM,
 	ZINJECT_IOTYPE_ALL	= ZIO_TYPES,
 	/* Room for future expansion for ZIO_TYPE_* */
-	ZINJECT_IOTYPES		= 16,
+	ZINJECT_IOTYPE_PROBE	= 16,
+	ZINJECT_IOTYPES,
 } zinject_iotype_t;
 
 typedef struct zfs_share {

man/man8/zinject.8

@@ -19,11 +19,11 @@
 .\" CDDL HEADER END
 .\"
 .\" Copyright 2013 Darik Horn <dajhorn@vanadac.com>. All rights reserved.
-.\" Copyright (c) 2024, Klara Inc.
+.\" Copyright (c) 2024, 2025, Klara, Inc.
 .\"
 .\" lint-ok: WARNING: sections out of conventional order: Sh SYNOPSIS
 .\"
-.Dd December 2, 2024
+.Dd January 14, 2025
 .Dt ZINJECT 8
 .Os
 .
@@ -265,15 +265,16 @@ will be translated to the appropriate blkid range according to the
 object's properties.
 .It Fl s Ar seconds
 Run for this many seconds before reporting failure.
-.It Fl T Ar failure
-Set the failure type to one of
-.Sy all ,
-.Sy flush ,
-.Sy claim ,
-.Sy free ,
-.Sy read ,
-or
-.Sy write .
+.It Fl T Ar type
+Inject the error into I/O of this type.
+.Bl -tag -compact -width "read, write, flush, claim, free"
+.It Sy read , Sy write , Sy flush , Sy claim , Sy free
+Fundamental I/O types
+.It Sy all
+All fundamental I/O types
+.It Sy probe
+Device probe I/O
+.El
 .It Fl t Ar mos_type
 Set this to
 .Bl -tag -compact -width "spacemap"

module/zfs/zio_inject.c

@@ -386,6 +386,10 @@ zio_match_iotype(zio_t *zio, uint32_t iotype)
 	if (iotype >= ZINJECT_IOTYPES)
 		return (B_FALSE);
 
+	/* Probe IOs only match IOTYPE_PROBE, regardless of their type. */
+	if (zio->io_flags & ZIO_FLAG_PROBE)
+		return (iotype == ZINJECT_IOTYPE_PROBE);
+
 	/* Standard IO types, match against ZIO type. */
 	if (iotype < ZINJECT_IOTYPE_ALL)
 		return (iotype == zio->io_type);
@@ -405,9 +409,11 @@ zio_handle_device_injection_impl(vdev_t *vd, zio_t *zio, int err1, int err2)
 
 	/*
 	 * We skip over faults in the labels unless it's during device open
-	 * (i.e. zio == NULL) or a device flush (offset is meaningless)
+	 * (i.e. zio == NULL) or a device flush (offset is meaningless). We let
+	 * probe IOs through so we can match them to probe inject records.
 	 */
-	if (zio != NULL && zio->io_type != ZIO_TYPE_FLUSH) {
+	if (zio != NULL && zio->io_type != ZIO_TYPE_FLUSH &&
+	    !(zio->io_flags & ZIO_FLAG_PROBE)) {
 		uint64_t offset = zio->io_offset;
 
 		if (offset < VDEV_LABEL_START_SIZE ||

tests/runfiles/common.run

@@ -159,7 +159,7 @@ tests = ['json_sanity']
 tags = ['functional', 'cli_root', 'json']
 
 [tests/functional/cli_root/zinject]
-tests = ['zinject_args', 'zinject_counts']
+tests = ['zinject_args', 'zinject_counts', 'zinject_probe']
 pre =
 post =
 tags = ['functional', 'cli_root', 'zinject']

tests/zfs-tests/tests/Makefile.am

@@ -616,6 +616,7 @@ nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \
 	functional/cli_root/json/json_sanity.ksh \
 	functional/cli_root/zinject/zinject_args.ksh \
 	functional/cli_root/zinject/zinject_counts.ksh \
+	functional/cli_root/zinject/zinject_probe.ksh \
 	functional/cli_root/zdb/zdb_002_pos.ksh \
 	functional/cli_root/zdb/zdb_003_pos.ksh \
 	functional/cli_root/zdb/zdb_004_pos.ksh \

tests/zfs-tests/tests/functional/cli_root/zinject/zinject_probe.ksh

@@ -0,0 +1,75 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or https://opensource.org/licenses/CDDL-1.0.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2025, Klara, Inc.
+#
+
+. $STF_SUITE/include/libtest.shlib
+
+verify_runnable "global"
+
+log_assert "Check zinject can correctly inject a probe failure."
+
+DISK1=${DISKS%% *}
+
+function cleanup
+{
+	log_pos zinject -c all
+	log_pos zpool clear $TESTPOOL
+	log_pos zpool destroy -f $TESTPOOL
+	log_pos restore_tunable TXG_TIMEOUT
+}
+
+log_onexit cleanup
+
+log_must zpool create $TESTPOOL $DISK1
+
+# set the txg timeout a long way out, to try and avoid the pool syncing
+# between error injection and writing
+save_tunable TXG_TIMEOUT
+log_must set_tunable32 TXG_TIMEOUT 600
+
+# force a sync now
+log_must zpool sync -f
+
+# write stuff. this should go into memory, not written yet
+log_must dd if=/dev/urandom of=/$TESTPOOL/file bs=1M count=1
+
+# inject faults
+log_must zinject -d $DISK1 -e io -T probe $TESTPOOL
+log_must zinject -d $DISK1 -e io -T write $TESTPOOL
+
+# force the sync now. backgrounded, because the pool will suspend and we don't
+# want to block.
+log_pos zpool sync &
+
+log_note "waiting for pool to suspend"
+typeset -i tries=30
+until [[ $(kstat_pool $TESTPOOL state) == "SUSPENDED" ]] ; do
+	if ((tries-- == 0)); then
+		log_fail "pool didn't suspend"
+	fi
+	sleep 1
+done
+
+log_pass "zinject can correctly inject a probe failure."