From 71d191ef25d1c60e6725c07b6b94a0184f7db2eb Mon Sep 17 00:00:00 2001 From: Herb Wartens Date: Wed, 19 Apr 2023 13:22:59 -0700 Subject: [PATCH] Allow MMP to bypass waiting for other threads At our site we have seen cases when multi-modifier protection is enabled (multihost=on) on our pool and the pool gets suspended due to a single disk that is failing and responding very slowly. Our pools have 90 disks in them and we expect disks to fail. The current version of MMP requires that we wait for other writers before moving on. When a disk is responding very slowly, we observed that waiting here was bad enough to cause the pool to suspend. This change allows the MMP thread to bypass waiting for other threads and reduces the chances the pool gets suspended. Reviewed-by: Brian Behlendorf Signed-off-by: Herb Wartens Closes #14659 --- include/sys/spa.h | 2 ++ module/zfs/mmp.c | 2 +- module/zfs/spa_misc.c | 29 ++++++++++++++++++++++++++--- 3 files changed, 29 insertions(+), 4 deletions(-) diff --git a/include/sys/spa.h b/include/sys/spa.h index a1c102020..b96a9ef1d 100644 --- a/include/sys/spa.h +++ b/include/sys/spa.h @@ -977,6 +977,8 @@ extern int spa_import_progress_set_state(uint64_t pool_guid, extern int spa_config_tryenter(spa_t *spa, int locks, const void *tag, krw_t rw); extern void spa_config_enter(spa_t *spa, int locks, const void *tag, krw_t rw); +extern void spa_config_enter_mmp(spa_t *spa, int locks, const void *tag, + krw_t rw); extern void spa_config_exit(spa_t *spa, int locks, const void *tag); extern int spa_config_held(spa_t *spa, int locks, krw_t rw); diff --git a/module/zfs/mmp.c b/module/zfs/mmp.c index ef0e01df3..25eea0752 100644 --- a/module/zfs/mmp.c +++ b/module/zfs/mmp.c @@ -445,7 +445,7 @@ mmp_write_uberblock(spa_t *spa) uint64_t offset; hrtime_t lock_acquire_time = gethrtime(); - spa_config_enter(spa, SCL_STATE, mmp_tag, RW_READER); + spa_config_enter_mmp(spa, SCL_STATE, mmp_tag, RW_READER); lock_acquire_time = gethrtime() - lock_acquire_time; if (lock_acquire_time > (MSEC2NSEC(MMP_MIN_INTERVAL) / 10)) zfs_dbgmsg("MMP SCL_STATE acquisition pool '%s' took %llu ns " diff --git a/module/zfs/spa_misc.c b/module/zfs/spa_misc.c index 1475a4a53..54a0eeccf 100644 --- a/module/zfs/spa_misc.c +++ b/module/zfs/spa_misc.c @@ -493,8 +493,9 @@ spa_config_tryenter(spa_t *spa, int locks, const void *tag, krw_t rw) return (1); } -void -spa_config_enter(spa_t *spa, int locks, const void *tag, krw_t rw) +static void +spa_config_enter_impl(spa_t *spa, int locks, const void *tag, krw_t rw, + int mmp_flag) { (void) tag; int wlocks_held = 0; @@ -509,7 +510,8 @@ spa_config_enter(spa_t *spa, int locks, const void *tag, krw_t rw) continue; mutex_enter(&scl->scl_lock); if (rw == RW_READER) { - while (scl->scl_writer || scl->scl_write_wanted) { + while (scl->scl_writer || + (!mmp_flag && scl->scl_write_wanted)) { cv_wait(&scl->scl_cv, &scl->scl_lock); } } else { @@ -527,6 +529,27 @@ spa_config_enter(spa_t *spa, int locks, const void *tag, krw_t rw) ASSERT3U(wlocks_held, <=, locks); } +void +spa_config_enter(spa_t *spa, int locks, const void *tag, krw_t rw) +{ + spa_config_enter_impl(spa, locks, tag, rw, 0); +} + +/* + * The spa_config_enter_mmp() allows the mmp thread to cut in front of + * outstanding write lock requests. This is needed since the mmp updates are + * time sensitive and failure to service them promptly will result in a + * suspended pool. This pool suspension has been seen in practice when there is + * a single disk in a pool that is responding slowly and presumably about to + * fail. + */ + +void +spa_config_enter_mmp(spa_t *spa, int locks, const void *tag, krw_t rw) +{ + spa_config_enter_impl(spa, locks, tag, rw, 1); +} + void spa_config_exit(spa_t *spa, int locks, const void *tag) {