zhack: add "mmp reclaim" to recover a pool stranded by MMP

cmd/zhack.c

@@ -45,6 +45,7 @@
 #include <sys/dsl_synctask.h>
 #include <sys/vdev.h>
 #include <sys/vdev_impl.h>
+#include <sys/mmp.h>
 #include <sys/fs/zfs.h>
 #include <sys/dmu_objset.h>
 #include <sys/dsl_pool.h>
@@ -111,6 +112,13 @@ usage(void)
 	    "\n"
 	    "    <device> : path to vdev\n"
 	    "\n"
+	    "    mmp reclaim <pool>\n"
+	    "        import a pool whose MMP claim cannot reach every mirror\n"
+	    "        leg the config still expects, then mark the unreachable\n"
+	    "        leaves offline so ordinary imports succeed.  Manual\n"
+	    "        recovery: fence the peer first, this cannot see a\n"
+	    "        live host whose legs are all invisible from here\n"
+	    "\n"
 	    "    metaslab leak <pool>\n"
 	    "        apply allocation map from zdb to specified pool\n");
 	exit(1);
@@ -601,6 +609,174 @@ zhack_do_action_idle(int argc, char **argv)
 	spa_close(spa, FTAG);
 }
 
+/*
+ * Collect the mirror legs this host could not open.  vdev_not_present is set
+ * during import for any leaf whose open failed (vdev.c), and a leg in that
+ * state is what the relaxed claim forgives, so it is also what has to be
+ * marked offline for the ordinary imports which follow to succeed.
+ */
+static void
+zhack_collect_absent(vdev_t *vd, uint64_t *guids, uint_t *n, uint_t max)
+{
+	/*
+	 * Skip the same subtrees mmp_claim_uberblock_sync() skips.  The claim
+	 * never counts these, so offlining them buys nothing, and offlining a
+	 * log leg would drag in spa_reset_logs().  Pruned at the interior node
+	 * because the log flag lives on the top-level vdev.
+	 */
+	if (vd->vdev_islog || vd->vdev_isspare || vd->vdev_isl2cache ||
+	    vd->vdev_ishole || vd->vdev_ops == &vdev_indirect_ops)
+		return;
+
+	for (uint64_t c = 0; c < vd->vdev_children; c++)
+		zhack_collect_absent(vd->vdev_child[c], guids, n, max);
+
+	if (!vd->vdev_ops->vdev_op_leaf || !vd->vdev_not_present)
+		return;
+
+	/*
+	 * Take only the legs the relaxed claim can forgive, which are the
+	 * direct children of a top-level mirror: the relaxation lives in the
+	 * nparity == 0 branch and walks that vdev's children.  A raidz or
+	 * draid member is required as parity+1 in aggregate and never demanded
+	 * individually, so an absent one does not raise the requirement and
+	 * offlining it would be a persistent change that buys nothing.
+	 */
+	if (vd->vdev_parent != vd->vdev_top ||
+	    vdev_get_nparity(vd->vdev_top) != 0)
+		return;
+
+	VERIFY3U(*n, <, max);
+	guids[(*n)++] = vd->vdev_guid;
+}
+
+static int
+zhack_do_mmp_reclaim(int argc, char **argv)
+{
+	spa_t *spa;
+	char *target;
+	uint64_t *guids;
+	uint_t nguids = 0, max;
+	int c, failed = 0;
+
+	optind = 1;
+	while ((c = getopt(argc, argv, "+")) != -1) {
+		switch (c) {
+		default:
+			usage();
+			break;
+		}
+	}
+	argc -= optind;
+	argv += optind;
+
+	if (argc < 1) {
+		(void) fprintf(stderr, "error: missing pool name\n");
+		usage();
+	}
+	target = argv[0];
+
+	/*
+	 * Relax the claim for this import only.  The write, the wait and the
+	 * re-read are untouched, so a competing importer which shares any
+	 * visibility with us is still refused.
+	 */
+	mmp_claim_relaxed = B_TRUE;
+	zhack_spa_open(target, B_FALSE, FTAG, &spa);
+	mmp_claim_relaxed = B_FALSE;
+
+	/*
+	 * Nothing to recover on a pool without multihost: no claim runs, so an
+	 * ordinary import already succeeds with the absent leaves simply
+	 * missing.  Offlining them here would be a permanent change to a pool
+	 * that never needed this tool.
+	 */
+	if (!spa_multihost(spa)) {
+		(void) fprintf(stderr, "%s: multihost is off, so no uberblock "
+		    "claim runs and an ordinary import will succeed; refusing "
+		    "to offline anything\n", target);
+		spa_close(spa, FTAG);
+		return (1);
+	}
+
+	/* Takes SCL_VDEV itself, so size the array before we hold it. */
+	max = MAX(vdev_count_leaves(spa), 1);
+	guids = umem_zalloc(max * sizeof (uint64_t), UMEM_NOFAIL);
+
+	spa_config_enter(spa, SCL_VDEV, FTAG, RW_READER);
+	zhack_collect_absent(spa->spa_root_vdev, guids, &nguids, max);
+	spa_config_exit(spa, SCL_VDEV, FTAG);
+
+	if (nguids == 0) {
+		(void) fprintf(stdout, "%s: imported, no absent leaves to "
+		    "mark offline\n", target);
+	}
+
+	for (uint_t i = 0; i < nguids; i++) {
+		int error = vdev_offline(spa, guids[i], 0);
+
+		if (error == 0) {
+			(void) fprintf(stdout, "%s: marked absent leaf %llu "
+			    "offline\n", target, (u_longlong_t)guids[i]);
+			continue;
+		}
+
+		failed++;
+		if (error == EBUSY) {
+			(void) fprintf(stderr, "%s: leaf %llu holds data no "
+			    "other leaf has, left online\n", target,
+			    (u_longlong_t)guids[i]);
+		} else {
+			(void) fprintf(stderr, "%s: could not offline leaf "
+			    "%llu: %s\n", target, (u_longlong_t)guids[i],
+			    strerror(error));
+		}
+	}
+
+	if (failed != 0) {
+		/*
+		 * Not "the pool still needs zhack": this run exports cleanly
+		 * under our own hostid, so the next import here skips the
+		 * activity check entirely.  The cost lands on the next host
+		 * to take the pool, whose claim will count the leaves left
+		 * online and refuse.
+		 */
+		(void) fprintf(stderr, "%s: %d absent leaves are still "
+		    "online; a later import from another host will count "
+		    "them and be refused\n", target, failed);
+	}
+
+	umem_free(guids, max * sizeof (uint64_t));
+	spa_close(spa, FTAG);
+
+	return (failed == 0 ? 0 : 1);
+}
+
+static int
+zhack_do_mmp(int argc, char **argv)
+{
+	char *subcommand;
+
+	argc--;
+	argv++;
+	if (argc == 0) {
+		(void) fprintf(stderr,
+		    "error: no mmp operation specified\n");
+		usage();
+	}
+
+	subcommand = argv[0];
+	if (strcmp(subcommand, "reclaim") == 0) {
+		return (zhack_do_mmp_reclaim(argc, argv));
+	} else {
+		(void) fprintf(stderr, "error: unknown subcommand: %s\n",
+		    subcommand);
+		usage();
+	}
+
+	return (0);
+}
+
 static int
 zhack_do_action(int argc, char **argv)
 {
@@ -1372,6 +1548,8 @@ main(int argc, char **argv)
 		rv = zhack_do_action(argc, argv);
 	} else if (strcmp(subcommand, "feature") == 0) {
 		rv = zhack_do_feature(argc, argv);
+	} else if (strcmp(subcommand, "mmp") == 0) {
+		rv = zhack_do_mmp(argc, argv);
 	} else if (strcmp(subcommand, "label") == 0) {
 		return (zhack_do_label(argc, argv));
 	} else if (strcmp(subcommand, "metaslab") == 0) {

include/sys/mmp.h

@@ -74,6 +74,11 @@ extern uint64_t zfs_multihost_interval;
 extern uint_t zfs_multihost_fail_intervals;
 extern uint_t zfs_multihost_import_intervals;
 
+#ifndef _KERNEL
+/* Manual recovery only, see the comment in mmp.c.  Never in the module. */
+extern boolean_t mmp_claim_relaxed;
+#endif
+
 #ifdef	__cplusplus
 }
 #endif

man/man1/zhack.1

@@ -23,7 +23,7 @@
 .\"
 .\" lint-ok: WARNING: sections out of conventional order: Sh SYNOPSIS
 .\"
-.Dd May 3, 2023
+.Dd August 3, 2026
 .Dt ZHACK 1
 .Os
 .
@@ -124,6 +124,47 @@ Example:
 .
 .It Xo
 .Nm zhack
+.Cm mmp reclaim
+.Ar pool
+.Xc
+Import a
+.Ar pool
+whose uberblock claim cannot reach every mirror leg the configuration still
+expects to be present, then mark the unreachable leaves offline so that
+ordinary imports succeed, and export it again.
+.Pp
+This exists for one situation: a host has failed together with the mirror legs
+attached to it, so the surviving labels still describe those legs as healthy,
+and every later import demands writes to them that nobody can make.
+.Pp
+.Sy This does not make the operation safe on its own .
+A live host whose legs are
+.Em all
+invisible from here cannot be detected by any write and read scheme, and this
+command will take the pool from it.
+Fence the failed host, or otherwise prove it is powered off, before running
+this.
+.Pp
+The activity check itself is unchanged: every write the reachable legs allow,
+the wait, and the re-read all still happen, so a competing host which shares
+any leg with this one is still detected and the import is refused.
+Only the number of writes the claim demands is lowered, and only for mirror
+legs this host cannot open, each of which is then marked offline.
+.Pp
+The pool is left exported, and imports degraded with those leaves offline.
+Bring each one back with
+.Xr zpool-online 8
+once the hardware returns.
+A leaf holding the only copy of some data cannot be offlined: it is reported
+and left online, and imports from other hosts stay refused.
+No leaf is marked offline on a pool which does not have
+.Sy multihost
+set, since no claim runs for it and an ordinary import already succeeds.
+Leaves of log, cache, and spare vdevs are not touched, as the claim does not
+write to them.
+.
+.It Xo
+.Nm zhack
 .Cm metaslab leak
 .Op Fl f
 .Ar pool
@@ -181,8 +222,23 @@ descriptions_obj:
 .No # Nm zhack Cm feature enable Fl d No 'Predict future disk failures.' Ar tank com.example:clairvoyance
 .No # Nm zhack Cm feature ref Ar tank com.example:clairvoyance
 .Ed
+.Pp
+Recover a pool whose peer host died together with its mirror legs, after
+confirming that host is down:
+.Bd -literal
+.No # Nm zpool Cm import Fl f Ar tank
+cannot import 'tank': pool is imported on host '<unknown>' (hostid=0).
+Export the pool on the other system, then run 'zpool import'.
+.No # Nm zhack Cm mmp reclaim Ar tank
+tank: marked absent leaf 5646371977210937898 offline
+.No # Nm zpool Cm import Ar tank
+.No # Nm zpool Cm online Ar tank Ar /dev/disk/by-id/...
+.Ed
 .
 .Sh SEE ALSO
 .Xr ztest 1 ,
 .Xr zpool-features 7 ,
-.Xr zfs 8
+.Xr zpoolprops 7 ,
+.Xr zfs 8 ,
+.Xr zpool-import 8 ,
+.Xr zpool-online 8

module/zfs/mmp.c

@@ -196,6 +196,22 @@ uint_t zfs_multihost_import_intervals = MMP_DEFAULT_IMPORT_INTERVALS;
  */
 uint_t zfs_multihost_fail_intervals = MMP_DEFAULT_FAIL_INTERVALS;
 
+#ifndef _KERNEL
+/*
+ * Manual recovery only, set by zhack.  Drops the mirror legs this host
+ * cannot open from the uberblock claim's required write count, so a pool
+ * whose config still expects legs that died with a peer host can be
+ * imported by hand.  Every leg which can be opened is still required.
+ *
+ * This is deliberately absent from the kernel module.  The write, the wait
+ * and the re-read are untouched, so a competing importer which shares any
+ * visibility with us is still caught; a live peer whose legs are all
+ * invisible from here is not, and cannot be by any write-and-read scheme.
+ * That residual is why this is a manual operation guarded by fencing.
+ */
+boolean_t mmp_claim_relaxed = B_FALSE;
+#endif
+
 static const void *const mmp_tag = "mmp_write_uberblock";
 static __attribute__((noreturn)) void mmp_thread(void *arg);
 
@@ -600,6 +616,19 @@ mmp_claim_uberblock_sync(zio_t *zio, uint64_t *good_writes,
 				for (uint64_t l = 0; l < cvd->vdev_children;
 				    l++) {
 					vdev_t *lvd = cvd->vdev_child[l];
+#ifndef _KERNEL
+					/*
+					 * Manual recovery forgives the legs
+					 * this host could not open, and marks
+					 * that same set offline before it
+					 * exports, so the relaxed claim
+					 * accepts nothing the later ordinary
+					 * imports would not.
+					 */
+					if (mmp_claim_relaxed &&
+					    lvd->vdev_not_present)
+						continue;
+#endif
 					if (!lvd->vdev_offline &&
 					    !lvd->vdev_faulted &&
 					    !lvd->vdev_removed)
@@ -669,7 +698,8 @@ mmp_claim_uberblock(spa_t *spa, vdev_t *vd, uberblock_t *ub)
 	/*
 	 * To guarantee visibility from a remote host we require a minimum
 	 * number of good writes. For raidz/draid vdevs parity+1 writes, for
-	 * mirrors 2 writes, and for singletons 1 write.
+	 * mirrors one write per leg the config expects present, and for
+	 * singletons 1 write.
 	 */
 	if (req_writes == 0 || good_writes < req_writes)
 		return (SET_ERROR(EIO));