test(dst): FoundationDB-style buggify — 9 legal-but-pessimal points at real library sites

dist/Makefile.in

@@ -1667,6 +1667,13 @@ test_sim_swarm: test_sim_swarm@o@ $(DEF_LIB)
 	    $(LDFLAGS) test_sim_swarm@o@ $(DEF_LIB) $(TEST_LIBS) $(LIBS)
 	$(POSTLINK) $@
 
+test_sim_buggify@o@: $(testdir)/sim/test_sim_buggify.c
+	$(CC) $(DST_CFLAGS) $(DEPFLAGS) $<
+test_sim_buggify: test_sim_buggify@o@ $(DEF_LIB)
+	$(CCLINK) -o $@ \
+	    $(LDFLAGS) test_sim_buggify@o@ $(DEF_LIB) $(TEST_LIBS) $(LIBS)
+	$(POSTLINK) $@
+
 test_sim_ckp_lsn@o@: $(testdir)/sim/test_sim_ckp_lsn.c
 	$(CC) $(DST_CFLAGS) $(DEPFLAGS) $<
 test_sim_ckp_lsn: test_sim_ckp_lsn@o@ $(DEF_LIB)
@@ -1785,7 +1792,8 @@ dst_tests: test_sim_rng test_sim_crash_recover test_sim_torn \
 	test_sim_stale_meta test_sim_compound_fault test_sim_logrollover_crash \
 	mp_failchk_pilot \
 	test_sim_crash_in_recovery test_sim_recovery_undo_crash \
-	test_sim_recovery_redo_crash test_sim_recovery_ckp_crash
+	test_sim_recovery_redo_crash test_sim_recovery_ckp_crash \
+	test_sim_buggify
 
 ##################################################
 # Malloc-failure injection (SQLite-style) -- test/faultinject/.

src/btree/bt_put.c

@@ -49,6 +49,10 @@
 #include "dbinc/lock.h"
 #include "dbinc/mp.h"
 
+#ifdef HAVE_DST
+#include "sim_buggify.h"		/* DST buggify points (--enable-dst only). */
+#endif
+
 static int __bam_build
 	       __P((DBC *, u_int32_t, DBT *, PAGE *, u_int32_t, u_int32_t));
 static int __bam_dup_check __P((DBC *, u_int32_t,
@@ -277,6 +281,31 @@ __bam_iitem(dbc, key, data, op, flags)
 	if (P_FREESPACE(dbp, h) < needed)
 		return (DB_NEEDSPLIT);
 
+#ifdef HAVE_DST
+	/*
+	 * BUGGIFY bt.split_early: force an EARLY split even though the item
+	 * fits, to stress the split/merge machinery (which normally runs
+	 * only when a page fills).  Legal-but-pessimal: the caller catches
+	 * DB_NEEDSPLIT, splits, and retries the put -- the retry has more
+	 * room and succeeds, so the result is identical, only slower and
+	 * with a taller/wider tree.
+	 *
+	 * Two guards keep it legal AND terminating.  (a) leaf pages only
+	 * (P_LBTREE / P_LDUP / P_LRECNO) with NUM_ENT(h) >= 4, so the
+	 * split-point picker has a couple of entries per side.  (b) fire
+	 * ONLY when the page is already more than THREE-QUARTERS full
+	 * (P_FREESPACE < pgsize/4): the buggify coin is cached per run, so
+	 * it re-fires on the post-split RETRY too -- after splitting a
+	 * >3/4-full page each half is well under half full, so the retry no
+	 * longer qualifies and the put completes (no split loop).
+	 */
+	if ((TYPE(h) == P_LBTREE || TYPE(h) == P_LDUP ||
+	    TYPE(h) == P_LRECNO) && NUM_ENT(h) >= 4 &&
+	    P_FREESPACE(dbp, h) < dbp->pgsize / 4 &&
+	    DB_BUGGIFY(BUGGIFY_BT_SPLIT_EARLY))
+		return (DB_NEEDSPLIT);
+#endif
+
 	/*
 	 * Check to see if we will convert to off page duplicates -- if
 	 * so, we'll need a page.

src/hash/hash_page.c

@@ -57,6 +57,10 @@
 #include "dbinc/lock.h"
 #include "dbinc/mp.h"
 
+#ifdef HAVE_DST
+#include "sim_buggify.h"		/* DST buggify points (--enable-dst only). */
+#endif
+
 static int __hamc_delpg
     __P((DBC *, db_pgno_t, db_pgno_t, u_int32_t, db_ham_mode, u_int32_t *));
 static int __ham_getindex_sorted
@@ -2667,6 +2671,19 @@ __ham_add_el(dbc, key, val, type)
 	if (do_expand || (hcp->hdr->ffactor != 0 &&
 	    (u_int32_t)H_NUMPAIRS(hcp->page) > hcp->hdr->ffactor))
 		F_SET(hcp, H_EXPAND);
+#ifdef HAVE_DST
+	/*
+	 * BUGGIFY hash.expand_early: expand the hash table (split a bucket)
+	 * sooner than the fill factor requires, to stress the bucket-split
+	 * path (which normally fires only when a bucket exceeds ffactor).
+	 * Legal-but-pessimal: __ham_expand_table is safe to run at any time
+	 * (it just adds a bucket and rehashes; it even ignores ENOSPC for a
+	 * non-txn cursor), so forcing it early changes only the table shape
+	 * and timing, never a lookup result.
+	 */
+	if (DB_BUGGIFY(BUGGIFY_HASH_EXPAND_EARLY))
+		F_SET(hcp, H_EXPAND);
+#endif
 	return (0);
 }
 

src/lock/lock.c

@@ -13,6 +13,10 @@
 #include "dbinc/log.h"
 #include "dbinc/txn.h"
 
+#ifdef HAVE_DST
+#include "sim_buggify.h"		/* DST buggify points (--enable-dst only). */
+#endif
+
 static int __lock_allocobj __P((DB_LOCKTAB *, u_int32_t));
 static int __lock_alloclock __P((DB_LOCKTAB *, u_int32_t));
 static int  __lock_freelock __P((DB_LOCKTAB *,
@@ -574,6 +578,21 @@ __lock_vec(env, sh_locker, flags, list, nlist, elistp)
 	if (ret == 0 && region->detect != DB_LOCK_NORUN &&
 	     (region->need_dd || timespecisset(&region->next_timeout)))
 		run_dd = 1;
+#ifdef HAVE_DST
+	/*
+	 * BUGGIFY lock.dd_now: run the deadlock detector on this lock-vec
+	 * operation even though nothing flagged a possible deadlock
+	 * (need_dd clear, no timeout).  Legal-but-pessimal: the detector is
+	 * a read-only graph walk that aborts a txn ONLY on a real cycle, so
+	 * running it more often can never manufacture a false deadlock or
+	 * change a result -- it just exercises the detector (and the abort/
+	 * retry path when it does find a genuine cycle) far more than a lazy
+	 * need_dd policy would.  Only when detection is enabled at all.
+	 */
+	if (ret == 0 && region->detect != DB_LOCK_NORUN &&
+	    DB_BUGGIFY(BUGGIFY_LOCK_DD_NOW))
+		run_dd = 1;
+#endif
 	LOCK_SYSTEM_UNLOCK(lt, region);
 
 	if (run_dd)
@@ -1324,6 +1343,21 @@ in_abort:	newl->status = DB_LSTAT_WAITING;
 		 */
 		if (region->detect != DB_LOCK_NORUN && !no_dd)
 			(void)__lock_detect(env, region->detect, &did_abort);
+#ifdef HAVE_DST
+		/*
+		 * BUGGIFY lock.dd_wait_now: run the deadlock detector before
+		 * this thread blocks even in the no_dd case (the locker holds
+		 * no locks yet, so it is in no cycle).  Legal-but-pessimal: the
+		 * detector is a read-only cycle finder that aborts only on a
+		 * genuine deadlock, so running it here finds nothing to abort
+		 * for this locker and cannot change a result -- it just drives
+		 * the detector on the blocking path far more often, surfacing
+		 * detector-vs-block races.  Only when detection is enabled.
+		 */
+		else if (region->detect != DB_LOCK_NORUN &&
+		    DB_BUGGIFY(BUGGIFY_LOCK_DD_WAIT_NOW))
+			(void)__lock_detect(env, region->detect, &did_abort);
+#endif
 
 		ip = NULL;
 		if (env->thr_hashtab != NULL &&

src/log/log_put.c

@@ -18,6 +18,7 @@
 
 #ifdef HAVE_DST
 #include "sim_inject.h"			/* DST planted-bug harness. */
+#include "sim_buggify.h"			/* DST buggify points. */
 #endif
 
 static int __log_encrypt_record __P((ENV *, DBT *, HDR *, u_int32_t));
@@ -308,6 +309,18 @@ __log_put(env, lsnp, udbt, flags)
 	 * If a flush is not needed, see if WRITE_NOSYNC was set and we
 	 * need to write out the log buffer.
 	 */
+#ifdef HAVE_DST
+	/*
+	 * BUGGIFY log.flush_now: force a synchronous log flush on a put
+	 * that would otherwise buffer.  Legal-but-pessimal: an EXTRA fsync
+	 * only makes more of the log durable sooner -- it can never lose or
+	 * reorder a record -- so correctness (and crash recovery) is
+	 * unchanged; it just exercises the flush path far more often than a
+	 * lazy WRNOSYNC workload would.
+	 */
+	if (DB_BUGGIFY(BUGGIFY_LOG_FLUSH_NOW))
+		LF_SET(DB_FLUSH);
+#endif
 	if (LF_ISSET(DB_FLUSH | DB_LOG_WRNOSYNC)) {
 		if (!lock_held) {
 			LOG_SYSTEM_LOCK(env);
@@ -470,6 +483,26 @@ __log_put_next(env, lsn, dbt, hdr, old_lsnp)
 		adv_file = 1;
 	}
 
+#ifdef HAVE_DST
+	/*
+	 * BUGGIFY log.newfile_early: roll over to a fresh log file before
+	 * the current one is actually full, to stress log-file switching
+	 * and the archive/rollover path (which a steady workload hits only
+	 * every few MB).  Legal-but-pessimal: a rollover is exactly what
+	 * the engine does when a file fills; forcing it early just yields
+	 * more, smaller files -- recovery and archiving handle any number
+	 * of files, so the result is unchanged.  Gated on the file already
+	 * being at least half full (offset*2 > log_size) so files stay a
+	 * bounded size (no one-record-per-file explosion that would swamp
+	 * the write-back crash model's fixed file table).
+	 */
+	if (!adv_file && lp->lsn.offset != 0 &&
+	    (u_int32_t)lp->lsn.offset * 2 > lp->log_size &&
+	    hdr->size + sizeof(LOGP) + dbt->size <= lp->log_nsize &&
+	    DB_BUGGIFY(BUGGIFY_LOG_NEWFILE_EARLY))
+		adv_file = 1;
+#endif
+
 	/*
 	 * If this information won't fit in the file, or if we're a
 	 * replication client environment and have been told to do so,

src/mp/mp_alloc.c

@@ -12,6 +12,10 @@
 #include "dbinc/mp.h"
 #include "dbinc/txn.h"
 
+#ifdef HAVE_DST
+#include "sim_buggify.h"		/* DST buggify points (--enable-dst only). */
+#endif
+
 /*
  * This configuration parameter limits the number of hash buckets which
  * __memp_alloc() searches through while excluding buffers with a 'high'
@@ -64,6 +68,23 @@ __memp_alloc(dbmp, infop, mfp, len, offsetp, retp)
 	buckets = buffers = put_counter = total_buckets = versions = 0;
 	aggressive = alloc_freeze = giveup = h_locked = 0;
 
+#ifdef HAVE_DST
+	/*
+	 * BUGGIFY mp.alloc_aggressive: start the buffer-pool scan in
+	 * AGGRESSIVE mode instead of ramping into it only after a full
+	 * sweep frees nothing.  Aggressive means "consider buffers of any
+	 * warmth, one per bucket" -- a strict SUPERSET of the normal
+	 * second-chance scan, so it still only ever evicts a legally
+	 * evictable buffer; it just picks victims more eagerly, stressing
+	 * the write-back / re-read path that a lazy cache rarely exercises.
+	 * aggressive==1 is the mildest level (its switch arm is a plain
+	 * break), so no sync/yield is forced here -- purely a scan-policy
+	 * change, never a correctness change.
+	 */
+	if (DB_BUGGIFY(BUGGIFY_MP_ALLOC_AGGRESSIVE))
+		aggressive = 1;
+#endif
+
 	/*
 	 * If we're allocating a buffer, and the one we're discarding is the
 	 * same size, we don't want to waste the time to re-integrate it into

src/mp/mp_fput.c

@@ -12,6 +12,10 @@
 #include "dbinc/log.h"
 #include "dbinc/mp.h"
 
+#ifdef HAVE_DST
+#include "sim_buggify.h"		/* DST buggify points (--enable-dst only). */
+#endif
+
 
 /*
  * __memp_fput_pp --
@@ -215,6 +219,19 @@ __memp_fput(dbmfp, ip, pgaddr, priority)
 	if (priority == DB_PRIORITY_VERY_LOW ||
 	    mfp->priority == MPOOL_PRI_VERY_LOW)
 		bhp->priority = MPOOL_CLOCK_VERY_LOW;
+#ifdef HAVE_DST
+	/*
+	 * BUGGIFY mp.evict_cold: pin this buffer at the COLDEST warmth
+	 * regardless of its access hint, so the CLOCK hand evicts it first.
+	 * Legal-but-pessimal: warmth is a pure eviction-ORDER hint (it never
+	 * affects what a page contains or whether it is written back before
+	 * reuse -- that is the dirty/latch machinery), so forcing a buffer
+	 * cold only makes the cache churn harder and re-read more often,
+	 * exercising the read-back path.  Never a correctness change.
+	 */
+	else if (DB_BUGGIFY(BUGGIFY_MP_EVICT_COLD))
+		bhp->priority = MPOOL_CLOCK_VERY_LOW;
+#endif
 	else if (priority == DB_PRIORITY_HIGH ||
 	    priority == DB_PRIORITY_VERY_HIGH) {
 		/* Explicit high-priority hint pins the buffer at the ceiling. */

src/txn/txn_chkpt.c

@@ -47,6 +47,7 @@
 
 #ifdef HAVE_DST
 #include "sim_inject.h"			/* DST planted-bug harness. */
+#include "sim_buggify.h"			/* DST buggify points. */
 #endif
 
 /*
@@ -173,6 +174,22 @@ __txn_checkpoint(env, kbytes, minutes, flags)
 		if (bytes == 0 && mbytes == 0)
 			goto err;
 
+#ifdef HAVE_DST
+		/*
+		 * BUGGIFY txn.chkpt_force: take the checkpoint NOW even though
+		 * the kbytes/minutes threshold has not been reached (there IS
+		 * log activity -- the quiescent guard above already bailed if
+		 * not).  Legal-but-pessimal: a more frequent checkpoint is
+		 * always correct (it only advances the recovery start point and
+		 * flushes the cache sooner); it just makes the checkpoint /
+		 * cache-flush / ckp-LSN machinery run far more often than a
+		 * time-or-bytes policy would, stressing checkpoint-vs-workload
+		 * interleavings.
+		 */
+		if (DB_BUGGIFY(BUGGIFY_TXN_CHKPT_FORCE))
+			goto do_ckp;
+#endif
+
 		/*
 		 * If either kbytes or minutes is non-zero, then only take the
 		 * checkpoint if more than "minutes" minutes have passed or if

test/sim/DESIGN.md

@@ -103,11 +103,45 @@ lets a harness assert 0 to *prove* a run was fully deterministic.
 
 ### 1.3 Buggify (per-run cached coin)
 
-`DB_SIM_BUGGIFY("name")`: a named point in real library code that, under sim,
-takes a legal-but-pessimal path. Unlike a per-call fault, a buggify point is a
-coin flipped **once per run per site**, cached, so all reaches of a name agree
-and the run replays. Drawn from the dedicated `BUGGIFY` stream so enabling it
-never perturbs the IO/FAULT streams. Compiles to constant 0 when DST is off.
+`DB_BUGGIFY(name)`: a named point in real library code that, under sim, takes a
+legal-but-pessimal path. Unlike a per-call fault, a buggify point is a coin
+flipped **once per run per site**, cached, so all reaches of a name agree and
+the run replays. Drawn from the dedicated `BUGGIFY` stream so enabling it never
+perturbs the IO/FAULT streams. Compiles to constant 0 when DST is off (verified:
+`nm` shows 0 sim symbols and no point name appears in any engine `.o`).
+
+The invariant that makes buggify safe: **every buggified path is legal**
+(correctness-preserving) — it only changes timing/sizing/path-choice, never a
+result. So the whole scenario suite must still pass with buggify forced on. If
+turning a point on ever breaks an invariant, either the point isn't actually
+legal (a bug in the point) or the engine mishandles a rare-but-legal path (a
+real engine bug) — that is buggify's purpose.
+
+**Point catalog** (9 points, all `#ifdef HAVE_DST`, each a legal-but-pessimal choice):
+
+| Point | Site | Pessimal choice |
+|---|---|---|
+| `bt.split_early` | `bt_put.c` | force `DB_NEEDSPLIT` when the page is >3/4 full (guarded ≥4 entries + <pgsize/4 free, so no split loop) |
+| `hash.expand_early` | `hash_page.c` | force `H_EXPAND` before the fill factor is reached |
+| `mp.alloc_aggressive` | `mp_alloc.c` | start the eviction scan in aggressive mode |
+| `mp.evict_cold` | `mp_fput.c` | pin the buffer at the coldest warmth |
+| `log.flush_now` | `log_put.c` | force `DB_FLUSH` on a would-be-buffered log put |
+| `log.newfile_early` | `log_put.c` | roll to a new log file at >half full |
+| `txn.chkpt_force` | `txn_chkpt.c` | checkpoint past the byte/time threshold |
+| `lock.dd_now` | `lock.c` | run the deadlock detector on a lock-vector op |
+| `lock.dd_wait_now` | `lock.c` | run the detector before blocking |
+
+**Measured activation** (`test_sim_buggify`, 24-seed sweep, all pessimal paths
+forced): bt.split_early 79%, hash.expand_early 67%, mp.alloc_aggressive 100%,
+mp.evict_cold 83%, log.flush_now 75%, log.newfile_early 75%, txn.chkpt_force 88%,
+lock.dd_now 83%, **lock.dd_wait_now 0%** (never reached — this single-writer
+workload has no blocked waiter to drive the site; the point is valid, the
+coverage is workload-limited; a WARN, not a failure). **0 invariant violations
+across all 24 seeds with every pessimal path on** — every committed txn survived
+crash+recovery, no uncommitted survived, both DBs verified clean. **No real
+engine bug found**: an early over-aggressive split point caused a non-termination
+loop, which was an *illegal* buggify point (self-inflicted), correctly fixed by
+the >3/4-full guard — buggify's own self-check working as designed.
 
 ### 1.4 Simulated I/O faults + the write-back crash model