diff --git a/dist/Makefile.in b/dist/Makefile.in index 595f7a20a..e6ab25b75 100644 --- a/dist/Makefile.in +++ b/dist/Makefile.in @@ -1738,6 +1738,29 @@ test_sim_logrollover_crash: test_sim_logrollover_crash@o@ $(DEF_LIB) $(LIBS) $(POSTLINK) $@ +test_sim_clockskew_timeout@o@: $(testdir)/sim/test_sim_clockskew_timeout.c + $(CC) $(DST_CFLAGS) $(DEPFLAGS) $< +test_sim_clockskew_timeout: test_sim_clockskew_timeout@o@ $(DEF_LIB) + $(CCLINK) -o $@ \ + $(LDFLAGS) test_sim_clockskew_timeout@o@ $(DEF_LIB) $(TEST_LIBS) \ + $(LIBS) + $(POSTLINK) $@ + +test_sim_clockskew_ckp@o@: $(testdir)/sim/test_sim_clockskew_ckp.c + $(CC) $(DST_CFLAGS) $(DEPFLAGS) $< +test_sim_clockskew_ckp: test_sim_clockskew_ckp@o@ $(DEF_LIB) + $(CCLINK) -o $@ \ + $(LDFLAGS) test_sim_clockskew_ckp@o@ $(DEF_LIB) $(TEST_LIBS) $(LIBS) + $(POSTLINK) $@ + +test_sim_clockskew_backward@o@: $(testdir)/sim/test_sim_clockskew_backward.c + $(CC) $(DST_CFLAGS) $(DEPFLAGS) $< +test_sim_clockskew_backward: test_sim_clockskew_backward@o@ $(DEF_LIB) + $(CCLINK) -o $@ \ + $(LDFLAGS) test_sim_clockskew_backward@o@ $(DEF_LIB) $(TEST_LIBS) \ + $(LIBS) + $(POSTLINK) $@ + # DST v2 multi-process failchk-recovery pilot (test/sim/mp_failchk_pilot.c). # Runs OUTSIDE the single-process test_sim harness: it forks + kill -9's real # processes sharing a real (non-DB_PRIVATE) region. Driver: mp-failchk.sh. @@ -1793,7 +1816,9 @@ dst_tests: test_sim_rng test_sim_crash_recover test_sim_torn \ mp_failchk_pilot \ test_sim_crash_in_recovery test_sim_recovery_undo_crash \ test_sim_recovery_redo_crash test_sim_recovery_ckp_crash \ - test_sim_buggify + test_sim_buggify \ + test_sim_clockskew_timeout test_sim_clockskew_ckp \ + test_sim_clockskew_backward ################################################## # Malloc-failure injection (SQLite-style) -- test/faultinject/. diff --git a/test/sim/test_sim_clockskew_backward.c b/test/sim/test_sim_clockskew_backward.c new file mode 100644 index 000000000..43827fe58 --- /dev/null +++ b/test/sim/test_sim_clockskew_backward.c @@ -0,0 +1,270 @@ +/*- + * Deterministic Simulation Testing (DST) for libdb. + * + * test_sim_clockskew_backward.c -- + * A pure BACKWARD clock jump mid-run: the dangerous case. + * + * The clock jumps only backward (never forward) partway through a + * txn-timeout wait. This is the exact FoundationDB clock-skew concern: + * `deadline = now + timeout` is set, then the clock jumps back below + * `now`, so later reads of `now2` stay < deadline for a while. The jump + * is TRANSIENT (bounded via __db_sim_clock_settle_after): after the + * disturbance the clock resumes advancing. A correct engine must fire + * the timeout once the recovered clock crosses the fixed deadline; a + * broken engine that RE-ARMS the deadline off the jumped-back clock, or + * that trusts a monotonic assumption, would lose the timeout FOREVER + * even after the clock recovers -- and the wall-clock alarm catches it. + * + * (A PERSISTENT adversary clock that never advances past the deadline + * could never fire ANY deadline-based timeout -- that is physics, not a + * libdb bug -- so this scenario uses a bounded, recovering jump.) + * + * Uses a TXN timeout (DB_SET_TXN_TIMEOUT) via a real DB_TXN + a lock + * contended between the txn and a helper-held lock, so the txn's wait is + * bounded by tx_expire (set from the skewed clock). Also asserts + * determinism: two runs of the same seed apply the identical number of + * skews (same skew sequence) and reach the identical verdict. + * + * Guarded by a hard wall-clock alarm: a LOST timeout hangs the waiter, + * and the alarm reports it as a REAL BUG with the seed, rather than + * wedging CI. + * + * Build/run (from build_unix, after configure --enable-dst): + * make test_sim_clockskew_backward && ./test_sim_clockskew_backward [seed] + */ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "db.h" +#include "sim_rng.h" +#include "sim_fault.h" +#include "sim_clock.h" + +#define HOME "TESTDIR_sim_clockskew_backward" +#define OBJNAME "contended" +#define TXN_TMOUT 60000 /* 60 ms txn timeout (microseconds) */ +#define WALL_LIMIT 20 + +static DB_ENV *g_env; +static u_int32_t g_holder_id; +static DB_TXN *g_waiter_txn; +static volatile int g_helper_ret = -12345; +static volatile int g_helper_done; +static uint64_t g_seed; + +static void +on_alarm(sig) + int sig; +{ + (void)sig; + fprintf(stderr, "test_sim_clockskew_backward: FAIL -- HUNG: a txn " + "timeout was LOST after a BACKWARD clock jump (never woke within " + "%ds); REAL BUG: backward jump defeats the deadline. Reproduce: " + "./test_sim_clockskew_backward 0x%llx\n", + WALL_LIMIT, (unsigned long long)g_seed); + _exit(3); +} + +static void * +helper(arg) + void *arg; +{ + DB_LOCKREQ req; + DBT obj; + u_int32_t wid; + int ret; + + (void)arg; + /* The waiter txn's locker id drives the txn timeout. */ + wid = g_waiter_txn->id(g_waiter_txn); + memset(&obj, 0, sizeof(obj)); + obj.data = (void *)OBJNAME; + obj.size = (u_int32_t)strlen(OBJNAME) + 1; + memset(&req, 0, sizeof(req)); + req.op = DB_LOCK_GET; + req.mode = DB_LOCK_WRITE; + req.obj = &obj; + + /* Blocks until the txn's tx_expire deadline expires it. */ + ret = g_env->lock_vec(g_env, wid, 0, &req, 1, NULL); + if (ret == 0) + (void)g_env->lock_put(g_env, &req.lock); + g_helper_ret = ret; + g_helper_done = 1; + return (NULL); +} + +/* One full run at a given seed; returns 0 on PASS, sets skews + verdict. */ +static int +run_once(seed, skews, verdict) + uint64_t seed; + unsigned long *skews; + int *verdict; +{ + DB_LOCK holder_lock; + DBT obj; + pthread_t tid; + struct timespec nap; + int ret, i, rejected; + + g_helper_ret = -12345; + g_helper_done = 0; + + if ((ret = db_env_create(&g_env, 0)) != 0) + return (ret); + if ((ret = g_env->open(g_env, HOME, DB_CREATE | DB_INIT_LOCK | + DB_INIT_MPOOL | DB_INIT_TXN, 0664)) != 0) + return (ret); + (void)g_env->set_lk_detect(g_env, DB_LOCK_EXPIRE); + + if ((ret = g_env->lock_id(g_env, &g_holder_id)) != 0) + return (ret); + /* Waiter runs inside a real txn with a txn timeout. */ + if ((ret = g_env->txn_begin(g_env, NULL, &g_waiter_txn, 0)) != 0) + return (ret); + + /* Main thread holds the write lock on the contended object. */ + memset(&obj, 0, sizeof(obj)); + obj.data = (void *)OBJNAME; + obj.size = (u_int32_t)strlen(OBJNAME) + 1; + memset(&holder_lock, 0, sizeof(holder_lock)); + if ((ret = g_env->lock_get(g_env, g_holder_id, 0, &obj, + DB_LOCK_WRITE, &holder_lock)) != 0) + return (ret); + + __db_sim_activate(seed); + /* + * Set the txn's deadline FIRST, on the TRUE clock: tx_expire = + * true_now + 60ms (a legitimate, correctly-computed deadline). + */ + if ((ret = g_waiter_txn->set_timeout(g_waiter_txn, TXN_TMOUT, + DB_SET_TXN_TIMEOUT)) != 0) + return (ret); + + /* + * NOW knock the clock backward (the TRANSIENT FoundationDB model): a + * big negative offset plus large backward-leaning jumps, for a BOUNDED + * number of reads, after which the clock settles back to true time. + * The deadline is a fixed, already-set target; the backward jump pushes + * every fresh `now2` reading below it for a while. A correct engine + * must fire the timeout once the recovered clock crosses the deadline; + * a broken engine that trusts a monotonic assumption, or re-arms the + * deadline off the jumped-back clock, would lose the timeout FOREVER -- + * and the wall-clock alarm catches that as a REAL BUG. + * + * (A PERSISTENT adversary clock that never advances past the deadline + * could never fire ANY deadline-based timeout -- physics, not a bug -- + * so the jump is bounded and recovering.) + */ + __db_sim_clock_enable( + /* offset */ -3LL * 1000 * 1000 * 1000, /* -3s steady */ + /* jitter */ 20LL * 1000 * 1000, /* +/-20ms */ + /* jump */ 10LL * 1000 * 1000 * 1000, /* up to 10s jumps */ + /* jump% */ 700); /* 70% of reads jump */ + /* Disturbance lasts ~60 clock reads, then the clock returns to true + * and the fixed deadline MUST be crossed. */ + __db_sim_clock_settle_after(60); + + /* The txn must own the tx_expire deadline: touch a lock under it so + * the locker exists, then start the waiter which contends. */ + if ((ret = pthread_create(&tid, NULL, helper, NULL)) != 0) + return (ret); + + nap.tv_sec = 0; + nap.tv_nsec = 5 * 1000 * 1000; + for (i = 0; i < 4000 && !g_helper_done; i++) { + rejected = 0; + (void)g_env->lock_detect(g_env, 0, DB_LOCK_EXPIRE, &rejected); + (void)nanosleep(&nap, NULL); + } + + (void)pthread_join(tid, NULL); + + *skews = __db_sim_clock_fire_count(); + *verdict = g_helper_ret; + __db_sim_deactivate(); + + (void)g_env->lock_put(g_env, &holder_lock); + (void)g_waiter_txn->abort(g_waiter_txn); + (void)g_env->close(g_env, 0); + + if (!g_helper_done) + return (-100); /* never resolved */ + return (0); +} + +int +main(argc, argv) + int argc; + char *argv[]; +{ + char cmd[512]; + unsigned long skews1, skews2; + int ret, verdict1, verdict2; + + g_seed = argc > 1 ? strtoull(argv[1], NULL, 0) : 0xC10C0003; + + (void)signal(SIGALRM, on_alarm); + (void)alarm(WALL_LIMIT); + + (void)snprintf(cmd, sizeof(cmd), "rm -rf %s && mkdir -p %s", + HOME, HOME); + if (system(cmd) != 0) + return (EXIT_FAILURE); + + if ((ret = run_once(g_seed, &skews1, &verdict1)) != 0) { + if (ret == -100) + fprintf(stderr, "test_sim_clockskew_backward: FAIL -- " + "waiter never resolved (lost timeout?) (seed " + "0x%llx)\n", (unsigned long long)g_seed); + else + fprintf(stderr, "test_sim_clockskew_backward: run " + "failed: %s\n", db_strerror(ret)); + return (EXIT_FAILURE); + } + + /* Timeout must have fired cleanly, never granted. */ + if (verdict1 != DB_LOCK_NOTGRANTED && verdict1 != DB_LOCK_DEADLOCK) { + fprintf(stderr, "test_sim_clockskew_backward: FAIL -- " + "unexpected verdict %d (%s) (seed 0x%llx)\n", + verdict1, db_strerror(verdict1), + (unsigned long long)g_seed); + return (EXIT_FAILURE); + } + + /* Determinism: rerun the same seed -> identical skew count + verdict. */ + (void)snprintf(cmd, sizeof(cmd), "rm -rf %s && mkdir -p %s", + HOME, HOME); + (void)system(cmd); + if (run_once(g_seed, &skews2, &verdict2) != 0) { + fprintf(stderr, "test_sim_clockskew_backward: FAIL -- replay " + "run did not resolve (seed 0x%llx)\n", + (unsigned long long)g_seed); + return (EXIT_FAILURE); + } + if (skews1 != skews2 || verdict1 != verdict2) { + fprintf(stderr, "test_sim_clockskew_backward: FAIL -- NOT " + "DETERMINISTIC: skews %lu vs %lu, verdict %d vs %d " + "(seed 0x%llx)\n", skews1, skews2, verdict1, verdict2, + (unsigned long long)g_seed); + return (EXIT_FAILURE); + } + + (void)alarm(0); + printf("test_sim_clockskew_backward: PASS -- txn timeout fired " + "(ret=%s) despite a backward-trending clock; no timeout lost, " + "deterministic (%lu skews, replay identical) (seed 0x%llx)\n", + verdict1 == DB_LOCK_NOTGRANTED ? "NOTGRANTED" : "DEADLOCK", + skews1, (unsigned long long)g_seed); + return (EXIT_SUCCESS); +} diff --git a/test/sim/test_sim_clockskew_ckp.c b/test/sim/test_sim_clockskew_ckp.c new file mode 100644 index 000000000..7068bb880 --- /dev/null +++ b/test/sim/test_sim_clockskew_ckp.c @@ -0,0 +1,219 @@ +/*- + * Deterministic Simulation Testing (DST) for libdb. + * + * test_sim_clockskew_ckp.c -- + * Checkpoint + recovery under a large FORWARD clock jump. + * + * Arms the clock-skew fault with an aggressive forward jump (the clock + * leaps ahead by up to an hour, repeatedly) while a transactional + * workload runs and checkpoints are taken. A forward jump is what a + * naive scheduler treats as "lots of time has passed" -- it must not + * make checkpoint bookkeeping (the checkpoint LSN, the durable frontier) + * inconsistent. After a mid-workload crash the environment recovers + * clean and every fsync-acked commit is present. + * + * Invariant: a forward clock jump does not corrupt checkpoint/recovery + * state; checkpoints still make progress; committed data survives. + * + * Honest scope note: on a platform with libc time() (HAVE_TIME, i.e. + * Linux), the checkpoint MINUTE-interval decision in __txn_checkpoint + * reads libc time() directly, NOT __os_gettime, so it is not reachable + * by this __os_gettime skew -- which is itself a finding (that decision + * is inherently robust to __os_gettime skew). This scenario therefore + * drives checkpoints EXPLICITLY (as a real application's checkpoint + * thread does) and proves the skew perturbs nothing downstream in the + * checkpoint/recovery path. See DESIGN.md clock-skew section. + * + * Build/run (from build_unix, after configure --enable-dst): + * make test_sim_clockskew_ckp && ./test_sim_clockskew_ckp [seed] + */ + +#include "sim_scenario.h" +#include "sim_clock.h" + +#include + +#define HOME "TESTDIR_sim_clockskew_ckp" +#define DBFILE "ckpskew.db" +#define NTXN 256 +#define CKP_EVERY 32 +#define WALL_LIMIT 30 + +static unsigned char g_committed[NTXN]; +static uint64_t g_seed; + +static void +on_alarm(sig) + int sig; +{ + (void)sig; + fprintf(stderr, "test_sim_clockskew_ckp: FAIL -- HUNG: checkpoint " + "stopped making progress under a forward clock jump (>%ds); " + "REAL BUG. Reproduce: ./test_sim_clockskew_ckp 0x%llx\n", + WALL_LIMIT, (unsigned long long)g_seed); + _exit(3); +} + +static int +open_db(env, dbp, create) + DB_ENV *env; + DB **dbp; + int create; +{ + DB *db; + int ret; + + if ((ret = db_create(&db, env, 0)) != 0) + return (ret); + if ((ret = db->open(db, NULL, DBFILE, NULL, DB_BTREE, + (create ? DB_CREATE : 0) | DB_AUTO_COMMIT, 0664)) != 0) + return (ret); + *dbp = db; + return (0); +} + +static int +populate(seed) + uint64_t seed; +{ + DB_ENV *env; + DB *db; + DB_TXN *txn; + DBT key, data; + char kbuf[32], vbuf[48]; + int i, ret; + FILE *fp; + uint64_t tok; + + __db_sim_activate(seed); + __db_sim_wb_enable(1); + + if ((ret = db_env_create(&env, 0)) != 0) + return (ret); + if ((ret = env->open(env, HOME, DB_CREATE | DB_INIT_LOCK | + DB_INIT_LOG | DB_INIT_MPOOL | DB_INIT_TXN, 0664)) != 0) + return (ret); + if ((ret = open_db(env, &db, 1)) != 0) + return (ret); + + memset(g_committed, 0, sizeof(g_committed)); + + /* + * Arm a large forward jump: the clock leaps ahead by up to an hour on + * 60% of reads. __os_gettime readers (lock/txn deadlines, rep timers, + * and the checkpoint LOG-region timestamps) all see the jumped clock. + */ + __db_sim_clock_enable( + /* offset */ 0, /* seeded steady skew */ + /* jitter */ 5LL * 1000 * 1000, /* +/-5ms */ + /* jump */ 3600LL * 1000 * 1000 * 1000, /* up to +1h jumps */ + /* jump% */ 600); + + for (i = 0; i < NTXN; i++) { + tok = __db_sim_rng(DB_SIM_RNG_APP); + (void)snprintf(kbuf, sizeof(kbuf), "ck-%08d", i); + (void)snprintf(vbuf, sizeof(vbuf), "cv-%016llx", + (unsigned long long)tok); + if (env->txn_begin(env, NULL, &txn, 0) != 0) + continue; + memset(&key, 0, sizeof(key)); + memset(&data, 0, sizeof(data)); + key.data = kbuf; key.size = (u_int32_t)strlen(kbuf) + 1; + data.data = vbuf; data.size = (u_int32_t)strlen(vbuf) + 1; + if (db->put(db, txn, &key, &data, 0) != 0) { + (void)txn->abort(txn); + continue; + } + if (txn->commit(txn, DB_TXN_SYNC) == 0) + g_committed[i] = 1; + /* Explicit checkpoint mid-workload, WITH the clock jumping. */ + if (i % CKP_EVERY == CKP_EVERY - 1) + (void)env->txn_checkpoint(env, 0, 0, DB_FORCE); + } + + if ((fp = fopen(HOME "/committed.map", "wb")) != NULL) { + (void)fwrite(g_committed, 1, sizeof(g_committed), fp); + (void)fclose(fp); + } + + /* One last checkpoint under the jumping clock, then crash. */ + (void)env->txn_checkpoint(env, 0, 0, DB_FORCE); + __db_sim_clock_disable(); /* the truncation must not be skewed */ + SIM_CRASH_EXIT(); + return (0); +} + +int +main(argc, argv) + int argc; + char *argv[]; +{ + DB_ENV *env; + DB *db; + DBT key, data; + char kbuf[32]; + unsigned char committed[NTXN]; + FILE *fp; + int i, ret, missing = 0, ncommitted = 0; + + g_seed = argc > 1 ? strtoull(argv[1], NULL, 0) : 0xC10C0002; + + (void)signal(SIGALRM, on_alarm); + (void)alarm(WALL_LIMIT); + + if (sim_fresh_home(HOME) != 0) + return (EXIT_FAILURE); + if (sim_run_crash_child(g_seed, populate) != 0) + return (EXIT_FAILURE); + + memset(committed, 0, sizeof(committed)); + if ((fp = fopen(HOME "/committed.map", "rb")) != NULL) { + (void)fread(committed, 1, sizeof(committed), fp); + (void)fclose(fp); + } + + if (sim_env_recover(HOME, &env) != 0) + return (EXIT_FAILURE); + if (open_db(env, &db, 0) != 0) + return (EXIT_FAILURE); + + __db_sim_activate(g_seed); + for (i = 0; i < NTXN; i++) { + if (!committed[i]) + continue; + ncommitted++; + (void)snprintf(kbuf, sizeof(kbuf), "ck-%08d", i); + memset(&key, 0, sizeof(key)); + memset(&data, 0, sizeof(data)); + key.data = kbuf; key.size = (u_int32_t)strlen(kbuf) + 1; + if (db->get(db, NULL, &key, &data, 0) != 0) + missing++; + } + __db_sim_deactivate(); + (void)db->close(db, 0); + + if ((ret = db_create(&db, env, 0)) != 0) + return (EXIT_FAILURE); + if ((ret = db->verify(db, DBFILE, NULL, NULL, 0)) != 0) { + fprintf(stderr, "test_sim_clockskew_ckp: verify FAILED: %s " + "(seed 0x%llx)\n", db_strerror(ret), + (unsigned long long)g_seed); + (void)env->close(env, 0); + return (EXIT_FAILURE); + } + (void)env->close(env, 0); + (void)alarm(0); + + if (missing != 0) { + fprintf(stderr, "test_sim_clockskew_ckp: FAIL -- %d of %d " + "committed txns lost across a forward-clock-jump checkpoint " + "(seed 0x%llx)\n", missing, ncommitted, + (unsigned long long)g_seed); + return (EXIT_FAILURE); + } + printf("test_sim_clockskew_ckp: PASS -- checkpoints made progress " + "under a forward clock jump; all %d committed txns durable, tree " + "clean after recovery (seed 0x%llx)\n", + ncommitted, (unsigned long long)g_seed); + return (EXIT_SUCCESS); +} diff --git a/test/sim/test_sim_clockskew_timeout.c b/test/sim/test_sim_clockskew_timeout.c new file mode 100644 index 000000000..b4dd09720 --- /dev/null +++ b/test/sim/test_sim_clockskew_timeout.c @@ -0,0 +1,236 @@ +/*- + * Deterministic Simulation Testing (DST) for libdb. + * + * test_sim_clockskew_timeout.c -- + * Lock timeout under a skewed / jumping / non-monotonic clock. + * + * The dangerous pattern (FoundationDB clock-skew model): a timeout is + * a deadline computed as `deadline = now + timeout` (__clock_set_expires), + * compared later against `now2 >= deadline` (__clock_expired) by the + * deadlock detector's DB_LOCK_EXPIRE scan. If the clock jumps BACKWARD + * between the two reads, now2 can be < deadline forever and the timeout + * never fires -- the waiter HANGS. + * + * Scenario: the main thread holds a write lock on an object; a helper + * thread requests a conflicting write lock with a short lock timeout and + * blocks. The main thread arms the clock-skew fault (fixed offset + + * jitter + occasional forward AND backward jumps) and then drives the + * deadlock detector's expiry scan repeatedly. Invariant: the blocked + * lock request EVENTUALLY returns DB_LOCK_NOTGRANTED (the timeout fires) + * and the run resolves -- no hang, no premature abort-storm. + * + * The whole run is guarded by a hard wall-clock alarm: a clock bug that + * loses the timeout would hang the helper, and the alarm turns that hang + * into a reported FAILURE + the exact seed to reproduce (rather than a + * CI process that wedges forever). Same seed => same skew sequence => + * deterministic outcome. + * + * Build/run (from build_unix, after configure --enable-dst): + * make test_sim_clockskew_timeout && ./test_sim_clockskew_timeout [seed] + */ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "db.h" +#include "sim_rng.h" +#include "sim_fault.h" +#include "sim_clock.h" + +#define HOME "TESTDIR_sim_clockskew_timeout" +#define OBJNAME "hot-object" +#define LK_TIMEOUT 50000 /* 50 ms lock timeout (microseconds) */ +#define WALL_LIMIT 20 /* hard wall-clock guard: fail if we hang */ + +static DB_ENV *g_env; +static u_int32_t g_holder_id, g_waiter_id; +static volatile int g_helper_ret = -12345; /* sentinel: not set yet */ +static volatile int g_helper_done; + +/* Wall-clock watchdog: if a lost timeout hangs us, SIGALRM aborts with a + * clear "hung" verdict and the seed, instead of wedging CI forever. */ +static uint64_t g_seed; +static void +on_alarm(sig) + int sig; +{ + (void)sig; + fprintf(stderr, "test_sim_clockskew_timeout: FAIL -- HUNG: a lock " + "timeout was LOST under clock skew (waiter never woke within %ds); " + "REAL BUG: non-monotonic clock defeats the deadline. Reproduce: " + "./test_sim_clockskew_timeout 0x%llx\n", + WALL_LIMIT, (unsigned long long)g_seed); + _exit(3); +} + +/* + * Helper thread: request the same object with a conflicting write lock and + * a lock timeout. Blocks until the main thread's detector expires it. + */ +static void * +helper(arg) + void *arg; +{ + DB_LOCKREQ req; + DB_LOCK lock; + DBT obj; + int ret; + + (void)arg; + memset(&obj, 0, sizeof(obj)); + obj.data = (void *)OBJNAME; + obj.size = (u_int32_t)strlen(OBJNAME) + 1; + memset(&req, 0, sizeof(req)); + req.op = DB_LOCK_GET_TIMEOUT; + req.mode = DB_LOCK_WRITE; + req.timeout = LK_TIMEOUT; + req.obj = &obj; + memset(&lock, 0, sizeof(lock)); + + /* This blocks inside __lock_get_internal until the expiry scan wakes + * it; the return code tells us how it resolved. */ + ret = g_env->lock_vec(g_env, g_waiter_id, 0, &req, 1, NULL); + if (ret == 0) /* granted (should not happen: main holds it) */ + (void)g_env->lock_put(g_env, &req.lock); + g_helper_ret = ret; + g_helper_done = 1; + return (NULL); +} + +int +main(argc, argv) + int argc; + char *argv[]; +{ + DB_LOCK holder_lock; + DBT obj; + pthread_t tid; + char cmd[512]; + struct timespec nap; + int ret, i, rejected; + + g_seed = argc > 1 ? strtoull(argv[1], NULL, 0) : 0xC10C0001; + + (void)snprintf(cmd, sizeof(cmd), "rm -rf %s && mkdir -p %s", + HOME, HOME); + if (system(cmd) != 0) + return (EXIT_FAILURE); + + /* Hard wall-clock guard against a hung (lost) timeout. */ + (void)signal(SIGALRM, on_alarm); + (void)alarm(WALL_LIMIT); + + if ((ret = db_env_create(&g_env, 0)) != 0) + goto envfail; + /* We drive the expiry scan by hand, so no background detector. */ + if ((ret = g_env->open(g_env, HOME, DB_CREATE | DB_INIT_LOCK | + DB_INIT_MPOOL, 0664)) != 0) + goto envfail; + /* Auto-run the detector's expiry scan on each lock request too. */ + (void)g_env->set_lk_detect(g_env, DB_LOCK_EXPIRE); + + if ((ret = g_env->lock_id(g_env, &g_holder_id)) != 0) + goto envfail; + if ((ret = g_env->lock_id(g_env, &g_waiter_id)) != 0) + goto envfail; + + /* Main thread grabs the write lock on the hot object. */ + memset(&obj, 0, sizeof(obj)); + obj.data = (void *)OBJNAME; + obj.size = (u_int32_t)strlen(OBJNAME) + 1; + memset(&holder_lock, 0, sizeof(holder_lock)); + if ((ret = g_env->lock_get(g_env, g_holder_id, 0, &obj, + DB_LOCK_WRITE, &holder_lock)) != 0) + goto envfail; + + /* + * Arm the clock-skew fault: a seeded fixed offset, per-read jitter, + * and frequent forward+BACKWARD jumps of up to 5s -- an aggressively + * non-monotonic clock. Drawn from the CLOCK stream, so arming it does + * not perturb anything else. This is the exact condition that could + * defeat `deadline = now + timeout; ... now2 >= deadline`. + */ + __db_sim_activate(g_seed); + __db_sim_clock_enable( + /* offset */ 250LL * 1000 * 1000, /* +250ms steady skew */ + /* jitter */ 10LL * 1000 * 1000, /* +/-10ms per read */ + /* jump */ 5LL * 1000 * 1000 * 1000, /* up to 5s jumps */ + /* jump% */ 400); /* 40% of reads jump */ + + /* Start the waiter; it blocks on the conflicting lock. */ + if ((ret = pthread_create(&tid, NULL, helper, NULL)) != 0) { + fprintf(stderr, "pthread_create: %s\n", strerror(ret)); + goto envfail; + } + + /* + * Drive the expiry scan. Each __lock_detect(DB_LOCK_EXPIRE) reads + * the (skewed) clock via __clock_expired. On a monotonic-enough + * reading the deadline passes and the waiter is expired + woken; a + * backward jump can push "now" back before the deadline, delaying it. + * We loop with a real nap between scans; the wall-clock alarm bounds + * the total. A robust engine resolves within a bounded number of + * scans DESPITE the skew (the offset+jitter+jumps average out and the + * true wall clock keeps advancing between naps, so some scan reads a + * value past the deadline). + */ + nap.tv_sec = 0; + nap.tv_nsec = 5 * 1000 * 1000; /* 5 ms between scans */ + for (i = 0; i < 4000 && !g_helper_done; i++) { + rejected = 0; + (void)g_env->lock_detect(g_env, 0, DB_LOCK_EXPIRE, &rejected); + (void)nanosleep(&nap, NULL); + } + + (void)pthread_join(tid, NULL); + (void)alarm(0); + + __db_sim_deactivate(); + + /* Release the holder lock and shut down. */ + (void)g_env->lock_put(g_env, &holder_lock); + (void)g_env->close(g_env, 0); + + /* + * Verdict. The waiter MUST have resolved (not still blocked) and the + * resolution must be a clean timeout (DB_LOCK_NOTGRANTED) or a + * deadlock verdict (DB_LOCK_DEADLOCK) -- never granted (main held it), + * never a crash. The point: under an aggressively non-monotonic clock + * the timeout still EVENTUALLY fires; no hang, no corruption. + */ + if (!g_helper_done) { + fprintf(stderr, "test_sim_clockskew_timeout: FAIL -- waiter " + "never resolved under clock skew (seed 0x%llx)\n", + (unsigned long long)g_seed); + return (EXIT_FAILURE); + } + if (g_helper_ret != DB_LOCK_NOTGRANTED && + g_helper_ret != DB_LOCK_DEADLOCK) { + fprintf(stderr, "test_sim_clockskew_timeout: FAIL -- waiter " + "resolved with unexpected ret %d (%s) (seed 0x%llx)\n", + g_helper_ret, db_strerror(g_helper_ret), + (unsigned long long)g_seed); + return (EXIT_FAILURE); + } + + printf("test_sim_clockskew_timeout: PASS -- lock timeout fired " + "(ret=%s) under a non-monotonic clock (%lu skews applied); " + "no hang, no corruption (seed 0x%llx)\n", + g_helper_ret == DB_LOCK_NOTGRANTED ? "NOTGRANTED" : "DEADLOCK", + __db_sim_clock_fire_count(), (unsigned long long)g_seed); + return (EXIT_SUCCESS); + +envfail: + fprintf(stderr, "test_sim_clockskew_timeout: setup failed: %s\n", + db_strerror(ret)); + return (EXIT_FAILURE); +} diff --git a/test/sim/test_sim_swarm.c b/test/sim/test_sim_swarm.c index 76e64ebeb..6cf25f56f 100644 --- a/test/sim/test_sim_swarm.c +++ b/test/sim/test_sim_swarm.c @@ -325,13 +325,22 @@ main(argc, argv) */ if (n_seeds >= 64) { int gap = 0; - for (cls = 0; cls < DB_SIM_FC_NCLASSES; cls++) + for (cls = 0; cls < DB_SIM_FC_NCLASSES; cls++) { + /* + * The clock-skew class is exercised by the dedicated + * test_sim_clockskew_* scenarios, not this I/O swarm + * (which arms only the disk-fault knobs), so it is not + * expected to fire here -- skip it in the gap guard. + */ + if (cls == DB_SIM_FC_CLOCK) + continue; if (act[cls] == 0) { printf("FAIL: fault class '%s' NEVER activated " "across %ld seeds -- a coverage gap\n", __db_sim_fault_class_name(cls), n_seeds); gap = 1; } + } if (gap) return (EXIT_FAILURE); }