From 8f015082f6ec3b43d9e6e508a9cf292fdbb9b12d Mon Sep 17 00:00:00 2001 From: Ishan Bhatt Date: Thu, 29 Oct 2026 20:50:56 +0000 Subject: [PATCH 1/2] txncache: spill to disk --- book/api/metrics-generated.md | 21 + contrib/ci/bench/bench.toml | 1 + src/app/firedancer-dev/commands/backtest.c | 3 +- .../firedancer-dev/commands/snapshot_load.c | 3 +- src/app/firedancer/callbacks.c | 4 +- .../firedancer/config/bench-zen3-32core.toml | 3 + .../firedancer/config/bench-zen4-128core.toml | 3 + src/app/firedancer/config/default.toml | 21 + src/app/firedancer/topology.c | 15 +- src/app/firedancer/topology.h | 3 +- src/app/shared/commands/run/run.c | 22 + src/app/shared/commands/run/run.h | 3 + src/app/shared/fd_config.c | 11 + src/app/shared/fd_config.h | 2 + src/app/shared/fd_config_json.c | 5 +- src/app/shared/fd_config_parse.c | 2 + src/app/shared/test_config_json.c | 26 + src/app/shared/test_config_parse.c | 1 + src/app/shared_dev/commands/bench/bench.c | 1 + src/app/shared_dev/commands/dev.c | 1 + src/disco/events/schema/events.proto | 194 +++++ src/disco/events/schema/metrics_execle.json | 51 +- src/disco/events/schema/metrics_execrp.json | 51 +- src/disco/events/schema/metrics_replay.json | 213 +++--- src/disco/metrics/generated/fd_metrics_all.c | 21 + .../metrics/generated/fd_metrics_execle.h | 36 +- .../metrics/generated/fd_metrics_execrp.h | 36 +- .../metrics/generated/fd_metrics_replay.h | 68 +- .../metrics/generated/fd_metrics_sample_gen.c | 105 ++- src/disco/metrics/metrics.xml | 29 + src/discof/backup/fd_snapmk_tile.c | 12 +- .../backup/fd_snapmk_tile.seccomppolicy | 13 +- src/discof/backup/fd_txncache_writer.c | 232 +----- src/discof/backup/fd_txncache_writer.h | 34 +- .../backup/generated/fd_snapmk_tile_seccomp.h | 40 +- src/discof/backup/test_snap_roundtrip.c | 581 +++++--------- src/discof/execle/fd_execle_tile.c | 17 +- .../execle/fd_execle_tile.seccomppolicy | 13 +- .../execle/generated/fd_execle_tile_seccomp.h | 34 +- src/discof/execle/test_execle_tile.c | 15 +- src/discof/execrp/fd_execrp_tile.c | 17 +- .../execrp/fd_execrp_tile.seccomppolicy | 13 +- .../execrp/generated/fd_execrp_tile_seccomp.h | 34 +- src/discof/execrp/test_execrp_tile.c | 15 +- src/discof/replay/fd_replay_tile.c | 22 +- .../replay/fd_replay_tile.seccomppolicy | 13 +- .../replay/generated/fd_replay_tile_seccomp.h | 42 +- src/discof/replay/test_replay_tile.c | 1 + src/discof/restore/fd_snapin_tile.c | 11 +- .../restore/fd_snapin_tile.seccomppolicy | 12 +- .../generated/fd_snapin_tile_seccomp.h | 32 +- src/discof/restore/test_snapin_tile.c | 35 +- src/flamenco/runtime/Local.mk | 4 +- src/flamenco/runtime/fd_txncache.c | 715 +++++++++++++++--- src/flamenco/runtime/fd_txncache.h | 94 ++- src/flamenco/runtime/fd_txncache_private.h | 281 +++---- src/flamenco/runtime/fd_txncache_shmem.c | 112 ++- src/flamenco/runtime/fd_txncache_shmem.h | 16 +- .../runtime/fuzz_txncache_fork_graph.c | 137 ++-- src/flamenco/runtime/test_txncache.c | 464 ++++++++++-- src/flamenco/runtime/tests/fd_svm_mini.c | 12 +- 61 files changed, 2746 insertions(+), 1282 deletions(-) diff --git a/book/api/metrics-generated.md b/book/api/metrics-generated.md index 6786101f294..9452a8b6518 100644 --- a/book/api/metrics-generated.md +++ b/book/api/metrics-generated.md @@ -677,6 +677,11 @@ | execle_​accdb_​disk_​written_​bytes | counter | Bytes written to the account database file | | execle_​accdb_​disk_​written | counter | Writes to the account database file | | execle_​accdb_​copied_​bytes | counter | Account data bytes copied in memory between cache lines, staging buffers and callers; disk reads excluded | +| execle_​txncache_​query_​miss | counter | Transaction cache queries by this tile that found a page not in memory | +| execle_​txncache_​insert_​miss | counter | Transaction cache inserts by this tile that found a page not in memory | +| execle_​txncache_​operation_​stalled | counter | Transaction cache queries and inserts by this tile that fell back to the exclusive lock, stalling every tile; pages they load are read while holding it | +| execle_​txncache_​query_​duration_​nanos | histogram | Duration of one transaction cache query | +| execle_​txncache_​insert_​duration_​nanos | histogram | Duration of one transaction cache insert | @@ -1036,6 +1041,17 @@ | replay_​store_​fec_​cache_​used | gauge | FEC payload slots in use in the store RAM cache | | replay_​store_​fec_​cache_​capacity | gauge | FEC payload capacity of the store RAM cache | | replay_​store_​fec_​cache_​pinned | gauge | FEC payload views currently pinned against eviction | +| replay_​txncache_​page_​miss | counter | Transaction cache lookups under the read lock that found their page not resident | +| replay_​txncache_​page_​spilled | counter | Dirty transaction cache pages written to the spill file on eviction | +| replay_​txncache_​page_​spill_​read | counter | Transaction cache pages read back from the spill file | +| replay_​txncache_​frame_​evicted | counter | Transaction cache frames evicted to load another page | +| replay_​txncache_​spill_​duration_​nanos | counter | Time in transaction cache spill file reads and writes | +| replay_​txncache_​lock_​read_​wait_​duration_​nanos | counter | Time all tiles spent waiting to take the transaction cache shared lock | +| replay_​txncache_​lock_​write_​wait_​duration_​nanos | counter | Time all tiles spent waiting to take the transaction cache exclusive lock | +| replay_​txncache_​page_​used | gauge | Transaction cache logical pages in use, resident or spilled | +| replay_​txncache_​page_​capacity | gauge | Maximum transaction cache logical pages | +| replay_​txncache_​frame_​used | gauge | Transaction cache in-memory frames holding a page | +| replay_​txncache_​frame_​capacity | gauge | Transaction cache in-memory frames, from runtime.transaction_cache_size_mib | | replay_​is_​caught_​up | gauge | Whether replay has caught up to the cluster tip; set once per run, never cleared | | replay_​root_​slot | gauge | Most recent consensus root, 0 if none | | replay_​root_​slot_​distance | gauge | Slots between the current reset slot and our root | @@ -1323,6 +1339,11 @@ | execrp_​accdb_​disk_​written_​bytes | counter | Bytes written to the account database file | | execrp_​accdb_​disk_​written | counter | Writes to the account database file | | execrp_​accdb_​copied_​bytes | counter | Account data bytes copied in memory between cache lines, staging buffers and callers; disk reads excluded | +| execrp_​txncache_​query_​miss | counter | Transaction cache queries by this tile that found a page not in memory | +| execrp_​txncache_​insert_​miss | counter | Transaction cache inserts by this tile that found a page not in memory | +| execrp_​txncache_​operation_​stalled | counter | Transaction cache queries and inserts by this tile that fell back to the exclusive lock, stalling every tile; pages they load are read while holding it | +| execrp_​txncache_​query_​duration_​nanos | histogram | Duration of one transaction cache query | +| execrp_​txncache_​insert_​duration_​nanos | histogram | Duration of one transaction cache insert | diff --git a/contrib/ci/bench/bench.toml b/contrib/ci/bench/bench.toml index eef24f56c20..a38da6370e0 100644 --- a/contrib/ci/bench/bench.toml +++ b/contrib/ci/bench/bench.toml @@ -19,6 +19,7 @@ telemetry = false enabled = false [runtime] max_live_slots = 256 + transaction_cache_size_mib = 3072 [accounts] max_accounts = 4000000 cache_size_gib = 3 diff --git a/src/app/firedancer-dev/commands/backtest.c b/src/app/firedancer-dev/commands/backtest.c index ef694140325..139bd404be1 100644 --- a/src/app/firedancer-dev/commands/backtest.c +++ b/src/app/firedancer-dev/commands/backtest.c @@ -407,7 +407,7 @@ backtest_topo( config_t * config ) { FD_TEST( fd_pod_insertf_ulong( topo->props, banks_obj->id, "banks" ) ); fd_topob_wksp( topo, "txncache" ); - fd_topo_obj_t * txncache_obj = setup_topo_txncache( topo, "txncache", config->firedancer.runtime.max_live_slots, 2UL*config->limits.max_txn_per_slot ); + fd_topo_obj_t * txncache_obj = setup_topo_txncache( topo, "txncache", config->firedancer.runtime.max_live_slots, 2UL*config->limits.max_txn_per_slot, config->firedancer.runtime.transaction_cache_size_mib<<20 ); fd_topob_tile_uses( topo, replay_tile, txncache_obj, FD_SHMEM_JOIN_MODE_READ_WRITE ); if( FD_LIKELY( !disable_snap_loader ) ) { fd_topob_tile_uses( topo, &topo->tiles[ fd_topo_find_tile( topo, "snapin", 0UL ) ], txncache_obj, FD_SHMEM_JOIN_MODE_READ_WRITE ); @@ -539,6 +539,7 @@ backtest_cmd_fn( args_t * args, initialize_cost_tracker_fd( config ); initialize_vote_stakes_fd( config ); initialize_stake_delegations_fd( config ); + initialize_txncache_fd( config ); initialize_store_fds( config ); initialize_snapshot_fds( config ); diff --git a/src/app/firedancer-dev/commands/snapshot_load.c b/src/app/firedancer-dev/commands/snapshot_load.c index 2eb64d159d5..49450b50cc8 100644 --- a/src/app/firedancer-dev/commands/snapshot_load.c +++ b/src/app/firedancer-dev/commands/snapshot_load.c @@ -116,7 +116,7 @@ snapshot_load_topo( config_t * config ) { fd_topob_wksp( topo, "txncache" ); fd_topo_obj_t * txncache_obj = setup_topo_txncache( topo, "txncache", config->firedancer.runtime.max_live_slots, - 2UL*config->limits.max_txn_per_slot ); + 2UL*config->limits.max_txn_per_slot, config->firedancer.runtime.transaction_cache_size_mib<<20 ); FD_TEST( fd_pod_insertf_ulong( topo->props, txncache_obj->id, "txncache" ) ); fd_topob_wksp( topo, "accdb" ); @@ -496,6 +496,7 @@ snapshot_load_cmd_fn( args_t * args, initialize_cost_tracker_fd( config ); initialize_vote_stakes_fd( config ); initialize_stake_delegations_fd( config ); + initialize_txncache_fd( config ); initialize_store_fds( config ); initialize_snapshot_fds( config ); diff --git a/src/app/firedancer/callbacks.c b/src/app/firedancer/callbacks.c index de275697436..587a6b6abc9 100644 --- a/src/app/firedancer/callbacks.c +++ b/src/app/firedancer/callbacks.c @@ -182,7 +182,7 @@ fd_topo_obj_callbacks_t fd_obj_cb_accdb = { static ulong txncache_footprint( fd_topo_t const * topo, fd_topo_obj_t const * obj ) { - return fd_txncache_shmem_footprint( VAL("max_live_slots"), VAL("max_txn_per_slot") ); + return fd_txncache_shmem_footprint( VAL("max_live_slots"), VAL("max_txn_per_slot"), VAL("cache_footprint") ); } static ulong @@ -194,7 +194,7 @@ txncache_align( fd_topo_t const * topo FD_FN_UNUSED, static void txncache_new( fd_topo_t const * topo, fd_topo_obj_t const * obj ) { - FD_TEST( fd_txncache_shmem_new( fd_topo_obj_laddr( topo, obj->id ), VAL("max_live_slots"), VAL("max_txn_per_slot"), VAL("seed") ) ); + FD_TEST( fd_txncache_shmem_new( fd_topo_obj_laddr( topo, obj->id ), VAL("max_live_slots"), VAL("max_txn_per_slot"), VAL("cache_footprint"), VAL("seed") ) ); } fd_topo_obj_callbacks_t fd_obj_cb_txncache = { diff --git a/src/app/firedancer/config/bench-zen3-32core.toml b/src/app/firedancer/config/bench-zen3-32core.toml index 73c9132a3ff..4d40b27f850 100644 --- a/src/app/firedancer/config/bench-zen3-32core.toml +++ b/src/app/firedancer/config/bench-zen3-32core.toml @@ -16,6 +16,9 @@ shred_tile_count = 1 sign_tile_count = 2 +[runtime] + transaction_cache_size_mib = 3072 + [development.genesis] fund_initial_accounts = 32768 diff --git a/src/app/firedancer/config/bench-zen4-128core.toml b/src/app/firedancer/config/bench-zen4-128core.toml index 5b8cb872134..b3b66ec537b 100644 --- a/src/app/firedancer/config/bench-zen4-128core.toml +++ b/src/app/firedancer/config/bench-zen4-128core.toml @@ -11,6 +11,9 @@ shred_tile_count = 1 sign_tile_count = 2 +[runtime] + transaction_cache_size_mib = 3072 + [development.genesis] fund_initial_accounts = 32768 diff --git a/src/app/firedancer/config/default.toml b/src/app/firedancer/config/default.toml index 94b9c3db7f0..5bde1a775a5 100644 --- a/src/app/firedancer/config/default.toml +++ b/src/app/firedancer/config/default.toml @@ -248,6 +248,16 @@ telemetry = true # Firedancer instance. vote_stakes = "" + # File path used to create the temporary transaction-cache spill + # file. If no path is provided, it defaults to the `txncache.db` + # path within the base directory above. + # + # Two substitutions will be performed on this string. If "{user}" + # is present it will be replaced with the user running Firedancer, + # as above, and "{name}" will be replaced with the name of the + # Firedancer instance. + txncache = "" + # File path to the shreds database file. # # The shreds database stores recorded and reconstructed shreds @@ -817,6 +827,17 @@ telemetry = true # or test environments to decrease memory usage or test edge cases. bank_cache_cnt = 16 + # Memory budget for cached transactions in MiB, excluding metadata. + # When full, the cache spills to paths.txncache. This value has + # been chosen and tuned around mainnet network behavior. Lowering + # this value risks introducing performance regressions. To support + # worst-case snapshot loading for mainnet, the cache size must be at + # least 565 on mainnet. + # + # It is not recommended to change this value, except in development + # or test environments to decrease memory usage to test edge cases. + transaction_cache_size_mib = 480 + # CPU cores in Firedancer are carefully managed. Where a typical # program lets the operating system scheduler determine which threads to # run on which cores and for how long, Firedancer overrides most of this diff --git a/src/app/firedancer/topology.c b/src/app/firedancer/topology.c index e03747cd8f1..ada5258a934 100644 --- a/src/app/firedancer/topology.c +++ b/src/app/firedancer/topology.c @@ -23,8 +23,6 @@ #include "../../discof/restore/fd_snapct_tile.h" #include "../../disco/gui/fd_gui_config_parse.h" #include "../../disco/quic/fd_tpu.h" -#include "../../disco/pack/fd_pack_cost.h" -#include "../../disco/pack/fd_pack.h" #include "../../disco/tiles.h" #include "../../disco/topo/fd_topob.h" #include "../../disco/topo/fd_cpu_topo.h" @@ -38,6 +36,7 @@ #include "../../flamenco/capture/fd_solcap_writer.h" #include "../../flamenco/progcache/fd_progcache.h" #include "../../flamenco/runtime/fd_cost_tracker.h" +#include "../../flamenco/runtime/fd_txncache.h" #include #include @@ -208,13 +207,21 @@ fd_topo_obj_t * setup_topo_txncache( fd_topo_t * topo, char const * wksp_name, ulong max_live_slots, - ulong max_txn_per_slot ) { + ulong max_txn_per_slot, + ulong cache_footprint ) { + ulong scratch_min_sz = fd_txncache_snapin_scratch_min_sz( max_txn_per_slot ); + if( FD_UNLIKELY( cache_footprint>20, (scratch_min_sz+(1UL<<20)-1UL)>>20 )); + } + ulong seed; FD_TEST( fd_rng_secure( &seed, sizeof( ulong ) ) ); fd_topo_obj_t * obj = fd_topob_obj( topo, "txncache", wksp_name ); FD_TEST( fd_pod_insertf_ulong( topo->props, max_live_slots, "obj.%lu.max_live_slots", obj->id ) ); FD_TEST( fd_pod_insertf_ulong( topo->props, max_txn_per_slot, "obj.%lu.max_txn_per_slot", obj->id ) ); + FD_TEST( fd_pod_insertf_ulong( topo->props, cache_footprint, "obj.%lu.cache_footprint", obj->id ) ); FD_TEST( fd_pod_insertf_ulong( topo->props, seed, "obj.%lu.seed", obj->id ) ); return obj; @@ -1179,7 +1186,7 @@ fd_topo_initialize( config_t * config ) { if( rserve_enabled ) fd_topob_tile_uses( topo, &topo->tiles[ fd_topo_find_tile( topo, "rserve", 0UL ) ], store_obj, FD_SHMEM_JOIN_MODE_READ_WRITE ); FD_TEST( fd_pod_insertf_ulong( topo->props, store_obj->id, "store" ) ); - fd_topo_obj_t * txncache_obj = setup_topo_txncache( topo, "txncache", config->firedancer.runtime.max_live_slots, 2UL*config->limits.max_txn_per_slot /* sig + message hash */ ); + fd_topo_obj_t * txncache_obj = setup_topo_txncache( topo, "txncache", config->firedancer.runtime.max_live_slots, 2UL*config->limits.max_txn_per_slot /* sig + message hash */, config->firedancer.runtime.transaction_cache_size_mib<<20 ); fd_topob_tile_uses( topo, &topo->tiles[ fd_topo_find_tile( topo, "replay", 0UL ) ], txncache_obj, FD_SHMEM_JOIN_MODE_READ_WRITE ); if( FD_LIKELY( snapshots_enabled ) ) { fd_topob_tile_uses( topo, &topo->tiles[ fd_topo_find_tile( topo, "snapin", 0UL ) ], txncache_obj, FD_SHMEM_JOIN_MODE_READ_WRITE ); diff --git a/src/app/firedancer/topology.h b/src/app/firedancer/topology.h index e806cdf74ca..6d06e24d8cf 100644 --- a/src/app/firedancer/topology.h +++ b/src/app/firedancer/topology.h @@ -70,7 +70,8 @@ fd_topo_obj_t * setup_topo_txncache( fd_topo_t * topo, char const * wksp_name, ulong max_live_slots, - ulong max_txn_per_slot ); + ulong max_txn_per_slot, + ulong cache_footprint ); void fd_topo_configure_tile( fd_topo_tile_t * tile, diff --git a/src/app/shared/commands/run/run.c b/src/app/shared/commands/run/run.c index 24b57b8e34f..f35874556cc 100644 --- a/src/app/shared/commands/run/run.c +++ b/src/app/shared/commands/run/run.c @@ -5,6 +5,7 @@ #include "../../../../flamenco/runtime/fd_cost_tracker_store.h" #include "../../../../flamenco/stakes/fd_vote_stakes.h" #include "../../../../flamenco/stakes/fd_stake_delegations.h" +#include "../../../../flamenco/runtime/fd_txncache.h" #include "../../../../disco/store/fd_store.h" #include @@ -362,6 +363,7 @@ main_pid_namespace( void * _args ) { initialize_cost_tracker_fd( config ); initialize_vote_stakes_fd( config ); initialize_stake_delegations_fd( config ); + initialize_txncache_fd( config ); initialize_store_fds( config ); ulong store_obj_id = fd_pod_query_ulong( config->topo.props, "store", ULONG_MAX ); int has_store = store_obj_id!=ULONG_MAX; @@ -449,6 +451,12 @@ main_pid_namespace( void * _args ) { if( FD_UNLIKELY( -1==fcntl( FD_ACCDB_FD_RO, F_SETFD, FD_CLOEXEC ) ) ) FD_LOG_ERR(( "fcntl(F_SETFD,FD_CLOEXEC) failed (%i-%s)", errno, fd_io_strerror( errno ) )); } + ulong txncache_obj_id = fd_pod_query_ulong( config->topo.props, "txncache", ULONG_MAX ); + int tile_uses_txncache = 0; + for( ulong i=0UL; iuses_obj_cnt; i++ ) tile_uses_txncache |= tile->uses_obj_id[ i ]==txncache_obj_id; + if( FD_UNLIKELY( fcntl( FD_TXNCACHE_FD, F_SETFD, tile_uses_txncache ? 0 : FD_CLOEXEC )<0 ) ) + FD_LOG_ERR(( "fcntl(FD_TXNCACHE_FD,F_SETFD) failed (%i-%s)", errno, fd_io_strerror( errno ) )); + if( FD_LIKELY( has_store ) ) { int tile_uses_store = 0; for( ulong i=0UL; iuses_obj_cnt; i++ ) tile_uses_store |= tile->uses_obj_id[ i ]==store_obj_id; @@ -547,6 +555,7 @@ main_pid_namespace( void * _args ) { if( FD_UNLIKELY( -1==close( FD_ACCDB_FD_RW ) ) ) FD_LOG_ERR(( "close() failed (%i-%s)", errno, fd_io_strerror( errno ) )); if( FD_UNLIKELY( -1==close( FD_ACCDB_FD_RO ) ) ) FD_LOG_ERR(( "close() failed (%i-%s)", errno, fd_io_strerror( errno ) )); + if( FD_UNLIKELY( -1==close( FD_TXNCACHE_FD ) ) ) FD_LOG_ERR(( "close() failed (%i-%s)", errno, fd_io_strerror( errno ) )); if( FD_LIKELY( has_store ) ) { if( FD_UNLIKELY( -1==close( FD_STORE_FD_RW ) ) ) FD_LOG_ERR(( "close() failed (%i-%s)", errno, fd_io_strerror( errno ) )); if( FD_UNLIKELY( -1==close( FD_STORE_FD_RO ) ) ) FD_LOG_ERR(( "close() failed (%i-%s)", errno, fd_io_strerror( errno ) )); @@ -1161,6 +1170,19 @@ initialize_stake_delegations_fd( config_t const * config ) { } } +void +initialize_txncache_fd( config_t const * config ) { + char const * spill_path = config->paths.txncache; + int spill_fd = open( spill_path, O_RDWR|O_CREAT|O_TRUNC|O_NOATIME|O_DIRECT, S_IRUSR|S_IWUSR ); + if( FD_UNLIKELY( -1==spill_fd ) ) FD_LOG_ERR(( "failed to open %s (%i-%s)", spill_path, errno, fd_io_strerror( errno ) )); + if( FD_UNLIKELY( -1==unlink( spill_path ) ) ) FD_LOG_ERR(( "unlink(%s) failed (%i-%s)", spill_path, errno, fd_io_strerror( errno ) )); + + if( FD_LIKELY( spill_fd!=FD_TXNCACHE_FD ) ) { + if( FD_UNLIKELY( -1==dup2( spill_fd, FD_TXNCACHE_FD ) ) ) FD_LOG_ERR(( "dup2() failed (%i-%s)", errno, fd_io_strerror( errno ) )); + if( FD_UNLIKELY( -1==close( spill_fd ) ) ) FD_LOG_ERR(( "close() failed (%i-%s)", errno, fd_io_strerror( errno ) )); + } +} + void initialize_store_fds( config_t const * config ) { fd_topo_t const * topo = &config->topo; diff --git a/src/app/shared/commands/run/run.h b/src/app/shared/commands/run/run.h index 4189756f712..522153fd91a 100644 --- a/src/app/shared/commands/run/run.h +++ b/src/app/shared/commands/run/run.h @@ -38,6 +38,9 @@ initialize_vote_stakes_fd( config_t const * config ); void initialize_stake_delegations_fd( config_t const * config ); +void +initialize_txncache_fd( config_t const * config ); + void initialize_store_fds( config_t const * config ); diff --git a/src/app/shared/fd_config.c b/src/app/shared/fd_config.c index e4998b7be3c..48c9ee93d3d 100644 --- a/src/app/shared/fd_config.c +++ b/src/app/shared/fd_config.c @@ -156,6 +156,13 @@ fd_config_fillf( fd_config_t * config ) { FD_TEST( fd_cstr_printf_check( config->paths.vote_stakes, sizeof(config->paths.vote_stakes), NULL, "%s/votestakes.db", config->paths.base ) ); } + if( FD_UNLIKELY( strcmp( config->paths.txncache, "" ) ) ) { + replace( config->paths.txncache, "{user}", config->user ); + replace( config->paths.txncache, "{name}", config->name ); + } else { + FD_TEST( fd_cstr_printf_check( config->paths.txncache, sizeof(config->paths.txncache), NULL, "%s/txncache.db", config->paths.base ) ); + } + if( FD_UNLIKELY( strcmp( config->paths.shredb, "" ) ) ) { replace( config->paths.shredb, "{user}", config->user ); replace( config->paths.shredb, "{name}", config->name ); @@ -517,6 +524,10 @@ fd_config_validatef( fd_configf_t const * config ) { } CFG_HAS_NON_ZERO( runtime.program_cache_size_mib ); + CFG_HAS_NON_ZERO( runtime.transaction_cache_size_mib ); + if( FD_UNLIKELY( config->runtime.transaction_cache_size_mib>(ULONG_MAX>>20) ) ) { + FD_LOG_ERR(( "runtime.transaction_cache_size_mib is too large: %lu", config->runtime.transaction_cache_size_mib )); + } } void diff --git a/src/app/shared/fd_config.h b/src/app/shared/fd_config.h index b309ef0a4cd..95fca348cdd 100644 --- a/src/app/shared/fd_config.h +++ b/src/app/shared/fd_config.h @@ -41,6 +41,7 @@ struct fd_configf { ulong stake_delegation_cache_cnt; ulong program_cache_size_mib; ulong bank_cache_cnt; + ulong transaction_cache_size_mib; } runtime; struct { @@ -224,6 +225,7 @@ struct fd_config { char epoch_credits[ PATH_MAX ]; char cost_tracker[ PATH_MAX ]; char vote_stakes[ PATH_MAX ]; + char txncache[ PATH_MAX ]; char shredb[ PATH_MAX ]; char guidb[ PATH_MAX ]; char vote_history[ PATH_MAX ]; diff --git a/src/app/shared/fd_config_json.c b/src/app/shared/fd_config_json.c index 2ffb30e48f2..cf4f0738546 100644 --- a/src/app/shared/fd_config_json.c +++ b/src/app/shared/fd_config_json.c @@ -12,7 +12,7 @@ or knowingly skipped) before the constant is bumped. String keys of the user's own file are separately forced through the classification lists below. */ -FD_STATIC_ASSERT( sizeof(fd_config_t)==24458384UL, update_fd_config_to_json_for_the_layout_change ); +FD_STATIC_ASSERT( sizeof(fd_config_t)==24462488UL, update_fd_config_to_json_for_the_layout_change ); #define REDACTED "[redacted]" @@ -118,6 +118,7 @@ static char const * const jw_redacted_keys[] = { "paths.epoch_credits", "paths.cost_tracker", "paths.vote_stakes", + "paths.txncache", "paths.shredb", "paths.guidb", "paths.vote_history", @@ -327,6 +328,7 @@ fd_config_to_json( fd_config_t const * config, jw_path( &w, "epoch_credits", config->paths.epoch_credits ); jw_path( &w, "cost_tracker", config->paths.cost_tracker ); jw_path( &w, "vote_stakes", config->paths.vote_stakes ); + jw_path( &w, "txncache", config->paths.txncache ); jw_path( &w, "shredb", config->paths.shredb ); jw_path( &w, "guidb", config->paths.guidb ); jw_path( &w, "vote_history", config->paths.vote_history ); @@ -385,6 +387,7 @@ fd_config_to_json( fd_config_t const * config, jw_ulong( &w, "stake_delegation_cache_cnt", f->runtime.stake_delegation_cache_cnt ); jw_ulong( &w, "program_cache_size_mib", f->runtime.program_cache_size_mib ); jw_ulong( &w, "bank_cache_cnt", f->runtime.bank_cache_cnt ); + jw_ulong( &w, "transaction_cache_size_mib", f->runtime.transaction_cache_size_mib ); jw_obj_close( &w ); jw_obj_open( &w, "snapshots" ); diff --git a/src/app/shared/fd_config_parse.c b/src/app/shared/fd_config_parse.c index 138ce4ee862..157b31c354b 100644 --- a/src/app/shared/fd_config_parse.c +++ b/src/app/shared/fd_config_parse.c @@ -46,6 +46,7 @@ fd_config_extract_podf( uchar * pod, CFG_POP ( ulong, runtime.stake_delegation_cache_cnt ); CFG_POP ( ulong, runtime.program_cache_size_mib ); + CFG_POP ( ulong, runtime.transaction_cache_size_mib ); CFG_POP ( ulong, runtime.bank_cache_cnt ); @@ -121,6 +122,7 @@ fd_config_extract_pod( uchar * pod, CFG_POP ( cstr, paths.epoch_credits ); CFG_POP ( cstr, paths.cost_tracker ); CFG_POP ( cstr, paths.vote_stakes ); + CFG_POP ( cstr, paths.txncache ); CFG_POP ( cstr, paths.shredb ); CFG_POP ( cstr, paths.guidb ); CFG_POP ( cstr, paths.vote_history ); diff --git a/src/app/shared/test_config_json.c b/src/app/shared/test_config_json.c index a38d143381a..23f4852c5b6 100644 --- a/src/app/shared/test_config_json.c +++ b/src/app/shared/test_config_json.c @@ -56,6 +56,7 @@ main( int argc, static fd_config_t config[1]; fd_config_load( 0, default_config, strlen( default_config ), NULL, NULL, 0UL, NULL, 0UL, NULL, config, 0 ); + FD_TEST( config->firedancer.runtime.transaction_cache_size_mib==480UL ); char expected_stake_delegations[ PATH_MAX ]; FD_TEST( fd_cstr_printf_check( expected_stake_delegations, sizeof(expected_stake_delegations), @@ -84,6 +85,13 @@ main( int argc, "%s/votestakes.db", config->paths.base ) ); FD_TEST( !strcmp( config->paths.vote_stakes, expected_vote_stakes ) ); + char expected_txncache[ PATH_MAX ]; + FD_TEST( fd_cstr_printf_check( expected_txncache, + sizeof(expected_txncache), + NULL, + "%s/txncache.db", + config->paths.base ) ); + FD_TEST( !strcmp( config->paths.txncache, expected_txncache ) ); strcpy( config->tiles.bundle.url, "https://user:hunter2@mainnet.example.com:443/v1/txns?api-key=SECRET#frag" ); strcpy( config->tiles.event.url, "https://events.example.com/submit" ); @@ -149,6 +157,7 @@ main( int argc, " port = 8001\n" "[paths]\n" " accounts = \"/data/accounts\"\n" + " txncache = \"/data/txncache\"\n" "[tiles.pack]\n" " account_blocklist = []\n"; strcpy( config->user_config, user_toml ); @@ -159,6 +168,7 @@ main( int argc, FD_TEST( user_len && user_len==strlen( user_json ) ); FD_TEST( !strstr( user_json, "SECRET" ) ); FD_TEST( !strstr( user_json, "/data/accounts" ) ); + FD_TEST( !strstr( user_json, "/data/txncache" ) ); FD_TEST( strstr( user_json, "\"port\":8001" ) ); FD_TEST( !strstr( user_json, "entrypoint.mainnet-beta.solana.com" ) ); FD_TEST( strstr( user_json, "\"name\":\"fd1\"" ) ); @@ -211,6 +221,22 @@ main( int argc, } } + /* Custom spill paths use the same substitutions as the other paths. */ + static char const spill_toml[] = "[paths]\n txncache = \"/data/{user}/{name}/spill.db\"\n"; + fd_config_load( 0, default_config, strlen( default_config ), NULL, NULL, 0UL, spill_toml, strlen( spill_toml ), "spill.toml", config, 0 ); + FD_TEST( fd_cstr_printf_check( expected_txncache, + sizeof(expected_txncache), + NULL, + "/data/%s/%s/spill.db", + config->user, config->name ) ); + FD_TEST( !strcmp( config->paths.txncache, expected_txncache ) ); + + static char const cache_toml[] = "[runtime]\n transaction_cache_size_mib = 3072\n"; + fd_config_load( 0, default_config, strlen( default_config ), NULL, NULL, 0UL, cache_toml, strlen( cache_toml ), "cache.toml", config, 0 ); + FD_TEST( config->firedancer.runtime.transaction_cache_size_mib==3072UL ); + FD_TEST( fd_config_to_json( config, json, sizeof(json) ) ); + FD_TEST( strstr( json, "\"transaction_cache_size_mib\":3072" ) ); + FD_LOG_NOTICE(( "rendered %lu bytes", len )); FD_LOG_NOTICE(( "pass" )); fd_halt(); diff --git a/src/app/shared/test_config_parse.c b/src/app/shared/test_config_parse.c index ccbf91fa08a..60b2b06aab2 100644 --- a/src/app/shared/test_config_parse.c +++ b/src/app/shared/test_config_parse.c @@ -198,6 +198,7 @@ main( int argc, config->firedancer.accounts.max_accounts = 1UL; config->firedancer.accounts.cache_size_gib = 1UL; config->firedancer.runtime.program_cache_size_mib = 32UL; + config->firedancer.runtime.transaction_cache_size_mib = 640UL; config->tiles.repair.slot_max = 1UL; strcpy( config->firedancer.layout.mode, "performance" ); diff --git a/src/app/shared_dev/commands/bench/bench.c b/src/app/shared_dev/commands/bench/bench.c index a4b73d2d28d..8df8b9ebbc9 100644 --- a/src/app/shared_dev/commands/bench/bench.c +++ b/src/app/shared_dev/commands/bench/bench.c @@ -239,6 +239,7 @@ bench_cmd_fn( args_t * args, initialize_cost_tracker_fd( config ); initialize_vote_stakes_fd( config ); initialize_stake_delegations_fd( config ); + initialize_txncache_fd( config ); initialize_store_fds( config ); initialize_snapshot_fds( config ); diff --git a/src/app/shared_dev/commands/dev.c b/src/app/shared_dev/commands/dev.c index 418fbbedfab..b01ce6641d6 100644 --- a/src/app/shared_dev/commands/dev.c +++ b/src/app/shared_dev/commands/dev.c @@ -123,6 +123,7 @@ run_firedancer_threaded( config_t * config, initialize_cost_tracker_fd( config ); initialize_vote_stakes_fd( config ); initialize_stake_delegations_fd( config ); + initialize_txncache_fd( config ); initialize_store_fds( config ); initialize_snapshot_fds( config ); diff --git a/src/disco/events/schema/events.proto b/src/disco/events/schema/events.proto index 01cc2ddafc9..3ac57787183 100644 --- a/src/disco/events/schema/events.proto +++ b/src/disco/events/schema/events.proto @@ -2854,6 +2854,82 @@ message MetricsPackRegimeDurationNanos { uint64 backpressure_sleeping = 10; } +// Duration of one transaction cache insert +message MetricsExecleTxncacheInsertDurationNanos { + // [0, 10) nanos + uint64 b0 = 1; + // [10, 32) nanos + uint64 b1 = 2; + // [32, 101) nanos + uint64 b2 = 3; + // [101, 319) nanos + uint64 b3 = 4; + // [319, 1008) nanos + uint64 b4 = 5; + // [1008, 3185) nanos + uint64 b5 = 6; + // [3185, 10064) nanos + uint64 b6 = 7; + // [10064, 31800) nanos + uint64 b7 = 8; + // [31800, 100480) nanos + uint64 b8 = 9; + // [100480, 317492) nanos + uint64 b9 = 10; + // [317492, 1003197) nanos + uint64 b10 = 11; + // [1003197, 3169857) nanos + uint64 b11 = 12; + // [3169857, 10015972) nanos + uint64 b12 = 13; + // [10015972, 31648020) nanos + uint64 b13 = 14; + // [31648020, 100000000) nanos + uint64 b14 = 15; + // [100000000, inf) nanos + uint64 b15 = 16; + // Sum of samples (nanos) + uint64 sum = 17; +} + +// Duration of one transaction cache query +message MetricsExecleTxncacheQueryDurationNanos { + // [0, 10) nanos + uint64 b0 = 1; + // [10, 32) nanos + uint64 b1 = 2; + // [32, 101) nanos + uint64 b2 = 3; + // [101, 319) nanos + uint64 b3 = 4; + // [319, 1008) nanos + uint64 b4 = 5; + // [1008, 3185) nanos + uint64 b5 = 6; + // [3185, 10064) nanos + uint64 b6 = 7; + // [10064, 31800) nanos + uint64 b7 = 8; + // [31800, 100480) nanos + uint64 b8 = 9; + // [100480, 317492) nanos + uint64 b9 = 10; + // [317492, 1003197) nanos + uint64 b10 = 11; + // [1003197, 3169857) nanos + uint64 b11 = 12; + // [3169857, 10015972) nanos + uint64 b12 = 13; + // [10015972, 31648020) nanos + uint64 b13 = 14; + // [31648020, 100000000) nanos + uint64 b14 = 15; + // [100000000, inf) nanos + uint64 b15 = 16; + // Sum of samples (nanos) + uint64 sum = 17; +} + // Accounts not found in the account database cache and read from disk, by cache size class message MetricsExecleAccdbAccountCacheMissed { // 0-128 B @@ -6030,6 +6106,82 @@ message MetricsIpechoRegimeDurationNanos { uint64 backpressure_sleeping = 10; } +// Duration of one transaction cache insert +message MetricsExecrpTxncacheInsertDurationNanos { + // [0, 10) nanos + uint64 b0 = 1; + // [10, 32) nanos + uint64 b1 = 2; + // [32, 101) nanos + uint64 b2 = 3; + // [101, 319) nanos + uint64 b3 = 4; + // [319, 1008) nanos + uint64 b4 = 5; + // [1008, 3185) nanos + uint64 b5 = 6; + // [3185, 10064) nanos + uint64 b6 = 7; + // [10064, 31800) nanos + uint64 b7 = 8; + // [31800, 100480) nanos + uint64 b8 = 9; + // [100480, 317492) nanos + uint64 b9 = 10; + // [317492, 1003197) nanos + uint64 b10 = 11; + // [1003197, 3169857) nanos + uint64 b11 = 12; + // [3169857, 10015972) nanos + uint64 b12 = 13; + // [10015972, 31648020) nanos + uint64 b13 = 14; + // [31648020, 100000000) nanos + uint64 b14 = 15; + // [100000000, inf) nanos + uint64 b15 = 16; + // Sum of samples (nanos) + uint64 sum = 17; +} + +// Duration of one transaction cache query +message MetricsExecrpTxncacheQueryDurationNanos { + // [0, 10) nanos + uint64 b0 = 1; + // [10, 32) nanos + uint64 b1 = 2; + // [32, 101) nanos + uint64 b2 = 3; + // [101, 319) nanos + uint64 b3 = 4; + // [319, 1008) nanos + uint64 b4 = 5; + // [1008, 3185) nanos + uint64 b5 = 6; + // [3185, 10064) nanos + uint64 b6 = 7; + // [10064, 31800) nanos + uint64 b7 = 8; + // [31800, 100480) nanos + uint64 b8 = 9; + // [100480, 317492) nanos + uint64 b9 = 10; + // [317492, 1003197) nanos + uint64 b10 = 11; + // [1003197, 3169857) nanos + uint64 b11 = 12; + // [3169857, 10015972) nanos + uint64 b12 = 13; + // [10015972, 31648020) nanos + uint64 b13 = 14; + // [31648020, 100000000) nanos + uint64 b14 = 15; + // [100000000, inf) nanos + uint64 b15 = 16; + // Sum of samples (nanos) + uint64 sum = 17; +} + // Accounts not found in the account database cache and read from disk, by cache size class message MetricsExecrpAccdbAccountCacheMissed { // 0-128 B @@ -9674,6 +9826,16 @@ message MetricsExecle { uint64 accdb_copied_bytes = 61; // Cache invalidations due to transaction execution from a different block uint64 bank_switched = 62; + // Transaction cache queries by this tile that found a page not in memory + uint64 txncache_query_miss = 63; + // Transaction cache inserts by this tile that found a page not in memory + uint64 txncache_insert_miss = 64; + // Transaction cache queries and inserts by this tile that fell back to the exclusive lock, stalling every tile; pages they load are read while holding it + uint64 txncache_operation_stalled = 65; + // Duration of one transaction cache query + MetricsExecleTxncacheQueryDurationNanos txncache_query_duration_nanos = 66; + // Duration of one transaction cache insert + MetricsExecleTxncacheInsertDurationNanos txncache_insert_duration_nanos = 67; } // Metrics of one shred tile instance, one row per sweep @@ -10112,6 +10274,28 @@ message MetricsReplay { uint64 leader_window_already_produced = 118; // Leader windows given up after votor sent them because certificates already let a later window start (Alpenglow) uint64 leader_window_superseded = 119; + // Transaction cache lookups under the read lock that found their page not resident + uint64 txncache_page_miss = 120; + // Dirty transaction cache pages written to the spill file on eviction + uint64 txncache_page_spilled = 121; + // Transaction cache pages read back from the spill file + uint64 txncache_page_spill_read = 122; + // Transaction cache frames evicted to load another page + uint64 txncache_frame_evicted = 123; + // Time in transaction cache spill file reads and writes + uint64 txncache_spill_duration_nanos = 124; + // Transaction cache logical pages in use, resident or spilled + uint64 txncache_page_used = 125; + // Maximum transaction cache logical pages + uint64 txncache_page_capacity = 126; + // Transaction cache in-memory frames holding a page + uint64 txncache_frame_used = 127; + // Transaction cache in-memory frames, from runtime.transaction_cache_size_mib + uint64 txncache_frame_capacity = 128; + // Time all tiles spent waiting to take the transaction cache shared lock + uint64 txncache_lock_read_wait_duration_nanos = 129; + // Time all tiles spent waiting to take the transaction cache exclusive lock + uint64 txncache_lock_write_wait_duration_nanos = 130; } // Metrics of one rotor tile instance, one row per sweep @@ -11572,6 +11756,16 @@ message MetricsExecrp { uint64 accdb_copied_bytes = 63; // Cache invalidations due to transaction execution from a different block uint64 bank_switched = 64; + // Transaction cache queries by this tile that found a page not in memory + uint64 txncache_query_miss = 65; + // Transaction cache inserts by this tile that found a page not in memory + uint64 txncache_insert_miss = 66; + // Transaction cache queries and inserts by this tile that fell back to the exclusive lock, stalling every tile; pages they load are read while holding it + uint64 txncache_operation_stalled = 67; + // Duration of one transaction cache query + MetricsExecrpTxncacheQueryDurationNanos txncache_query_duration_nanos = 68; + // Duration of one transaction cache insert + MetricsExecrpTxncacheInsertDurationNanos txncache_insert_duration_nanos = 69; } // Metrics of one votor tile instance, one row per sweep diff --git a/src/disco/events/schema/metrics_execle.json b/src/disco/events/schema/metrics_execle.json index eae5827a279..05ae67f1036 100644 --- a/src/disco/events/schema/metrics_execle.json +++ b/src/disco/events/schema/metrics_execle.json @@ -359,6 +359,55 @@ "accdb_disk_written_bytes": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Bytes written to the account database file" }, "accdb_disk_written": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Writes to the account database file" }, "accdb_copied_bytes": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Account data bytes copied in memory between cache lines, staging buffers and callers; disk reads excluded" }, - "bank_switched": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Cache invalidations due to transaction execution from a different block" } + "bank_switched": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Cache invalidations due to transaction execution from a different block" }, + "txncache_query_miss": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Transaction cache queries by this tile that found a page not in memory" }, + "txncache_insert_miss": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Transaction cache inserts by this tile that found a page not in memory" }, + "txncache_operation_stalled": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Transaction cache queries and inserts by this tile that fell back to the exclusive lock, stalling every tile; pages they load are read while holding it" }, + "txncache_query_duration_nanos": { + "type": "Tuple", + "description": "Duration of one transaction cache query", + "fields": { + "b0": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "[0, 10) nanos" }, + "b1": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "[10, 32) nanos" }, + "b2": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "[32, 101) nanos" }, + "b3": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "[101, 319) nanos" }, + "b4": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "[319, 1008) nanos" }, + "b5": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "[1008, 3185) nanos" }, + "b6": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "[3185, 10064) nanos" }, + "b7": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "[10064, 31800) nanos" }, + "b8": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "[31800, 100480) nanos" }, + "b9": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "[100480, 317492) nanos" }, + "b10": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "[317492, 1003197) nanos" }, + "b11": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "[1003197, 3169857) nanos" }, + "b12": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "[3169857, 10015972) nanos" }, + "b13": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "[10015972, 31648020) nanos" }, + "b14": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "[31648020, 100000000) nanos" }, + "b15": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "[100000000, inf) nanos" }, + "sum": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Sum of samples (nanos)" } + } + }, + "txncache_insert_duration_nanos": { + "type": "Tuple", + "description": "Duration of one transaction cache insert", + "fields": { + "b0": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "[0, 10) nanos" }, + "b1": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "[10, 32) nanos" }, + "b2": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "[32, 101) nanos" }, + "b3": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "[101, 319) nanos" }, + "b4": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "[319, 1008) nanos" }, + "b5": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "[1008, 3185) nanos" }, + "b6": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "[3185, 10064) nanos" }, + "b7": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "[10064, 31800) nanos" }, + "b8": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "[31800, 100480) nanos" }, + "b9": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "[100480, 317492) nanos" }, + "b10": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "[317492, 1003197) nanos" }, + "b11": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "[1003197, 3169857) nanos" }, + "b12": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "[3169857, 10015972) nanos" }, + "b13": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "[10015972, 31648020) nanos" }, + "b14": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "[31648020, 100000000) nanos" }, + "b15": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "[100000000, inf) nanos" }, + "sum": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Sum of samples (nanos)" } + } + } } } diff --git a/src/disco/events/schema/metrics_execrp.json b/src/disco/events/schema/metrics_execrp.json index 85fb8a1bdec..3051abc4560 100644 --- a/src/disco/events/schema/metrics_execrp.json +++ b/src/disco/events/schema/metrics_execrp.json @@ -349,6 +349,55 @@ "accdb_disk_written_bytes": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Bytes written to the account database file" }, "accdb_disk_written": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Writes to the account database file" }, "accdb_copied_bytes": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Account data bytes copied in memory between cache lines, staging buffers and callers; disk reads excluded" }, - "bank_switched": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Cache invalidations due to transaction execution from a different block" } + "bank_switched": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Cache invalidations due to transaction execution from a different block" }, + "txncache_query_miss": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Transaction cache queries by this tile that found a page not in memory" }, + "txncache_insert_miss": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Transaction cache inserts by this tile that found a page not in memory" }, + "txncache_operation_stalled": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Transaction cache queries and inserts by this tile that fell back to the exclusive lock, stalling every tile; pages they load are read while holding it" }, + "txncache_query_duration_nanos": { + "type": "Tuple", + "description": "Duration of one transaction cache query", + "fields": { + "b0": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "[0, 10) nanos" }, + "b1": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "[10, 32) nanos" }, + "b2": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "[32, 101) nanos" }, + "b3": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "[101, 319) nanos" }, + "b4": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "[319, 1008) nanos" }, + "b5": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "[1008, 3185) nanos" }, + "b6": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "[3185, 10064) nanos" }, + "b7": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "[10064, 31800) nanos" }, + "b8": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "[31800, 100480) nanos" }, + "b9": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "[100480, 317492) nanos" }, + "b10": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "[317492, 1003197) nanos" }, + "b11": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "[1003197, 3169857) nanos" }, + "b12": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "[3169857, 10015972) nanos" }, + "b13": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "[10015972, 31648020) nanos" }, + "b14": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "[31648020, 100000000) nanos" }, + "b15": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "[100000000, inf) nanos" }, + "sum": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Sum of samples (nanos)" } + } + }, + "txncache_insert_duration_nanos": { + "type": "Tuple", + "description": "Duration of one transaction cache insert", + "fields": { + "b0": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "[0, 10) nanos" }, + "b1": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "[10, 32) nanos" }, + "b2": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "[32, 101) nanos" }, + "b3": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "[101, 319) nanos" }, + "b4": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "[319, 1008) nanos" }, + "b5": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "[1008, 3185) nanos" }, + "b6": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "[3185, 10064) nanos" }, + "b7": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "[10064, 31800) nanos" }, + "b8": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "[31800, 100480) nanos" }, + "b9": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "[100480, 317492) nanos" }, + "b10": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "[317492, 1003197) nanos" }, + "b11": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "[1003197, 3169857) nanos" }, + "b12": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "[3169857, 10015972) nanos" }, + "b13": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "[10015972, 31648020) nanos" }, + "b14": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "[31648020, 100000000) nanos" }, + "b15": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "[100000000, inf) nanos" }, + "sum": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Sum of samples (nanos)" } + } + } } } diff --git a/src/disco/events/schema/metrics_replay.json b/src/disco/events/schema/metrics_replay.json index 0b94b826744..a3bc2411aad 100644 --- a/src/disco/events/schema/metrics_replay.json +++ b/src/disco/events/schema/metrics_replay.json @@ -4,20 +4,20 @@ "description": "Metrics of one replay tile instance, one row per sweep", "local": true, "fields": { - "sample_id": { "type": "UInt64", "description": "Identifier shared by every row of one sweep" }, - "sample_reason": { "type": "ref:MetricsSampleReason", "description": "Why the sweep was taken" }, - "sample_slot": { "type": "UInt64", "description": "Slot that triggered the sweep, 0 for periodic sweeps" }, - "sample_snapct_state": { "type": "ref:MetricsSnapctLoadState", "description": "Snapshot load state entered, for snapct_state sweeps; unknown otherwise" }, - "sample_caught_up": { "type": "Bool", "description": "Whether replay had caught up to the cluster tip when the sweep was taken (replay IsCaughtUp); set once per run, never cleared" }, - "sample_tile_cnt": { "type": "UInt64", "description": "Tile instances in the sweep, to detect partial sweeps" }, - "kind_id": { "type": "UInt64", "description": "Index of this tile instance among tiles of its kind" }, - "pid": { "type": "UInt64", "compression": "T64, ZSTD", "description": "Process ID of the tile" }, - "tid": { "type": "UInt64", "compression": "T64, ZSTD", "description": "Thread ID of the tile. Same as Pid in production, but might differ in development" }, - "last_cpu_index": { "type": "UInt64", "compression": "T64, ZSTD", "description": "CPU the tile last ran on" }, - "context_switch_involuntary": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "counter: cumulative since tile start, query deltas. Involuntary context switches" }, - "context_switch_voluntary": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "counter: cumulative since tile start, query deltas. Voluntary context switches" }, - "page_fault_major": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "counter: cumulative since tile start, query deltas. Major page faults" }, - "page_fault_minor": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "counter: cumulative since tile start, query deltas. Minor page faults" }, + "sample_id": { "type": "UInt64", "description": "Identifier shared by every row of one sweep" }, + "sample_reason": { "type": "ref:MetricsSampleReason", "description": "Why the sweep was taken" }, + "sample_slot": { "type": "UInt64", "description": "Slot that triggered the sweep, 0 for periodic sweeps" }, + "sample_snapct_state": { "type": "ref:MetricsSnapctLoadState", "description": "Snapshot load state entered, for snapct_state sweeps; unknown otherwise" }, + "sample_caught_up": { "type": "Bool", "description": "Whether replay had caught up to the cluster tip when the sweep was taken (replay IsCaughtUp); set once per run, never cleared" }, + "sample_tile_cnt": { "type": "UInt64", "description": "Tile instances in the sweep, to detect partial sweeps" }, + "kind_id": { "type": "UInt64", "description": "Index of this tile instance among tiles of its kind" }, + "pid": { "type": "UInt64", "compression": "T64, ZSTD", "description": "Process ID of the tile" }, + "tid": { "type": "UInt64", "compression": "T64, ZSTD", "description": "Thread ID of the tile. Same as Pid in production, but might differ in development" }, + "last_cpu_index": { "type": "UInt64", "compression": "T64, ZSTD", "description": "CPU the tile last ran on" }, + "context_switch_involuntary": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "counter: cumulative since tile start, query deltas. Involuntary context switches" }, + "context_switch_voluntary": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "counter: cumulative since tile start, query deltas. Voluntary context switches" }, + "page_fault_major": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "counter: cumulative since tile start, query deltas. Major page faults" }, + "page_fault_minor": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "counter: cumulative since tile start, query deltas. Minor page faults" }, "state": { "type": "LowCardinality(String)", "description": "Tile lifecycle state: 0=booting, 1=running, 2=shutdown", @@ -27,9 +27,9 @@ "shutdown": { "description": "Shutdown" } } }, - "heartbeat_time": { "type": "DateTime64(9)", "compression": "Delta, ZSTD", "description": "Last wall-clock time the tile heartbeated" }, - "is_backpressured": { "type": "Bool", "description": "Whether the tile is currently backpressured" }, - "backpressure": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "counter: cumulative since tile start, query deltas. Times the tile waited for one or more consumers to catch up to resume publishing" }, + "heartbeat_time": { "type": "DateTime64(9)", "compression": "Delta, ZSTD", "description": "Last wall-clock time the tile heartbeated" }, + "is_backpressured": { "type": "Bool", "description": "Whether the tile is currently backpressured" }, + "backpressure": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "counter: cumulative since tile start, query deltas. Times the tile waited for one or more consumers to catch up to resume publishing" }, "regime_duration_nanos": { "type": "Tuple", "description": "Tile wall time, by regime; regimes are exhaustive and groups Busy, Idle, Backpressure and Housekeeping partition it", @@ -82,15 +82,15 @@ "pending": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Pre-park recheck found work, did not park" } } }, - "device_irq": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "counter: cumulative since tile start, query deltas. Device IRQs on the tile's dedicated CPU, 0 unless it is the only live tile pinned there" }, - "tlb_shootdown": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "counter: cumulative since tile start, query deltas. TLB shootdowns on the tile's dedicated CPU, 0 unless it is the only live tile pinned there" }, - "timer_irq": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "counter: cumulative since tile start, query deltas. Local timer interrupts (LOC) on the tile's dedicated CPU, 0 unless it is the only live tile pinned there" }, - "identity_balance_lamports": { "type": "UInt64", "compression": "T64, ZSTD", "description": "Identity account balance at the optimistically confirmed slot, or the finalized slot under Alpenglow" }, - "active_stake_lamports": { "type": "UInt64", "compression": "T64, ZSTD", "description": "Our active stake at the optimistically confirmed slot, or the finalized slot under Alpenglow" }, - "cluster_active_stake_lamports": { "type": "UInt64", "compression": "T64, ZSTD", "description": "Cluster active stake at the optimistically confirmed slot, or the finalized slot under Alpenglow" }, - "epoch_credits": { "type": "UInt64", "compression": "T64, ZSTD", "description": "gauge: current value, do not sum over time. Credits our vote account earned this epoch, as of the optimistically confirmed slot, or the finalized slot under Alpenglow, where they are lamport-denominated reward credits" }, - "is_vote_account_inadmissible": { "type": "Bool", "description": "Whether our vote account's stake is not admitted: at the optimistically confirmed slot it fails the validator admission ticket filter (missing, below the V4 rent-exempt minimum, or not V4 with a BLS pubkey), or it passes but has not yet been admitted at an epoch boundary" }, - "last_rewarded_vote_slot": { "type": "UInt64", "compression": "T64, ZSTD", "description": "Latest slot for which our vote appears in a reward certificate, 0 if none observed (always 0 under Tower)" }, + "device_irq": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "counter: cumulative since tile start, query deltas. Device IRQs on the tile's dedicated CPU, 0 unless it is the only live tile pinned there" }, + "tlb_shootdown": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "counter: cumulative since tile start, query deltas. TLB shootdowns on the tile's dedicated CPU, 0 unless it is the only live tile pinned there" }, + "timer_irq": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "counter: cumulative since tile start, query deltas. Local timer interrupts (LOC) on the tile's dedicated CPU, 0 unless it is the only live tile pinned there" }, + "identity_balance_lamports": { "type": "UInt64", "compression": "T64, ZSTD", "description": "Identity account balance at the optimistically confirmed slot, or the finalized slot under Alpenglow" }, + "active_stake_lamports": { "type": "UInt64", "compression": "T64, ZSTD", "description": "Our active stake at the optimistically confirmed slot, or the finalized slot under Alpenglow" }, + "cluster_active_stake_lamports": { "type": "UInt64", "compression": "T64, ZSTD", "description": "Cluster active stake at the optimistically confirmed slot, or the finalized slot under Alpenglow" }, + "epoch_credits": { "type": "UInt64", "compression": "T64, ZSTD", "description": "gauge: current value, do not sum over time. Credits our vote account earned this epoch, as of the optimistically confirmed slot, or the finalized slot under Alpenglow, where they are lamport-denominated reward credits" }, + "is_vote_account_inadmissible": { "type": "Bool", "description": "Whether our vote account's stake is not admitted: at the optimistically confirmed slot it fails the validator admission ticket filter (missing, below the V4 rent-exempt minimum, or not V4 with a BLS pubkey), or it passes but has not yet been admitted at an epoch boundary" }, + "last_rewarded_vote_slot": { "type": "UInt64", "compression": "T64, ZSTD", "description": "Latest slot for which our vote appears in a reward certificate, 0 if none observed (always 0 under Tower)" }, "store_query_duration_nanos": { "type": "Tuple", "description": "Duration of ingesting one queried FEC set into the scheduler", @@ -114,31 +114,31 @@ "sum": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Sum of samples (nanos)" } } }, - "store_fec_queried": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "FEC sets queried from the store" }, - "store_fec_query_missed": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "FEC set store queries that found no entry" }, - "store_fec_spilled": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "FEC payloads spilled from the store RAM cache to disk" }, - "store_fec_spilled_bytes": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "FEC payload bytes spilled from the store RAM cache to disk" }, - "store_fec_spill_read": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Spilled FEC payloads read from disk" }, - "store_fec_spill_read_bytes": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Spilled FEC payload bytes read from disk" }, - "store_fec_cache_used": { "type": "UInt64", "compression": "T64, ZSTD", "description": "FEC payload slots in use in the store RAM cache" }, - "store_fec_cache_capacity": { "type": "UInt64", "compression": "T64, ZSTD", "description": "FEC payload capacity of the store RAM cache" }, - "store_fec_cache_pinned": { "type": "UInt64", "compression": "T64, ZSTD", "description": "gauge: current value, do not sum over time. FEC payload views currently pinned against eviction" }, - "is_caught_up": { "type": "Bool", "description": "Whether replay has caught up to the cluster tip; set once per run, never cleared" }, - "root_slot": { "type": "UInt64", "compression": "T64, ZSTD", "description": "Most recent consensus root, 0 if none" }, - "root_slot_distance": { "type": "UInt64", "compression": "T64, ZSTD", "description": "Slots between the current reset slot and our root" }, - "leader_slot": { "type": "UInt64", "compression": "T64, ZSTD", "description": "Slot at which we are currently leader, or 0 if none" }, - "next_leader_slot": { "type": "UInt64", "compression": "T64, ZSTD", "description": "Slot at which we are next leader, or 0 if none. If we are currently leader, this is the same as the current leader slot" }, - "reset_slot": { "type": "UInt64", "compression": "T64, ZSTD", "description": "Slot replay last reset to (Tower reset slot; under Alpenglow the newest replayed slot or parent-ready parent); can decrease" }, - "bank_capacity": { "type": "UInt64", "compression": "T64, ZSTD", "description": "Maximum number of banks we can have alive" }, - "bank_used": { "type": "UInt64", "compression": "T64, ZSTD", "description": "Banks we currently have alive" }, - "reassembly_free": { "type": "UInt64", "compression": "T64, ZSTD", "description": "Free FEC sets in the reassembly queue, 0 under Alpenglow (no reassembly queue)", "deprecated": true }, - "last_reassembly_dequeued_slot": { "type": "UInt64", "compression": "T64, ZSTD", "description": "Slot of the FEC set most recently dequeued from reassembly for replay, 0 if none" }, - "last_reassembly_dequeued_fec_index": { "type": "UInt64", "compression": "T64, ZSTD", "description": "FEC set index of the FEC set most recently dequeued from reassembly for replay; read with LastReassemblyDequeuedSlot" }, - "sched_active_bank_index": { "type": "UInt64", "compression": "T64, ZSTD", "description": "Bank pool index the replay scheduler is dispatching work for, ULONG_MAX if none" }, - "sched_last_dispatch_bank_index": { "type": "UInt64", "compression": "T64, ZSTD", "description": "Bank pool index the replay scheduler last dispatched work for, ULONG_MAX if none" }, - "sched_last_dispatch_time": { "type": "DateTime64(9)", "compression": "Delta, ZSTD", "description": "Wall-clock time the replay scheduler last dispatched work, 0 if never" }, - "sched_staging_lane_used": { "type": "UInt64", "compression": "T64, ZSTD", "description": "Staging lanes currently occupied in the replay scheduler. Up to four staging lanes can be occupied at once" }, - "sched_staging_lane_peak": { "type": "UInt64", "compression": "T64, ZSTD", "description": "High watermark of staging lanes that were at one point occupied in the replay scheduler" }, + "store_fec_queried": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "FEC sets queried from the store" }, + "store_fec_query_missed": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "FEC set store queries that found no entry" }, + "store_fec_spilled": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "FEC payloads spilled from the store RAM cache to disk" }, + "store_fec_spilled_bytes": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "FEC payload bytes spilled from the store RAM cache to disk" }, + "store_fec_spill_read": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Spilled FEC payloads read from disk" }, + "store_fec_spill_read_bytes": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Spilled FEC payload bytes read from disk" }, + "store_fec_cache_used": { "type": "UInt64", "compression": "T64, ZSTD", "description": "FEC payload slots in use in the store RAM cache" }, + "store_fec_cache_capacity": { "type": "UInt64", "compression": "T64, ZSTD", "description": "FEC payload capacity of the store RAM cache" }, + "store_fec_cache_pinned": { "type": "UInt64", "compression": "T64, ZSTD", "description": "gauge: current value, do not sum over time. FEC payload views currently pinned against eviction" }, + "is_caught_up": { "type": "Bool", "description": "Whether replay has caught up to the cluster tip; set once per run, never cleared" }, + "root_slot": { "type": "UInt64", "compression": "T64, ZSTD", "description": "Most recent consensus root, 0 if none" }, + "root_slot_distance": { "type": "UInt64", "compression": "T64, ZSTD", "description": "Slots between the current reset slot and our root" }, + "leader_slot": { "type": "UInt64", "compression": "T64, ZSTD", "description": "Slot at which we are currently leader, or 0 if none" }, + "next_leader_slot": { "type": "UInt64", "compression": "T64, ZSTD", "description": "Slot at which we are next leader, or 0 if none. If we are currently leader, this is the same as the current leader slot" }, + "reset_slot": { "type": "UInt64", "compression": "T64, ZSTD", "description": "Slot replay last reset to (Tower reset slot; under Alpenglow the newest replayed slot or parent-ready parent); can decrease" }, + "bank_capacity": { "type": "UInt64", "compression": "T64, ZSTD", "description": "Maximum number of banks we can have alive" }, + "bank_used": { "type": "UInt64", "compression": "T64, ZSTD", "description": "Banks we currently have alive" }, + "reassembly_free": { "type": "UInt64", "compression": "T64, ZSTD", "description": "Free FEC sets in the reassembly queue, 0 under Alpenglow (no reassembly queue)", "deprecated": true }, + "last_reassembly_dequeued_slot": { "type": "UInt64", "compression": "T64, ZSTD", "description": "Slot of the FEC set most recently dequeued from reassembly for replay, 0 if none" }, + "last_reassembly_dequeued_fec_index": { "type": "UInt64", "compression": "T64, ZSTD", "description": "FEC set index of the FEC set most recently dequeued from reassembly for replay; read with LastReassemblyDequeuedSlot" }, + "sched_active_bank_index": { "type": "UInt64", "compression": "T64, ZSTD", "description": "Bank pool index the replay scheduler is dispatching work for, ULONG_MAX if none" }, + "sched_last_dispatch_bank_index": { "type": "UInt64", "compression": "T64, ZSTD", "description": "Bank pool index the replay scheduler last dispatched work for, ULONG_MAX if none" }, + "sched_last_dispatch_time": { "type": "DateTime64(9)", "compression": "Delta, ZSTD", "description": "Wall-clock time the replay scheduler last dispatched work, 0 if never" }, + "sched_staging_lane_used": { "type": "UInt64", "compression": "T64, ZSTD", "description": "Staging lanes currently occupied in the replay scheduler. Up to four staging lanes can be occupied at once" }, + "sched_staging_lane_peak": { "type": "UInt64", "compression": "T64, ZSTD", "description": "High watermark of staging lanes that were at one point occupied in the replay scheduler" }, "sched_staging_lane_head_bank_index": { "type": "Tuple", "description": "Bank pool index at the head of each replay staging lane, ULONG_MAX if the lane is empty", @@ -149,12 +149,12 @@ "lane3": { "type": "UInt64", "compression": "T64, ZSTD", "description": "Staging lane 3" } } }, - "sched_txn_pool_used": { "type": "UInt64", "compression": "T64, ZSTD", "description": "Transaction pool entries held by the replay scheduler, from parse until release" }, - "sched_txn_pool_capacity": { "type": "UInt64", "compression": "T64, ZSTD", "description": "Maximum transactions the replay scheduler can hold" }, - "sched_microblock_pool_used": { "type": "UInt64", "compression": "T64, ZSTD", "description": "Microblocks available to be scheduled in the replay scheduler" }, - "sched_microblock_pool_capacity": { "type": "UInt64", "compression": "T64, ZSTD", "description": "Maximum microblocks the replay scheduler can hold" }, - "sched_block_pool_used": { "type": "UInt64", "compression": "T64, ZSTD", "description": "Blocks alive in the replay scheduler" }, - "sched_block_pool_capacity": { "type": "UInt64", "compression": "T64, ZSTD", "description": "Maximum blocks the replay scheduler can track" }, + "sched_txn_pool_used": { "type": "UInt64", "compression": "T64, ZSTD", "description": "Transaction pool entries held by the replay scheduler, from parse until release" }, + "sched_txn_pool_capacity": { "type": "UInt64", "compression": "T64, ZSTD", "description": "Maximum transactions the replay scheduler can hold" }, + "sched_microblock_pool_used": { "type": "UInt64", "compression": "T64, ZSTD", "description": "Microblocks available to be scheduled in the replay scheduler" }, + "sched_microblock_pool_capacity": { "type": "UInt64", "compression": "T64, ZSTD", "description": "Maximum microblocks the replay scheduler can hold" }, + "sched_block_pool_used": { "type": "UInt64", "compression": "T64, ZSTD", "description": "Blocks alive in the replay scheduler" }, + "sched_block_pool_capacity": { "type": "UInt64", "compression": "T64, ZSTD", "description": "Maximum blocks the replay scheduler can track" }, "sched_block_added": { "type": "Tuple", "description": "Blocks added to the replay scheduler, by staging state", @@ -167,11 +167,11 @@ "dead": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Added as the dead child of a dead block, never dispatched" } } }, - "sched_block_replayed": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Blocks the replay scheduler considers fully replayed" }, - "sched_block_abandoned": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Blocks the replay scheduler abandoned for any reason" }, - "sched_block_rejected": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Blocks the replay scheduler found invalid while parsing or verifying PoH; also counted in SchedBlockAbandoned once drained; replay-tile verdicts not counted" }, - "sched_block_promoted": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Blocks the replay scheduler promoted from unstaged to a staging lane" }, - "sched_block_demoted": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Blocks the replay scheduler demoted from a staging lane to unstaged" }, + "sched_block_replayed": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Blocks the replay scheduler considers fully replayed" }, + "sched_block_abandoned": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Blocks the replay scheduler abandoned for any reason" }, + "sched_block_rejected": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Blocks the replay scheduler found invalid while parsing or verifying PoH; also counted in SchedBlockAbandoned once drained; replay-tile verdicts not counted" }, + "sched_block_promoted": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Blocks the replay scheduler promoted from unstaged to a staging lane" }, + "sched_block_demoted": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Blocks the replay scheduler demoted from a staging lane to unstaged" }, "sched_block_deactivated": { "type": "Tuple", "description": "Times the replay scheduler stopped dispatching a block, by reason", @@ -185,10 +185,10 @@ "minority_fork": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "A block was pruned for being on a minority fork (not also counted as Abandoned)" } } }, - "sched_lane_switched": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Times the replay scheduler switched the lane it is dispatching from" }, - "sched_lane_promoted": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Times the replay scheduler promoted one or more blocks onto a staging lane" }, - "sched_lane_demoted": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Times the replay scheduler demoted one or more blocks from a staging lane" }, - "sched_fork_observed": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Forks the replay scheduler has observed" }, + "sched_lane_switched": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Times the replay scheduler switched the lane it is dispatching from" }, + "sched_lane_promoted": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Times the replay scheduler promoted one or more blocks onto a staging lane" }, + "sched_lane_demoted": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Times the replay scheduler demoted one or more blocks from a staging lane" }, + "sched_fork_observed": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Forks the replay scheduler has observed" }, "sched_txn_lut_resolve_attempted": { "type": "Tuple", "description": "Transactions with address lookup tables the replay scheduler tried to resolve, by result", @@ -200,26 +200,26 @@ "failed": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Address lookup table expansion failed" } } }, - "sched_txn_parsed_abandoned": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Transactions that were parsed but ended up being in abandoned blocks" }, - "sched_txn_executed_abandoned": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Transactions that were executed but ended up being in abandoned blocks" }, - "sched_txn_done_abandoned": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Transactions that were fully replayed but ended up being in abandoned blocks" }, - "sched_txn_in_flight_integral_nanos": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Integral of in-flight transactions over time (transaction-nanoseconds); divide by SchedTxnBusyDurationNanos for the average in-flight count" }, - "sched_txn_busy_duration_nanos": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Time the replay scheduler had at least one transaction in flight" }, - "sched_txn_idle_duration_nanos": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Time the replay scheduler had no transaction in flight" }, - "sched_txn_parsed": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Transactions the replay pipeline has parsed" }, - "sched_txn_executed": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Transactions the replay pipeline has executed" }, - "sched_txn_signature_verified": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Transactions the replay pipeline has sigverified" }, - "sched_txn_poh_mixed": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Transactions the replay pipeline has performed PoH mixin on" }, - "sched_txn_done": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Transactions that were fully replayed" }, - "sched_microblock_parsed": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Microblocks the replay pipeline has parsed" }, - "sched_microblock_hashed": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Microblocks the replay pipeline has fully hashed" }, - "sched_microblock_done": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Microblocks that were fully replayed" }, - "sched_ingested_bytes": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Bytes the replay scheduler has ingested" }, - "sched_trailing_skipped_bytes": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Trailing bytes of a block the replay scheduler ingested but did not parse" }, - "sched_dropped_bytes": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Bytes the replay scheduler refused to ingest because the block was abandoned" }, - "sched_fec_ingested": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "FEC sets the replay scheduler has been given" }, - "bank_frozen": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Banks frozen, including our own leader blocks" }, - "chain_txn_height": { "type": "UInt64", "compression": "T64, ZSTD", "description": "gauge: current value, do not sum over time. Transactions from genesis through the last replayed block, 0 if none; drops on a fork switch" }, + "sched_txn_parsed_abandoned": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Transactions that were parsed but ended up being in abandoned blocks" }, + "sched_txn_executed_abandoned": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Transactions that were executed but ended up being in abandoned blocks" }, + "sched_txn_done_abandoned": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Transactions that were fully replayed but ended up being in abandoned blocks" }, + "sched_txn_in_flight_integral_nanos": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Integral of in-flight transactions over time (transaction-nanoseconds); divide by SchedTxnBusyDurationNanos for the average in-flight count" }, + "sched_txn_busy_duration_nanos": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Time the replay scheduler had at least one transaction in flight" }, + "sched_txn_idle_duration_nanos": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Time the replay scheduler had no transaction in flight" }, + "sched_txn_parsed": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Transactions the replay pipeline has parsed" }, + "sched_txn_executed": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Transactions the replay pipeline has executed" }, + "sched_txn_signature_verified": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Transactions the replay pipeline has sigverified" }, + "sched_txn_poh_mixed": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Transactions the replay pipeline has performed PoH mixin on" }, + "sched_txn_done": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Transactions that were fully replayed" }, + "sched_microblock_parsed": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Microblocks the replay pipeline has parsed" }, + "sched_microblock_hashed": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Microblocks the replay pipeline has fully hashed" }, + "sched_microblock_done": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Microblocks that were fully replayed" }, + "sched_ingested_bytes": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Bytes the replay scheduler has ingested" }, + "sched_trailing_skipped_bytes": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Trailing bytes of a block the replay scheduler ingested but did not parse" }, + "sched_dropped_bytes": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Bytes the replay scheduler refused to ingest because the block was abandoned" }, + "sched_fec_ingested": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "FEC sets the replay scheduler has been given" }, + "bank_frozen": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Banks frozen, including our own leader blocks" }, + "chain_txn_height": { "type": "UInt64", "compression": "T64, ZSTD", "description": "gauge: current value, do not sum over time. Transactions from genesis through the last replayed block, 0 if none; drops on a fork switch" }, "fec_poll_skipped": { "type": "Tuple", "description": "Replay loop iterations that processed no FEC set, by reason", @@ -233,9 +233,9 @@ "bank_full": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "The bank pool was full" } } }, - "fec_parent_missing_dropped": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "FEC sets dropped because replay could not resolve the block's parent (Alpenglow), which was dropped on a dead lineage, evicted with its tracking slot reused, or rooted past" }, - "root_advance_deferred": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Times the storage root could not advance to the consensus root yet" }, - "progcache_rooted": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Program cache entries rooted" }, + "fec_parent_missing_dropped": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "FEC sets dropped because replay could not resolve the block's parent (Alpenglow), which was dropped on a dead lineage, evicted with its tracking slot reused, or rooted past" }, + "root_advance_deferred": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Times the storage root could not advance to the consensus root yet" }, + "progcache_rooted": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Program cache entries rooted" }, "progcache_class_used": { "type": "Tuple", "description": "Program cache value slots in use, by size class", @@ -362,17 +362,28 @@ "le10_mib": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "1-10 MiB" } } }, - "accdb_account_waited": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Accounts that had to wait for another reader to finish loading them into the cache" }, - "accdb_batch_acquired": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Batch acquire calls made by this tile" }, - "accdb_cache_reserve_retried": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Spin retries while waiting to reserve cache lines for a transaction" }, - "accdb_disk_read_bytes": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Bytes read from the account database file" }, - "accdb_disk_read": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Reads from the account database file" }, - "accdb_disk_written_bytes": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Bytes written to the account database file" }, - "accdb_disk_written": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Writes to the account database file" }, - "accdb_copied_bytes": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Account data bytes copied in memory between cache lines, staging buffers and callers; disk reads excluded" }, - "is_runtime_loaded": { "type": "Bool", "description": "Whether the runtime has loaded a bank and can replay" }, - "fec_rooted_past_skipped": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Rotor FEC sets of a slot at or below the published root, or of a block whose parent is below it, skipped rather than dropped: no redelivery can replay them" }, - "leader_window_already_produced": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Leader windows given up because the cluster already produced one of their slots, eg. our own past slots replayed after a restart (Alpenglow)" }, - "leader_window_superseded": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Leader windows given up after votor sent them because certificates already let a later window start (Alpenglow)" } + "accdb_account_waited": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Accounts that had to wait for another reader to finish loading them into the cache" }, + "accdb_batch_acquired": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Batch acquire calls made by this tile" }, + "accdb_cache_reserve_retried": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Spin retries while waiting to reserve cache lines for a transaction" }, + "accdb_disk_read_bytes": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Bytes read from the account database file" }, + "accdb_disk_read": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Reads from the account database file" }, + "accdb_disk_written_bytes": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Bytes written to the account database file" }, + "accdb_disk_written": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Writes to the account database file" }, + "accdb_copied_bytes": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Account data bytes copied in memory between cache lines, staging buffers and callers; disk reads excluded" }, + "is_runtime_loaded": { "type": "Bool", "description": "Whether the runtime has loaded a bank and can replay" }, + "fec_rooted_past_skipped": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Rotor FEC sets of a slot at or below the published root, or of a block whose parent is below it, skipped rather than dropped: no redelivery can replay them" }, + "leader_window_already_produced": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Leader windows given up because the cluster already produced one of their slots, eg. our own past slots replayed after a restart (Alpenglow)" }, + "leader_window_superseded": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Leader windows given up after votor sent them because certificates already let a later window start (Alpenglow)" }, + "txncache_page_miss": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Transaction cache lookups under the read lock that found their page not resident" }, + "txncache_page_spilled": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Dirty transaction cache pages written to the spill file on eviction" }, + "txncache_page_spill_read": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Transaction cache pages read back from the spill file" }, + "txncache_frame_evicted": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Transaction cache frames evicted to load another page" }, + "txncache_spill_duration_nanos": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Time in transaction cache spill file reads and writes" }, + "txncache_page_used": { "type": "UInt64", "compression": "T64, ZSTD", "description": "Transaction cache logical pages in use, resident or spilled" }, + "txncache_page_capacity": { "type": "UInt64", "compression": "T64, ZSTD", "description": "Maximum transaction cache logical pages" }, + "txncache_frame_used": { "type": "UInt64", "compression": "T64, ZSTD", "description": "Transaction cache in-memory frames holding a page" }, + "txncache_frame_capacity": { "type": "UInt64", "compression": "T64, ZSTD", "description": "Transaction cache in-memory frames, from runtime.transaction_cache_size_mib" }, + "txncache_lock_read_wait_duration_nanos": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Time all tiles spent waiting to take the transaction cache shared lock" }, + "txncache_lock_write_wait_duration_nanos": { "type": "UInt64", "compression": "DoubleDelta, ZSTD", "description": "Time all tiles spent waiting to take the transaction cache exclusive lock" } } } diff --git a/src/disco/metrics/generated/fd_metrics_all.c b/src/disco/metrics/generated/fd_metrics_all.c index 939c0933e4a..7c592c24bda 100644 --- a/src/disco/metrics/generated/fd_metrics_all.c +++ b/src/disco/metrics/generated/fd_metrics_all.c @@ -559,6 +559,11 @@ const fd_metrics_meta_t FD_METRICS_EXECLE[FD_METRICS_EXECLE_TOTAL] = { DECLARE_METRIC( EXECLE_ACCDB_DISK_WRITTEN_BYTES, COUNTER ), DECLARE_METRIC( EXECLE_ACCDB_DISK_WRITTEN, COUNTER ), DECLARE_METRIC( EXECLE_ACCDB_COPIED_BYTES, COUNTER ), + DECLARE_METRIC( EXECLE_TXNCACHE_QUERY_MISS, COUNTER ), + DECLARE_METRIC( EXECLE_TXNCACHE_INSERT_MISS, COUNTER ), + DECLARE_METRIC( EXECLE_TXNCACHE_OPERATION_STALLED, COUNTER ), + DECLARE_METRIC_HISTOGRAM_TICKS( EXECLE_TXNCACHE_QUERY_DURATION_NANOS ), + DECLARE_METRIC_HISTOGRAM_TICKS( EXECLE_TXNCACHE_INSERT_DURATION_NANOS ), }; const fd_metrics_meta_t FD_METRICS_SIGN[FD_METRICS_SIGN_TOTAL] = { @@ -882,6 +887,17 @@ const fd_metrics_meta_t FD_METRICS_REPLAY[FD_METRICS_REPLAY_TOTAL] = { DECLARE_METRIC( REPLAY_STORE_FEC_CACHE_USED, GAUGE ), DECLARE_METRIC( REPLAY_STORE_FEC_CACHE_CAPACITY, GAUGE ), DECLARE_METRIC( REPLAY_STORE_FEC_CACHE_PINNED, GAUGE ), + DECLARE_METRIC( REPLAY_TXNCACHE_PAGE_MISS, COUNTER ), + DECLARE_METRIC( REPLAY_TXNCACHE_PAGE_SPILLED, COUNTER ), + DECLARE_METRIC( REPLAY_TXNCACHE_PAGE_SPILL_READ, COUNTER ), + DECLARE_METRIC( REPLAY_TXNCACHE_FRAME_EVICTED, COUNTER ), + DECLARE_METRIC( REPLAY_TXNCACHE_SPILL_DURATION_NANOS, COUNTER ), + DECLARE_METRIC( REPLAY_TXNCACHE_LOCK_READ_WAIT_DURATION_NANOS, COUNTER ), + DECLARE_METRIC( REPLAY_TXNCACHE_LOCK_WRITE_WAIT_DURATION_NANOS, COUNTER ), + DECLARE_METRIC( REPLAY_TXNCACHE_PAGE_USED, GAUGE ), + DECLARE_METRIC( REPLAY_TXNCACHE_PAGE_CAPACITY, GAUGE ), + DECLARE_METRIC( REPLAY_TXNCACHE_FRAME_USED, GAUGE ), + DECLARE_METRIC( REPLAY_TXNCACHE_FRAME_CAPACITY, GAUGE ), DECLARE_METRIC( REPLAY_IS_CAUGHT_UP, GAUGE ), DECLARE_METRIC( REPLAY_ROOT_SLOT, GAUGE ), DECLARE_METRIC( REPLAY_ROOT_SLOT_DISTANCE, GAUGE ), @@ -1163,6 +1179,11 @@ const fd_metrics_meta_t FD_METRICS_EXECRP[FD_METRICS_EXECRP_TOTAL] = { DECLARE_METRIC( EXECRP_ACCDB_DISK_WRITTEN_BYTES, COUNTER ), DECLARE_METRIC( EXECRP_ACCDB_DISK_WRITTEN, COUNTER ), DECLARE_METRIC( EXECRP_ACCDB_COPIED_BYTES, COUNTER ), + DECLARE_METRIC( EXECRP_TXNCACHE_QUERY_MISS, COUNTER ), + DECLARE_METRIC( EXECRP_TXNCACHE_INSERT_MISS, COUNTER ), + DECLARE_METRIC( EXECRP_TXNCACHE_OPERATION_STALLED, COUNTER ), + DECLARE_METRIC_HISTOGRAM_TICKS( EXECRP_TXNCACHE_QUERY_DURATION_NANOS ), + DECLARE_METRIC_HISTOGRAM_TICKS( EXECRP_TXNCACHE_INSERT_DURATION_NANOS ), }; const fd_metrics_meta_t FD_METRICS_ACCDB[FD_METRICS_ACCDB_TOTAL] = { diff --git a/src/disco/metrics/generated/fd_metrics_execle.h b/src/disco/metrics/generated/fd_metrics_execle.h index a5e929ffef5..710ec383fa6 100644 --- a/src/disco/metrics/generated/fd_metrics_execle.h +++ b/src/disco/metrics/generated/fd_metrics_execle.h @@ -160,6 +160,13 @@ enum { FD_METRICS_COUNTER_EXECLE_ACCDB_DISK_WRITTEN_BYTES_OFF, FD_METRICS_COUNTER_EXECLE_ACCDB_DISK_WRITTEN_OFF, FD_METRICS_COUNTER_EXECLE_ACCDB_COPIED_BYTES_OFF, + FD_METRICS_COUNTER_EXECLE_TXNCACHE_QUERY_MISS_OFF, + FD_METRICS_COUNTER_EXECLE_TXNCACHE_INSERT_MISS_OFF, + FD_METRICS_COUNTER_EXECLE_TXNCACHE_OPERATION_STALLED_OFF, + FD_METRICS_HISTOGRAM_EXECLE_TXNCACHE_QUERY_DURATION_NANOS_OFF, + FD_METRICS_HISTOGRAM_EXECLE_TXNCACHE_QUERY_DURATION_NANOS_OFF_END = FD_METRICS_HISTOGRAM_EXECLE_TXNCACHE_QUERY_DURATION_NANOS_OFF + 16, + FD_METRICS_HISTOGRAM_EXECLE_TXNCACHE_INSERT_DURATION_NANOS_OFF, + FD_METRICS_HISTOGRAM_EXECLE_TXNCACHE_INSERT_DURATION_NANOS_OFF_END = FD_METRICS_HISTOGRAM_EXECLE_TXNCACHE_INSERT_DURATION_NANOS_OFF + 16, }; #define FD_METRICS_COUNTER_EXECLE_TXN_PROCESSED_NAME "execle_txn_processed" @@ -367,7 +374,34 @@ enum { #define FD_METRICS_COUNTER_EXECLE_ACCDB_COPIED_BYTES_DESC "Account data bytes copied in memory between cache lines, staging buffers and callers; disk reads excluded" #define FD_METRICS_COUNTER_EXECLE_ACCDB_COPIED_BYTES_CVT (FD_METRICS_CONVERTER_NONE) -#define FD_METRICS_EXECLE_TOTAL (138UL) +#define FD_METRICS_COUNTER_EXECLE_TXNCACHE_QUERY_MISS_NAME "execle_txncache_query_miss" +#define FD_METRICS_COUNTER_EXECLE_TXNCACHE_QUERY_MISS_TYPE (FD_METRICS_TYPE_COUNTER) +#define FD_METRICS_COUNTER_EXECLE_TXNCACHE_QUERY_MISS_DESC "Transaction cache queries by this tile that found a page not in memory" +#define FD_METRICS_COUNTER_EXECLE_TXNCACHE_QUERY_MISS_CVT (FD_METRICS_CONVERTER_NONE) + +#define FD_METRICS_COUNTER_EXECLE_TXNCACHE_INSERT_MISS_NAME "execle_txncache_insert_miss" +#define FD_METRICS_COUNTER_EXECLE_TXNCACHE_INSERT_MISS_TYPE (FD_METRICS_TYPE_COUNTER) +#define FD_METRICS_COUNTER_EXECLE_TXNCACHE_INSERT_MISS_DESC "Transaction cache inserts by this tile that found a page not in memory" +#define FD_METRICS_COUNTER_EXECLE_TXNCACHE_INSERT_MISS_CVT (FD_METRICS_CONVERTER_NONE) + +#define FD_METRICS_COUNTER_EXECLE_TXNCACHE_OPERATION_STALLED_NAME "execle_txncache_operation_stalled" +#define FD_METRICS_COUNTER_EXECLE_TXNCACHE_OPERATION_STALLED_TYPE (FD_METRICS_TYPE_COUNTER) +#define FD_METRICS_COUNTER_EXECLE_TXNCACHE_OPERATION_STALLED_DESC "Transaction cache queries and inserts by this tile that fell back to the exclusive lock, stalling every tile; pages they load are read while holding it" +#define FD_METRICS_COUNTER_EXECLE_TXNCACHE_OPERATION_STALLED_CVT (FD_METRICS_CONVERTER_NONE) + +#define FD_METRICS_HISTOGRAM_EXECLE_TXNCACHE_QUERY_DURATION_NANOS_NAME "execle_txncache_query_duration_nanos" +#define FD_METRICS_HISTOGRAM_EXECLE_TXNCACHE_QUERY_DURATION_NANOS_TYPE (FD_METRICS_TYPE_HISTOGRAM) +#define FD_METRICS_HISTOGRAM_EXECLE_TXNCACHE_QUERY_DURATION_NANOS_DESC "Duration of one transaction cache query" +#define FD_METRICS_HISTOGRAM_EXECLE_TXNCACHE_QUERY_DURATION_NANOS_CVT (FD_METRICS_CONVERTER_NANOSECONDS) +#define FD_METRICS_HISTOGRAM_EXECLE_TXNCACHE_QUERY_DURATION_NANOS_EDGES_NS { 10UL, 32UL, 101UL, 319UL, 1008UL, 3185UL, 10064UL, 31800UL, 100480UL, 317492UL, 1003197UL, 3169857UL, 10015972UL, 31648020UL, 100000000UL } + +#define FD_METRICS_HISTOGRAM_EXECLE_TXNCACHE_INSERT_DURATION_NANOS_NAME "execle_txncache_insert_duration_nanos" +#define FD_METRICS_HISTOGRAM_EXECLE_TXNCACHE_INSERT_DURATION_NANOS_TYPE (FD_METRICS_TYPE_HISTOGRAM) +#define FD_METRICS_HISTOGRAM_EXECLE_TXNCACHE_INSERT_DURATION_NANOS_DESC "Duration of one transaction cache insert" +#define FD_METRICS_HISTOGRAM_EXECLE_TXNCACHE_INSERT_DURATION_NANOS_CVT (FD_METRICS_CONVERTER_NANOSECONDS) +#define FD_METRICS_HISTOGRAM_EXECLE_TXNCACHE_INSERT_DURATION_NANOS_EDGES_NS { 10UL, 32UL, 101UL, 319UL, 1008UL, 3185UL, 10064UL, 31800UL, 100480UL, 317492UL, 1003197UL, 3169857UL, 10015972UL, 31648020UL, 100000000UL } + +#define FD_METRICS_EXECLE_TOTAL (143UL) extern const fd_metrics_meta_t FD_METRICS_EXECLE[FD_METRICS_EXECLE_TOTAL]; #endif /* HEADER_fd_src_disco_metrics_generated_fd_metrics_execle_h */ diff --git a/src/disco/metrics/generated/fd_metrics_execrp.h b/src/disco/metrics/generated/fd_metrics_execrp.h index 1d429b86769..a6c8226ef39 100644 --- a/src/disco/metrics/generated/fd_metrics_execrp.h +++ b/src/disco/metrics/generated/fd_metrics_execrp.h @@ -158,6 +158,13 @@ enum { FD_METRICS_COUNTER_EXECRP_ACCDB_DISK_WRITTEN_BYTES_OFF, FD_METRICS_COUNTER_EXECRP_ACCDB_DISK_WRITTEN_OFF, FD_METRICS_COUNTER_EXECRP_ACCDB_COPIED_BYTES_OFF, + FD_METRICS_COUNTER_EXECRP_TXNCACHE_QUERY_MISS_OFF, + FD_METRICS_COUNTER_EXECRP_TXNCACHE_INSERT_MISS_OFF, + FD_METRICS_COUNTER_EXECRP_TXNCACHE_OPERATION_STALLED_OFF, + FD_METRICS_HISTOGRAM_EXECRP_TXNCACHE_QUERY_DURATION_NANOS_OFF, + FD_METRICS_HISTOGRAM_EXECRP_TXNCACHE_QUERY_DURATION_NANOS_OFF_END = FD_METRICS_HISTOGRAM_EXECRP_TXNCACHE_QUERY_DURATION_NANOS_OFF + 16, + FD_METRICS_HISTOGRAM_EXECRP_TXNCACHE_INSERT_DURATION_NANOS_OFF, + FD_METRICS_HISTOGRAM_EXECRP_TXNCACHE_INSERT_DURATION_NANOS_OFF_END = FD_METRICS_HISTOGRAM_EXECRP_TXNCACHE_INSERT_DURATION_NANOS_OFF + 16, }; #define FD_METRICS_GAUGE_EXECRP_IS_PROCESSING_NAME "execrp_is_processing" @@ -374,7 +381,34 @@ enum { #define FD_METRICS_COUNTER_EXECRP_ACCDB_COPIED_BYTES_DESC "Account data bytes copied in memory between cache lines, staging buffers and callers; disk reads excluded" #define FD_METRICS_COUNTER_EXECRP_ACCDB_COPIED_BYTES_CVT (FD_METRICS_CONVERTER_NONE) -#define FD_METRICS_EXECRP_TOTAL (137UL) +#define FD_METRICS_COUNTER_EXECRP_TXNCACHE_QUERY_MISS_NAME "execrp_txncache_query_miss" +#define FD_METRICS_COUNTER_EXECRP_TXNCACHE_QUERY_MISS_TYPE (FD_METRICS_TYPE_COUNTER) +#define FD_METRICS_COUNTER_EXECRP_TXNCACHE_QUERY_MISS_DESC "Transaction cache queries by this tile that found a page not in memory" +#define FD_METRICS_COUNTER_EXECRP_TXNCACHE_QUERY_MISS_CVT (FD_METRICS_CONVERTER_NONE) + +#define FD_METRICS_COUNTER_EXECRP_TXNCACHE_INSERT_MISS_NAME "execrp_txncache_insert_miss" +#define FD_METRICS_COUNTER_EXECRP_TXNCACHE_INSERT_MISS_TYPE (FD_METRICS_TYPE_COUNTER) +#define FD_METRICS_COUNTER_EXECRP_TXNCACHE_INSERT_MISS_DESC "Transaction cache inserts by this tile that found a page not in memory" +#define FD_METRICS_COUNTER_EXECRP_TXNCACHE_INSERT_MISS_CVT (FD_METRICS_CONVERTER_NONE) + +#define FD_METRICS_COUNTER_EXECRP_TXNCACHE_OPERATION_STALLED_NAME "execrp_txncache_operation_stalled" +#define FD_METRICS_COUNTER_EXECRP_TXNCACHE_OPERATION_STALLED_TYPE (FD_METRICS_TYPE_COUNTER) +#define FD_METRICS_COUNTER_EXECRP_TXNCACHE_OPERATION_STALLED_DESC "Transaction cache queries and inserts by this tile that fell back to the exclusive lock, stalling every tile; pages they load are read while holding it" +#define FD_METRICS_COUNTER_EXECRP_TXNCACHE_OPERATION_STALLED_CVT (FD_METRICS_CONVERTER_NONE) + +#define FD_METRICS_HISTOGRAM_EXECRP_TXNCACHE_QUERY_DURATION_NANOS_NAME "execrp_txncache_query_duration_nanos" +#define FD_METRICS_HISTOGRAM_EXECRP_TXNCACHE_QUERY_DURATION_NANOS_TYPE (FD_METRICS_TYPE_HISTOGRAM) +#define FD_METRICS_HISTOGRAM_EXECRP_TXNCACHE_QUERY_DURATION_NANOS_DESC "Duration of one transaction cache query" +#define FD_METRICS_HISTOGRAM_EXECRP_TXNCACHE_QUERY_DURATION_NANOS_CVT (FD_METRICS_CONVERTER_NANOSECONDS) +#define FD_METRICS_HISTOGRAM_EXECRP_TXNCACHE_QUERY_DURATION_NANOS_EDGES_NS { 10UL, 32UL, 101UL, 319UL, 1008UL, 3185UL, 10064UL, 31800UL, 100480UL, 317492UL, 1003197UL, 3169857UL, 10015972UL, 31648020UL, 100000000UL } + +#define FD_METRICS_HISTOGRAM_EXECRP_TXNCACHE_INSERT_DURATION_NANOS_NAME "execrp_txncache_insert_duration_nanos" +#define FD_METRICS_HISTOGRAM_EXECRP_TXNCACHE_INSERT_DURATION_NANOS_TYPE (FD_METRICS_TYPE_HISTOGRAM) +#define FD_METRICS_HISTOGRAM_EXECRP_TXNCACHE_INSERT_DURATION_NANOS_DESC "Duration of one transaction cache insert" +#define FD_METRICS_HISTOGRAM_EXECRP_TXNCACHE_INSERT_DURATION_NANOS_CVT (FD_METRICS_CONVERTER_NANOSECONDS) +#define FD_METRICS_HISTOGRAM_EXECRP_TXNCACHE_INSERT_DURATION_NANOS_EDGES_NS { 10UL, 32UL, 101UL, 319UL, 1008UL, 3185UL, 10064UL, 31800UL, 100480UL, 317492UL, 1003197UL, 3169857UL, 10015972UL, 31648020UL, 100000000UL } + +#define FD_METRICS_EXECRP_TOTAL (142UL) extern const fd_metrics_meta_t FD_METRICS_EXECRP[FD_METRICS_EXECRP_TOTAL]; #endif /* HEADER_fd_src_disco_metrics_generated_fd_metrics_execrp_h */ diff --git a/src/disco/metrics/generated/fd_metrics_replay.h b/src/disco/metrics/generated/fd_metrics_replay.h index a9fc3100ad8..9e60954276e 100644 --- a/src/disco/metrics/generated/fd_metrics_replay.h +++ b/src/disco/metrics/generated/fd_metrics_replay.h @@ -24,6 +24,17 @@ enum { FD_METRICS_GAUGE_REPLAY_STORE_FEC_CACHE_USED_OFF, FD_METRICS_GAUGE_REPLAY_STORE_FEC_CACHE_CAPACITY_OFF, FD_METRICS_GAUGE_REPLAY_STORE_FEC_CACHE_PINNED_OFF, + FD_METRICS_COUNTER_REPLAY_TXNCACHE_PAGE_MISS_OFF, + FD_METRICS_COUNTER_REPLAY_TXNCACHE_PAGE_SPILLED_OFF, + FD_METRICS_COUNTER_REPLAY_TXNCACHE_PAGE_SPILL_READ_OFF, + FD_METRICS_COUNTER_REPLAY_TXNCACHE_FRAME_EVICTED_OFF, + FD_METRICS_COUNTER_REPLAY_TXNCACHE_SPILL_DURATION_NANOS_OFF, + FD_METRICS_COUNTER_REPLAY_TXNCACHE_LOCK_READ_WAIT_DURATION_NANOS_OFF, + FD_METRICS_COUNTER_REPLAY_TXNCACHE_LOCK_WRITE_WAIT_DURATION_NANOS_OFF, + FD_METRICS_GAUGE_REPLAY_TXNCACHE_PAGE_USED_OFF, + FD_METRICS_GAUGE_REPLAY_TXNCACHE_PAGE_CAPACITY_OFF, + FD_METRICS_GAUGE_REPLAY_TXNCACHE_FRAME_USED_OFF, + FD_METRICS_GAUGE_REPLAY_TXNCACHE_FRAME_CAPACITY_OFF, FD_METRICS_GAUGE_REPLAY_IS_CAUGHT_UP_OFF, FD_METRICS_GAUGE_REPLAY_ROOT_SLOT_OFF, FD_METRICS_GAUGE_REPLAY_ROOT_SLOT_DISTANCE_OFF, @@ -262,6 +273,61 @@ enum { #define FD_METRICS_GAUGE_REPLAY_STORE_FEC_CACHE_PINNED_DESC "FEC payload views currently pinned against eviction" #define FD_METRICS_GAUGE_REPLAY_STORE_FEC_CACHE_PINNED_CVT (FD_METRICS_CONVERTER_NONE) +#define FD_METRICS_COUNTER_REPLAY_TXNCACHE_PAGE_MISS_NAME "replay_txncache_page_miss" +#define FD_METRICS_COUNTER_REPLAY_TXNCACHE_PAGE_MISS_TYPE (FD_METRICS_TYPE_COUNTER) +#define FD_METRICS_COUNTER_REPLAY_TXNCACHE_PAGE_MISS_DESC "Transaction cache lookups under the read lock that found their page not resident" +#define FD_METRICS_COUNTER_REPLAY_TXNCACHE_PAGE_MISS_CVT (FD_METRICS_CONVERTER_NONE) + +#define FD_METRICS_COUNTER_REPLAY_TXNCACHE_PAGE_SPILLED_NAME "replay_txncache_page_spilled" +#define FD_METRICS_COUNTER_REPLAY_TXNCACHE_PAGE_SPILLED_TYPE (FD_METRICS_TYPE_COUNTER) +#define FD_METRICS_COUNTER_REPLAY_TXNCACHE_PAGE_SPILLED_DESC "Dirty transaction cache pages written to the spill file on eviction" +#define FD_METRICS_COUNTER_REPLAY_TXNCACHE_PAGE_SPILLED_CVT (FD_METRICS_CONVERTER_NONE) + +#define FD_METRICS_COUNTER_REPLAY_TXNCACHE_PAGE_SPILL_READ_NAME "replay_txncache_page_spill_read" +#define FD_METRICS_COUNTER_REPLAY_TXNCACHE_PAGE_SPILL_READ_TYPE (FD_METRICS_TYPE_COUNTER) +#define FD_METRICS_COUNTER_REPLAY_TXNCACHE_PAGE_SPILL_READ_DESC "Transaction cache pages read back from the spill file" +#define FD_METRICS_COUNTER_REPLAY_TXNCACHE_PAGE_SPILL_READ_CVT (FD_METRICS_CONVERTER_NONE) + +#define FD_METRICS_COUNTER_REPLAY_TXNCACHE_FRAME_EVICTED_NAME "replay_txncache_frame_evicted" +#define FD_METRICS_COUNTER_REPLAY_TXNCACHE_FRAME_EVICTED_TYPE (FD_METRICS_TYPE_COUNTER) +#define FD_METRICS_COUNTER_REPLAY_TXNCACHE_FRAME_EVICTED_DESC "Transaction cache frames evicted to load another page" +#define FD_METRICS_COUNTER_REPLAY_TXNCACHE_FRAME_EVICTED_CVT (FD_METRICS_CONVERTER_NONE) + +#define FD_METRICS_COUNTER_REPLAY_TXNCACHE_SPILL_DURATION_NANOS_NAME "replay_txncache_spill_duration_nanos" +#define FD_METRICS_COUNTER_REPLAY_TXNCACHE_SPILL_DURATION_NANOS_TYPE (FD_METRICS_TYPE_COUNTER) +#define FD_METRICS_COUNTER_REPLAY_TXNCACHE_SPILL_DURATION_NANOS_DESC "Time in transaction cache spill file reads and writes" +#define FD_METRICS_COUNTER_REPLAY_TXNCACHE_SPILL_DURATION_NANOS_CVT (FD_METRICS_CONVERTER_NANOSECONDS) + +#define FD_METRICS_COUNTER_REPLAY_TXNCACHE_LOCK_READ_WAIT_DURATION_NANOS_NAME "replay_txncache_lock_read_wait_duration_nanos" +#define FD_METRICS_COUNTER_REPLAY_TXNCACHE_LOCK_READ_WAIT_DURATION_NANOS_TYPE (FD_METRICS_TYPE_COUNTER) +#define FD_METRICS_COUNTER_REPLAY_TXNCACHE_LOCK_READ_WAIT_DURATION_NANOS_DESC "Time all tiles spent waiting to take the transaction cache shared lock" +#define FD_METRICS_COUNTER_REPLAY_TXNCACHE_LOCK_READ_WAIT_DURATION_NANOS_CVT (FD_METRICS_CONVERTER_NANOSECONDS) + +#define FD_METRICS_COUNTER_REPLAY_TXNCACHE_LOCK_WRITE_WAIT_DURATION_NANOS_NAME "replay_txncache_lock_write_wait_duration_nanos" +#define FD_METRICS_COUNTER_REPLAY_TXNCACHE_LOCK_WRITE_WAIT_DURATION_NANOS_TYPE (FD_METRICS_TYPE_COUNTER) +#define FD_METRICS_COUNTER_REPLAY_TXNCACHE_LOCK_WRITE_WAIT_DURATION_NANOS_DESC "Time all tiles spent waiting to take the transaction cache exclusive lock" +#define FD_METRICS_COUNTER_REPLAY_TXNCACHE_LOCK_WRITE_WAIT_DURATION_NANOS_CVT (FD_METRICS_CONVERTER_NANOSECONDS) + +#define FD_METRICS_GAUGE_REPLAY_TXNCACHE_PAGE_USED_NAME "replay_txncache_page_used" +#define FD_METRICS_GAUGE_REPLAY_TXNCACHE_PAGE_USED_TYPE (FD_METRICS_TYPE_GAUGE) +#define FD_METRICS_GAUGE_REPLAY_TXNCACHE_PAGE_USED_DESC "Transaction cache logical pages in use, resident or spilled" +#define FD_METRICS_GAUGE_REPLAY_TXNCACHE_PAGE_USED_CVT (FD_METRICS_CONVERTER_NONE) + +#define FD_METRICS_GAUGE_REPLAY_TXNCACHE_PAGE_CAPACITY_NAME "replay_txncache_page_capacity" +#define FD_METRICS_GAUGE_REPLAY_TXNCACHE_PAGE_CAPACITY_TYPE (FD_METRICS_TYPE_GAUGE) +#define FD_METRICS_GAUGE_REPLAY_TXNCACHE_PAGE_CAPACITY_DESC "Maximum transaction cache logical pages" +#define FD_METRICS_GAUGE_REPLAY_TXNCACHE_PAGE_CAPACITY_CVT (FD_METRICS_CONVERTER_NONE) + +#define FD_METRICS_GAUGE_REPLAY_TXNCACHE_FRAME_USED_NAME "replay_txncache_frame_used" +#define FD_METRICS_GAUGE_REPLAY_TXNCACHE_FRAME_USED_TYPE (FD_METRICS_TYPE_GAUGE) +#define FD_METRICS_GAUGE_REPLAY_TXNCACHE_FRAME_USED_DESC "Transaction cache in-memory frames holding a page" +#define FD_METRICS_GAUGE_REPLAY_TXNCACHE_FRAME_USED_CVT (FD_METRICS_CONVERTER_NONE) + +#define FD_METRICS_GAUGE_REPLAY_TXNCACHE_FRAME_CAPACITY_NAME "replay_txncache_frame_capacity" +#define FD_METRICS_GAUGE_REPLAY_TXNCACHE_FRAME_CAPACITY_TYPE (FD_METRICS_TYPE_GAUGE) +#define FD_METRICS_GAUGE_REPLAY_TXNCACHE_FRAME_CAPACITY_DESC "Transaction cache in-memory frames, from runtime.transaction_cache_size_mib" +#define FD_METRICS_GAUGE_REPLAY_TXNCACHE_FRAME_CAPACITY_CVT (FD_METRICS_CONVERTER_NONE) + #define FD_METRICS_GAUGE_REPLAY_IS_CAUGHT_UP_NAME "replay_is_caught_up" #define FD_METRICS_GAUGE_REPLAY_IS_CAUGHT_UP_TYPE (FD_METRICS_TYPE_GAUGE) #define FD_METRICS_GAUGE_REPLAY_IS_CAUGHT_UP_DESC "Whether replay has caught up to the cluster tip; set once per run, never cleared" @@ -665,7 +731,7 @@ enum { #define FD_METRICS_GAUGE_REPLAY_IS_RUNTIME_LOADED_DESC "Whether the runtime has loaded a bank and can replay" #define FD_METRICS_GAUGE_REPLAY_IS_RUNTIME_LOADED_CVT (FD_METRICS_CONVERTER_NONE) -#define FD_METRICS_REPLAY_TOTAL (157UL) +#define FD_METRICS_REPLAY_TOTAL (168UL) extern const fd_metrics_meta_t FD_METRICS_REPLAY[FD_METRICS_REPLAY_TOTAL]; #endif /* HEADER_fd_src_disco_metrics_generated_fd_metrics_replay_h */ diff --git a/src/disco/metrics/generated/fd_metrics_sample_gen.c b/src/disco/metrics/generated/fd_metrics_sample_gen.c index b5909a5fd61..88f5921fb88 100644 --- a/src/disco/metrics/generated/fd_metrics_sample_gen.c +++ b/src/disco/metrics/generated/fd_metrics_sample_gen.c @@ -1008,7 +1008,7 @@ static fd_metrics_sample_field_t const fields_pack[ 283 ] = { { 0, 0U, FD_METRICS_SAMPLE_CLOSE, NULL, 0UL }, }; -static fd_metrics_sample_field_t const fields_execle[ 206 ] = { +static fd_metrics_sample_field_t const fields_execle[ 247 ] = { { FD_METRICS_GAUGE_TILE_PID_OFF, 8U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, { FD_METRICS_GAUGE_TILE_TID_OFF, 9U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, { FD_METRICS_GAUGE_TILE_LAST_CPU_INDEX_OFF, 10U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, @@ -1215,6 +1215,47 @@ static fd_metrics_sample_field_t const fields_execle[ 206 ] = { { FD_METRICS_COUNTER_EXECLE_ACCDB_DISK_WRITTEN_BYTES_OFF, 59U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, { FD_METRICS_COUNTER_EXECLE_ACCDB_DISK_WRITTEN_OFF, 60U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, { FD_METRICS_COUNTER_EXECLE_ACCDB_COPIED_BYTES_OFF, 61U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_COUNTER_EXECLE_TXNCACHE_QUERY_MISS_OFF, 63U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_COUNTER_EXECLE_TXNCACHE_INSERT_MISS_OFF, 64U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_COUNTER_EXECLE_TXNCACHE_OPERATION_STALLED_OFF, 65U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_HISTOGRAM_EXECLE_TXNCACHE_QUERY_DURATION_NANOS_OFF, 66U, FD_METRICS_SAMPLE_OPEN, NULL, 0UL }, + { FD_METRICS_HISTOGRAM_EXECLE_TXNCACHE_QUERY_DURATION_NANOS_OFF+0, 1U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_HISTOGRAM_EXECLE_TXNCACHE_QUERY_DURATION_NANOS_OFF+1, 2U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_HISTOGRAM_EXECLE_TXNCACHE_QUERY_DURATION_NANOS_OFF+2, 3U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_HISTOGRAM_EXECLE_TXNCACHE_QUERY_DURATION_NANOS_OFF+3, 4U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_HISTOGRAM_EXECLE_TXNCACHE_QUERY_DURATION_NANOS_OFF+4, 5U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_HISTOGRAM_EXECLE_TXNCACHE_QUERY_DURATION_NANOS_OFF+5, 6U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_HISTOGRAM_EXECLE_TXNCACHE_QUERY_DURATION_NANOS_OFF+6, 7U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_HISTOGRAM_EXECLE_TXNCACHE_QUERY_DURATION_NANOS_OFF+7, 8U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_HISTOGRAM_EXECLE_TXNCACHE_QUERY_DURATION_NANOS_OFF+8, 9U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_HISTOGRAM_EXECLE_TXNCACHE_QUERY_DURATION_NANOS_OFF+9, 10U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_HISTOGRAM_EXECLE_TXNCACHE_QUERY_DURATION_NANOS_OFF+10, 11U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_HISTOGRAM_EXECLE_TXNCACHE_QUERY_DURATION_NANOS_OFF+11, 12U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_HISTOGRAM_EXECLE_TXNCACHE_QUERY_DURATION_NANOS_OFF+12, 13U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_HISTOGRAM_EXECLE_TXNCACHE_QUERY_DURATION_NANOS_OFF+13, 14U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_HISTOGRAM_EXECLE_TXNCACHE_QUERY_DURATION_NANOS_OFF+14, 15U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_HISTOGRAM_EXECLE_TXNCACHE_QUERY_DURATION_NANOS_OFF+15, 16U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_HISTOGRAM_EXECLE_TXNCACHE_QUERY_DURATION_NANOS_OFF+16, 17U, FD_METRICS_SAMPLE_NS, NULL, 0UL }, + { 0, 0U, FD_METRICS_SAMPLE_CLOSE, NULL, 0UL }, + { FD_METRICS_HISTOGRAM_EXECLE_TXNCACHE_INSERT_DURATION_NANOS_OFF, 67U, FD_METRICS_SAMPLE_OPEN, NULL, 0UL }, + { FD_METRICS_HISTOGRAM_EXECLE_TXNCACHE_INSERT_DURATION_NANOS_OFF+0, 1U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_HISTOGRAM_EXECLE_TXNCACHE_INSERT_DURATION_NANOS_OFF+1, 2U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_HISTOGRAM_EXECLE_TXNCACHE_INSERT_DURATION_NANOS_OFF+2, 3U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_HISTOGRAM_EXECLE_TXNCACHE_INSERT_DURATION_NANOS_OFF+3, 4U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_HISTOGRAM_EXECLE_TXNCACHE_INSERT_DURATION_NANOS_OFF+4, 5U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_HISTOGRAM_EXECLE_TXNCACHE_INSERT_DURATION_NANOS_OFF+5, 6U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_HISTOGRAM_EXECLE_TXNCACHE_INSERT_DURATION_NANOS_OFF+6, 7U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_HISTOGRAM_EXECLE_TXNCACHE_INSERT_DURATION_NANOS_OFF+7, 8U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_HISTOGRAM_EXECLE_TXNCACHE_INSERT_DURATION_NANOS_OFF+8, 9U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_HISTOGRAM_EXECLE_TXNCACHE_INSERT_DURATION_NANOS_OFF+9, 10U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_HISTOGRAM_EXECLE_TXNCACHE_INSERT_DURATION_NANOS_OFF+10, 11U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_HISTOGRAM_EXECLE_TXNCACHE_INSERT_DURATION_NANOS_OFF+11, 12U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_HISTOGRAM_EXECLE_TXNCACHE_INSERT_DURATION_NANOS_OFF+12, 13U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_HISTOGRAM_EXECLE_TXNCACHE_INSERT_DURATION_NANOS_OFF+13, 14U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_HISTOGRAM_EXECLE_TXNCACHE_INSERT_DURATION_NANOS_OFF+14, 15U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_HISTOGRAM_EXECLE_TXNCACHE_INSERT_DURATION_NANOS_OFF+15, 16U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_HISTOGRAM_EXECLE_TXNCACHE_INSERT_DURATION_NANOS_OFF+16, 17U, FD_METRICS_SAMPLE_NS, NULL, 0UL }, + { 0, 0U, FD_METRICS_SAMPLE_CLOSE, NULL, 0UL }, }; static fd_metrics_sample_field_t const fields_shred[ 177 ] = { @@ -1532,7 +1573,7 @@ static fd_metrics_sample_field_t const fields_accdb[ 132 ] = { { 0, 0U, FD_METRICS_SAMPLE_CLOSE, NULL, 0UL }, }; -static fd_metrics_sample_field_t const fields_replay[ 239 ] = { +static fd_metrics_sample_field_t const fields_replay[ 250 ] = { { FD_METRICS_GAUGE_TILE_PID_OFF, 8U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, { FD_METRICS_GAUGE_TILE_TID_OFF, 9U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, { FD_METRICS_GAUGE_TILE_LAST_CPU_INDEX_OFF, 10U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, @@ -1605,6 +1646,17 @@ static fd_metrics_sample_field_t const fields_replay[ 239 ] = { { FD_METRICS_GAUGE_REPLAY_STORE_FEC_CACHE_USED_OFF, 38U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, { FD_METRICS_GAUGE_REPLAY_STORE_FEC_CACHE_CAPACITY_OFF, 39U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, { FD_METRICS_GAUGE_REPLAY_STORE_FEC_CACHE_PINNED_OFF, 40U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_COUNTER_REPLAY_TXNCACHE_PAGE_MISS_OFF, 120U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_COUNTER_REPLAY_TXNCACHE_PAGE_SPILLED_OFF, 121U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_COUNTER_REPLAY_TXNCACHE_PAGE_SPILL_READ_OFF, 122U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_COUNTER_REPLAY_TXNCACHE_FRAME_EVICTED_OFF, 123U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_COUNTER_REPLAY_TXNCACHE_SPILL_DURATION_NANOS_OFF, 124U, FD_METRICS_SAMPLE_NS, NULL, 0UL }, + { FD_METRICS_COUNTER_REPLAY_TXNCACHE_LOCK_READ_WAIT_DURATION_NANOS_OFF, 129U, FD_METRICS_SAMPLE_NS, NULL, 0UL }, + { FD_METRICS_COUNTER_REPLAY_TXNCACHE_LOCK_WRITE_WAIT_DURATION_NANOS_OFF, 130U, FD_METRICS_SAMPLE_NS, NULL, 0UL }, + { FD_METRICS_GAUGE_REPLAY_TXNCACHE_PAGE_USED_OFF, 125U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_GAUGE_REPLAY_TXNCACHE_PAGE_CAPACITY_OFF, 126U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_GAUGE_REPLAY_TXNCACHE_FRAME_USED_OFF, 127U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_GAUGE_REPLAY_TXNCACHE_FRAME_CAPACITY_OFF, 128U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, { FD_METRICS_GAUGE_REPLAY_IS_CAUGHT_UP_OFF, 41U, FD_METRICS_SAMPLE_BOOL, NULL, 0UL }, { FD_METRICS_GAUGE_REPLAY_ROOT_SLOT_OFF, 42U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, { FD_METRICS_GAUGE_REPLAY_ROOT_SLOT_DISTANCE_OFF, 43U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, @@ -3221,7 +3273,7 @@ static fd_metrics_sample_field_t const fields_ipecho[ 55 ] = { { FD_METRICS_GAUGE_IPECHO_PORT_CHECK_TCP_IN_FLIGHT_OFF, 31U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, }; -static fd_metrics_sample_field_t const fields_execrp[ 203 ] = { +static fd_metrics_sample_field_t const fields_execrp[ 244 ] = { { FD_METRICS_GAUGE_TILE_PID_OFF, 8U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, { FD_METRICS_GAUGE_TILE_TID_OFF, 9U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, { FD_METRICS_GAUGE_TILE_LAST_CPU_INDEX_OFF, 10U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, @@ -3425,6 +3477,47 @@ static fd_metrics_sample_field_t const fields_execrp[ 203 ] = { { FD_METRICS_COUNTER_EXECRP_ACCDB_DISK_WRITTEN_BYTES_OFF, 61U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, { FD_METRICS_COUNTER_EXECRP_ACCDB_DISK_WRITTEN_OFF, 62U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, { FD_METRICS_COUNTER_EXECRP_ACCDB_COPIED_BYTES_OFF, 63U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_COUNTER_EXECRP_TXNCACHE_QUERY_MISS_OFF, 65U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_COUNTER_EXECRP_TXNCACHE_INSERT_MISS_OFF, 66U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_COUNTER_EXECRP_TXNCACHE_OPERATION_STALLED_OFF, 67U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_HISTOGRAM_EXECRP_TXNCACHE_QUERY_DURATION_NANOS_OFF, 68U, FD_METRICS_SAMPLE_OPEN, NULL, 0UL }, + { FD_METRICS_HISTOGRAM_EXECRP_TXNCACHE_QUERY_DURATION_NANOS_OFF+0, 1U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_HISTOGRAM_EXECRP_TXNCACHE_QUERY_DURATION_NANOS_OFF+1, 2U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_HISTOGRAM_EXECRP_TXNCACHE_QUERY_DURATION_NANOS_OFF+2, 3U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_HISTOGRAM_EXECRP_TXNCACHE_QUERY_DURATION_NANOS_OFF+3, 4U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_HISTOGRAM_EXECRP_TXNCACHE_QUERY_DURATION_NANOS_OFF+4, 5U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_HISTOGRAM_EXECRP_TXNCACHE_QUERY_DURATION_NANOS_OFF+5, 6U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_HISTOGRAM_EXECRP_TXNCACHE_QUERY_DURATION_NANOS_OFF+6, 7U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_HISTOGRAM_EXECRP_TXNCACHE_QUERY_DURATION_NANOS_OFF+7, 8U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_HISTOGRAM_EXECRP_TXNCACHE_QUERY_DURATION_NANOS_OFF+8, 9U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_HISTOGRAM_EXECRP_TXNCACHE_QUERY_DURATION_NANOS_OFF+9, 10U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_HISTOGRAM_EXECRP_TXNCACHE_QUERY_DURATION_NANOS_OFF+10, 11U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_HISTOGRAM_EXECRP_TXNCACHE_QUERY_DURATION_NANOS_OFF+11, 12U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_HISTOGRAM_EXECRP_TXNCACHE_QUERY_DURATION_NANOS_OFF+12, 13U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_HISTOGRAM_EXECRP_TXNCACHE_QUERY_DURATION_NANOS_OFF+13, 14U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_HISTOGRAM_EXECRP_TXNCACHE_QUERY_DURATION_NANOS_OFF+14, 15U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_HISTOGRAM_EXECRP_TXNCACHE_QUERY_DURATION_NANOS_OFF+15, 16U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_HISTOGRAM_EXECRP_TXNCACHE_QUERY_DURATION_NANOS_OFF+16, 17U, FD_METRICS_SAMPLE_NS, NULL, 0UL }, + { 0, 0U, FD_METRICS_SAMPLE_CLOSE, NULL, 0UL }, + { FD_METRICS_HISTOGRAM_EXECRP_TXNCACHE_INSERT_DURATION_NANOS_OFF, 69U, FD_METRICS_SAMPLE_OPEN, NULL, 0UL }, + { FD_METRICS_HISTOGRAM_EXECRP_TXNCACHE_INSERT_DURATION_NANOS_OFF+0, 1U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_HISTOGRAM_EXECRP_TXNCACHE_INSERT_DURATION_NANOS_OFF+1, 2U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_HISTOGRAM_EXECRP_TXNCACHE_INSERT_DURATION_NANOS_OFF+2, 3U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_HISTOGRAM_EXECRP_TXNCACHE_INSERT_DURATION_NANOS_OFF+3, 4U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_HISTOGRAM_EXECRP_TXNCACHE_INSERT_DURATION_NANOS_OFF+4, 5U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_HISTOGRAM_EXECRP_TXNCACHE_INSERT_DURATION_NANOS_OFF+5, 6U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_HISTOGRAM_EXECRP_TXNCACHE_INSERT_DURATION_NANOS_OFF+6, 7U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_HISTOGRAM_EXECRP_TXNCACHE_INSERT_DURATION_NANOS_OFF+7, 8U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_HISTOGRAM_EXECRP_TXNCACHE_INSERT_DURATION_NANOS_OFF+8, 9U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_HISTOGRAM_EXECRP_TXNCACHE_INSERT_DURATION_NANOS_OFF+9, 10U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_HISTOGRAM_EXECRP_TXNCACHE_INSERT_DURATION_NANOS_OFF+10, 11U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_HISTOGRAM_EXECRP_TXNCACHE_INSERT_DURATION_NANOS_OFF+11, 12U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_HISTOGRAM_EXECRP_TXNCACHE_INSERT_DURATION_NANOS_OFF+12, 13U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_HISTOGRAM_EXECRP_TXNCACHE_INSERT_DURATION_NANOS_OFF+13, 14U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_HISTOGRAM_EXECRP_TXNCACHE_INSERT_DURATION_NANOS_OFF+14, 15U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_HISTOGRAM_EXECRP_TXNCACHE_INSERT_DURATION_NANOS_OFF+15, 16U, FD_METRICS_SAMPLE_U64, NULL, 0UL }, + { FD_METRICS_HISTOGRAM_EXECRP_TXNCACHE_INSERT_DURATION_NANOS_OFF+16, 17U, FD_METRICS_SAMPLE_NS, NULL, 0UL }, + { 0, 0U, FD_METRICS_SAMPLE_CLOSE, NULL, 0UL }, }; static fd_metrics_sample_field_t const fields_votor[ 357 ] = { @@ -4390,10 +4483,10 @@ fd_metrics_sample_schema_t const fd_metrics_sample_tiles[] = { { "dedup", 28U, fields_dedup, 43UL, 552UL }, { "resolv", 29U, fields_resolv, 81UL, 915UL }, { "pack", 30U, fields_pack, 283UL, 3013UL }, - { "execle", 31U, fields_execle, 206UL, 2171UL }, + { "execle", 31U, fields_execle, 247UL, 2599UL }, { "shred", 32U, fields_shred, 177UL, 1962UL }, { "accdb", 33U, fields_accdb, 132UL, 1361UL }, - { "replay", 34U, fields_replay, 239UL, 2568UL }, + { "replay", 34U, fields_replay, 250UL, 2700UL }, { "rotor", 35U, fields_rotor, 158UL, 1769UL }, { "rserve", 36U, fields_rserve, 145UL, 1593UL }, { "gossip", 37U, fields_gossip, 184UL, 1952UL }, @@ -4412,7 +4505,7 @@ fd_metrics_sample_schema_t const fd_metrics_sample_tiles[] = { { "waker", 50U, fields_waker, 40UL, 536UL }, { "mwaitx", 51U, fields_mwaitx, 44UL, 564UL }, { "ipecho", 52U, fields_ipecho, 55UL, 674UL }, - { "execrp", 53U, fields_execrp, 203UL, 2147UL }, + { "execrp", 53U, fields_execrp, 244UL, 2575UL }, { "votor", 54U, fields_votor, 357UL, 3719UL }, { "gui", 55U, fields_gui, 270UL, 2862UL }, { "event", 56U, fields_event, 54UL, 699UL }, diff --git a/src/disco/metrics/metrics.xml b/src/disco/metrics/metrics.xml index 7068b68659f..580758f2db9 100644 --- a/src/disco/metrics/metrics.xml +++ b/src/disco/metrics/metrics.xml @@ -783,6 +783,15 @@ deliberate exception with lint-allow="") + + + + + Duration of one transaction cache query + + + Duration of one transaction cache insert + @@ -933,6 +942,17 @@ deliberate exception with lint-allow="") + + + + + + + + + + + @@ -1750,6 +1770,15 @@ deliberate exception with lint-allow="") + + + + + Duration of one transaction cache query + + + Duration of one transaction cache insert + diff --git a/src/discof/backup/fd_snapmk_tile.c b/src/discof/backup/fd_snapmk_tile.c index 78429475a8e..a396c29f57e 100644 --- a/src/discof/backup/fd_snapmk_tile.c +++ b/src/discof/backup/fd_snapmk_tile.c @@ -338,7 +338,7 @@ populate_allowed_fds( fd_topo_t const * topo, ulong out_fds_cnt, int * out_fds ) { fd_snapmk_t * ctx = fd_topo_obj_laddr( topo, tile->tile_obj_id ); - if( FD_UNLIKELY( out_fds_cnt<5UL+(ulong)ctx->snap_max ) ) FD_LOG_ERR(( "out_fds_cnt %lu", out_fds_cnt )); + if( FD_UNLIKELY( out_fds_cnt<6UL+(ulong)ctx->snap_max ) ) FD_LOG_ERR(( "out_fds_cnt %lu", out_fds_cnt )); ulong out_cnt = 0UL; out_fds[ out_cnt++ ] = 2; /* stderr */ if( FD_LIKELY( -1!=fd_log_private_logfile_fd() ) ) @@ -346,6 +346,7 @@ populate_allowed_fds( fd_topo_t const * topo, out_fds[ out_cnt++ ] = ctx->snap_dir_fd; out_fds[ out_cnt++ ] = FD_ACCDB_FD_RO; out_fds[ out_cnt++ ] = FD_VOTE_STAKES_FD; /* vote stakes disk spill */ + out_fds[ out_cnt++ ] = FD_TXNCACHE_FD; /* transaction cache spill */ for( uint i=0U; isnap_max; i++ ) out_fds[ out_cnt++ ] = FD_SNAP_FD( i ); /* snapshot pool */ return out_cnt; @@ -363,7 +364,8 @@ populate_allowed_seccomp( fd_topo_t const * topo, (uint)ctx->snap_dir_fd, (uint)FD_SNAP_FD( 0 ), (uint)FD_SNAP_FD( ctx->snap_max-1U ), (uint)FD_ACCDB_FD_RO, - (uint)FD_VOTE_STAKES_FD ); + (uint)FD_VOTE_STAKES_FD, + (uint)FD_TXNCACHE_FD ); return sock_filter_policy_fd_snapmk_tile_instr_cnt; } @@ -415,7 +417,7 @@ unprivileged_init( fd_topo_t const * topo, fd_txncache_shmem_t * tc_shmem = fd_txncache_shmem_join( fd_topo_obj_laddr( topo, tile->snapmk.txncache_obj_id ) ); FD_TEST( tc_shmem ); - ctx->txncache = fd_txncache_join( fd_txncache_new( _txnc_lj, tc_shmem ) ); + ctx->txncache = fd_txncache_join( fd_txncache_new( _txnc_lj, tc_shmem, FD_TXNCACHE_FD ) ); FD_TEST( ctx->txncache ); ulong * zp_fseq = fd_fseq_join( fd_topo_obj_laddr( topo, tile->snapmk.zp_fseq_id ) ); FD_TEST( zp_fseq ); @@ -721,9 +723,7 @@ snapmk_status_cache_prepare( fd_snapmk_t * ctx ) { &slot_history_lamports, &slot_history_exec, slot_history_owner, ctx->raw, &slot_history_sz ); if( FD_UNLIKELY( !slot_history_lamports ) ) slot_history_sz = 0UL; - if( FD_UNLIKELY( !fd_txncache_writer_init( ctx->txncache_writer, ctx->txncache, bank->txncache_fork_id, bank->f.slot, ctx->raw, slot_history_sz, ctx->status_cache_writer_arena, ctx->status_cache_writer_arena_sz ) ) ) { - FD_LOG_CRIT(( "cannot serialize txncache fork %hu of snapshot bank at slot %lu because the root moved or SlotHistory lacks execution slots", bank->txncache_fork_id.val, bank->f.slot )); - } + fd_txncache_writer_init( ctx->txncache_writer, ctx->txncache, bank->f.slot, ctx->raw, slot_history_sz, ctx->status_cache_writer_arena, ctx->status_cache_writer_arena_sz ); ctx->status_cache_sz = 0UL; ctx->status_cache_expected_sz = fd_txncache_writer_serialized_sz( ctx->txncache_writer ); diff --git a/src/discof/backup/fd_snapmk_tile.seccomppolicy b/src/discof/backup/fd_snapmk_tile.seccomppolicy index 9f3e401271a..99c3bf3f7e3 100644 --- a/src/discof/backup/fd_snapmk_tile.seccomppolicy +++ b/src/discof/backup/fd_snapmk_tile.seccomppolicy @@ -17,7 +17,10 @@ # vote_stakes_fd: Disk backing for t-1 vote stakes sets beyond the # in-memory cache. Any tile reading a set that is not # cached loads it, writing back the evicted set. -uint logfile_fd, uint dir_fd, uint pool_min_fd, uint pool_max_fd, uint accdb_ro_fd, uint vote_stakes_fd +# +# txncache_fd: Disk backing for transaction cache pages beyond the +# in-memory cache. +uint logfile_fd, uint dir_fd, uint pool_min_fd, uint pool_max_fd, uint accdb_ro_fd, uint vote_stakes_fd, uint txncache_fd # logging: all log messages are written to a file and/or pipe # @@ -94,8 +97,12 @@ preadv2: (eq (arg 0) accdb_ro_fd) # vote stakes: load a spilled t-1 set into the cache and write back the # evicted set. -pread64: (eq (arg 0) vote_stakes_fd) -pwrite64: (eq (arg 0) vote_stakes_fd) +# +# transaction cache: read spilled pages and write back cache pages. +pread64: (or (eq (arg 0) vote_stakes_fd) + (eq (arg 0) txncache_fd)) +pwrite64: (or (eq (arg 0) vote_stakes_fd) + (eq (arg 0) txncache_fd)) # snapshot: before reading accounts, the tile parks the accdb # compaction epoch and yields while waiting for the accdb tile to diff --git a/src/discof/backup/fd_txncache_writer.c b/src/discof/backup/fd_txncache_writer.c index 4aff893be96..1a4efda0eb4 100644 --- a/src/discof/backup/fd_txncache_writer.c +++ b/src/discof/backup/fd_txncache_writer.c @@ -2,7 +2,6 @@ #include "../../flamenco/runtime/fd_txncache_private.h" #include "../../flamenco/runtime/sysvar/fd_sysvar_slot_history.h" #include "../../util/fd_util.h" -#include "../../util/racesan/fd_racesan_target.h" #define STATE_HEADER 1 #define STATE_SLOT 2 @@ -22,16 +21,9 @@ #define WALK_COUNT 0 #define WALK_FILL 1 -#define WALK_ANOMALY_NONE 0 -#define WALK_ANOMALY_CHAIN_TXN_IDX_OOB 1 -#define WALK_ANOMALY_CYCLE 2 -#define WALK_ANOMALY_CAPTURED_FORK_NO_EXEC_SLOT 3 -#define WALK_ANOMALY_GROUP_COUNT_EXCEEDED 4 - -FD_STATIC_ASSERT( FD_TXNCACHE_WRITER_CHECK_INTERVAL, check_interval_nonzero ); - FD_STATIC_ASSERT( FD_TXNCACHE_WRITER_MAX_SLOT_DELTAS<=USHORT_MAX, group_slot_i ); FD_STATIC_ASSERT( FD_TXNCACHE_WRITER_MAX_BLOCKHASHES<=USHORT_MAX, group_blockhash_i ); +FD_STATIC_ASSERT( FD_TXNCACHE_MAX_ROOTS<=FD_TXNCACHE_WRITER_MAX_BLOCKHASHES, roots ); static inline ulong group_key( ulong slot_i, ulong blockhash_i ) { @@ -104,13 +96,12 @@ writer_slots_init( fd_txncache_writer_t * writer, snapshot slot. The initial genesis descriptor may precede the named execution slots; it remains usable as a referenced blockhash but has no execution slot mapping. */ -static int +static void writer_map_execution_slots( fd_txncache_writer_t * writer ) { ulong blockhash_cnt = writer->blockhash_cnt; ulong slot_cnt = writer->slot_cnt; if( FD_UNLIKELY( blockhash_cnt>slot_cnt+1UL ) ) { - FD_LOG_WARNING(( "txncache has %lu rooted blockhash descriptors but SlotHistory names only %lu execution slots", blockhash_cnt, slot_cnt )); - return 0; + FD_LOG_CRIT(( "txncache has %lu rooted blockhash descriptors but SlotHistory names only %lu execution slots", blockhash_cnt, slot_cnt )); } memset( writer->fork_id_to_blockhash_i, 0xFF, sizeof(writer->fork_id_to_blockhash_i) ); @@ -128,166 +119,42 @@ writer_map_execution_slots( fd_txncache_writer_t * writer ) { ulong slot_i = slot_i0 +pair_i; writer->blockhash_descs[ blockhash_i ].slot_i = (ushort)slot_i; } - return 1; } -/* Walks every hash chain of a rooted blockcache without taking the - txncache lock. For each transaction that matches a captured rooted - descriptor, the function either counts the transaction into its - (execution slot, blockhash) group, or copies its hash into the - group's arena range. */ - static void writer_walk_blockhash( fd_txncache_writer_t * writer, - ulong blockhash_i, + ulong blockhash_idx, int mode ) { - fd_txncache_t const * tc = writer->tc; - fd_txncache_writer_blockhash_desc_t const * blockhash_desc = &writer->blockhash_descs[ blockhash_i ]; - - /* Loop invariant variables are hoisted because the compiler fences - would otherwise force the address chains to be reloaded every - bucket. */ - fd_txncache_shmem_t const * shmem = tc->shmem; - uint const * heads = tc->blockcache_pool[ blockhash_desc->blockcache_idx ].heads; - fd_txncache_txnpage_t const * txnpages = tc->txnpages; - ulong bucket_cnt = shmem->bucket_cnt; - ulong txn_cap = shmem->max_txnpages*FD_TXNCACHE_TXNS_PER_PAGE; - ulong root_gen_init = writer->root_gen; - fd_txnhash_t * arena = writer->arena; - ulong group_lo = writer->group_i; - ulong group_hi = writer->batch_hi; - - uint bucket_entry_cnt[ FD_TXNCACHE_WRITER_MAX_SLOT_DELTAS ] = {0}; - ushort touched_slot_i[ FD_TXNCACHE_WRITER_MAX_SLOT_DELTAS ]; - - for( ulong bucket=0UL; bucketmutation_gen, __ATOMIC_ACQUIRE ); - if( FD_UNLIKELY( gen0&1UL ) ) { - fd_racesan_hook( "txncache_writer:mutation_in_progress" ); - FD_SPIN_PAUSE(); - continue; - } - /* root_gen only changes inside a mutation bracket, so we check it - once up front. If mutation_gen checks out in the end, that - implies root_gen is also good. */ - if( FD_UNLIKELY( __atomic_load_n( &shmem->root_gen, __ATOMIC_RELAXED )!=root_gen_init ) ) { - FD_LOG_CRIT(( "txncache root advanced while snapshot of slot %lu was in progress", writer->snapshot_slot )); - } - - ulong touched_cnt = 0UL; - ulong steps = 0UL; - int anomaly = WALK_ANOMALY_NONE; - uint anomaly_txn_idx = UINT_MAX; - fd_txncache_writer_blockhash_desc_t const * anomaly_desc = NULL; - - fd_racesan_hook( "txncache_writer:bucket_started" ); - - for( uint head=__atomic_load_n( &heads[ bucket ], __ATOMIC_ACQUIRE ); head!=UINT_MAX; ) { - if( FD_UNLIKELY( (ulong)head>=txn_cap ) ) { - anomaly = WALK_ANOMALY_CHAIN_TXN_IDX_OOB; - anomaly_txn_idx = head; - break; - } - fd_txncache_single_txn_t const * txn = txnpages[ head/FD_TXNCACHE_TXNS_PER_PAGE ].txns[ head%FD_TXNCACHE_TXNS_PER_PAGE ]; - /* Pigeonhole principle. If we visited more than the max number - of entries, then at least one entry has been visited twice, - meaning a potential cycle. */ - if( FD_UNLIKELY( ++steps>txn_cap ) ) { - /* Record the anomaly rather than fatally crash immediately. - A concurrent compaction for example could legitimately send - us into a loop. We should simply retry in that case. */ - anomaly = WALK_ANOMALY_CYCLE; - break; - } - - fd_txncache_writer_blockhash_desc_t const * exec_desc = writer_exec_desc( writer, txn ); - if( FD_LIKELY( exec_desc ) ) { - ushort slot_i = exec_desc->slot_i; - if( FD_UNLIKELY( slot_i==USHORT_MAX ) ) { - anomaly = WALK_ANOMALY_CAPTURED_FORK_NO_EXEC_SLOT; - anomaly_desc = exec_desc; - break; - } - ulong key = group_key( slot_i, blockhash_i ); - if( mode==WALK_COUNT ) { - if( FD_UNLIKELY( !bucket_entry_cnt[ slot_i ] ) ) touched_slot_i[ touched_cnt++ ] = slot_i; - bucket_entry_cnt[ slot_i ]++; - } else { - ulong group_i = writer->group_i_by_key[ key ]; - if( group_i>=group_lo && group_igroups[ group_i ]; - uint entry_i = group->filled_entry_cnt+bucket_entry_cnt[ slot_i ]; - if( FD_UNLIKELY( entry_ifilled_entry_cnt || entry_i>=group->entry_cnt ) ) { - anomaly = WALK_ANOMALY_GROUP_COUNT_EXCEEDED; - break; - } - if( FD_UNLIKELY( !bucket_entry_cnt[ slot_i ] ) ) touched_slot_i[ touched_cnt++ ] = slot_i; - - /* Speculative. These txnhash values are considered - committed when entry_cnt is bumped from local - accumulators, after the final generation check. */ - memcpy( arena[ group->arena_entry_off+entry_i ], txn->txnhash, sizeof(arena[0]) ); - bucket_entry_cnt[ slot_i ]++; - fd_racesan_hook( "txncache_writer:txnhash_staged" ); - } - } - } - - if( FD_UNLIKELY( !(steps%FD_TXNCACHE_WRITER_CHECK_INTERVAL) ) ) { - FD_HW_MFENCE_LD(); - if( __atomic_load_n( &shmem->mutation_gen, __ATOMIC_RELAXED )!=gen0 ) { break; } - } - - /* Volatile: head is bounds checked and then used to index the - pool, so it must be loaded exactly once. */ - head = FD_VOLATILE_CONST( txn->blockcache_next ); - } + fd_txncache_blockcache_iter_t iter[1]; + for( fd_txncache_blockcache_iter_init( iter, writer->tc, writer->blockhash_descs[ blockhash_idx ].blockcache_idx ); + !fd_txncache_blockcache_iter_done( iter ); + fd_txncache_blockcache_iter_next( iter ) ) { + fd_txncache_single_txn_t const * txn = fd_txncache_blockcache_iter_ele( iter ); - if( FD_UNLIKELY( anomaly!=WALK_ANOMALY_NONE ) ) { fd_racesan_hook( "txncache_writer:anomaly_detected" ); } + fd_txncache_writer_blockhash_desc_t const * exec_desc = writer_exec_desc( writer, txn ); + if( FD_UNLIKELY( !exec_desc ) ) continue; - /* Drain reads before we check generation number. */ - FD_HW_MFENCE_LD(); - ulong gen1 = __atomic_load_n( &shmem->mutation_gen, __ATOMIC_RELAXED ); - - if( FD_UNLIKELY( gen1!=gen0 ) ) { - for( ulong i=0UL; iblockcache_idx, anomaly_txn_idx, txn_cap )); - case WALK_ANOMALY_CYCLE: - FD_LOG_CRIT(( "txncache chain walk of bucket %lu of blockcache %lu exceeded the pool's %lu transactions: cycle", bucket, blockhash_desc->blockcache_idx, txn_cap )); - case WALK_ANOMALY_CAPTURED_FORK_NO_EXEC_SLOT: - FD_LOG_CRIT(( "txncache transaction on captured fork %lu (descriptor %ld) has no execution slot", anomaly_desc->blockcache_idx, (long)(anomaly_desc-writer->blockhash_descs) )); - case WALK_ANOMALY_GROUP_COUNT_EXCEEDED: - FD_LOG_CRIT(( "txncache group in blockcache %lu exceeds its counted transaction count while snapshot of slot %lu is in progress", blockhash_desc->blockcache_idx, writer->snapshot_slot )); - default: - FD_LOG_CRIT(( "invalid txncache walk anomaly %d", anomaly )); - } + ushort slot_i = exec_desc->slot_i; + if( FD_UNLIKELY( slot_i==USHORT_MAX ) ) { + FD_LOG_CRIT(( "txncache transaction on captured fork %lu (descriptor %ld) has no execution slot", exec_desc->blockcache_idx, (long)(exec_desc-writer->blockhash_descs) )); + } + ulong key = group_key( slot_i, blockhash_idx ); + if( mode==WALK_COUNT ) { + writer->entry_cnt_by_key[ key ]++; + continue; + } - /* Yay! This was a successful bucket walk. Aggregate local - accumulators into writer. Then break out of the retry loop to - start the next bucket. */ - for( ulong i=0UL; ientry_cnt_by_key[ key ] += delta; - else writer->groups[ writer->group_i_by_key[ key ] ].filled_entry_cnt += delta; - bucket_entry_cnt[ slot_i ] = 0U; - } - break; + ulong group_i = writer->group_i_by_key[ key ]; + if( group_igroup_i || group_i>=writer->batch_hi ) continue; + fd_txncache_writer_group_t * group = &writer->groups[ group_i ]; + if( FD_UNLIKELY( group->filled_entry_cnt>=group->entry_cnt ) ) { + FD_LOG_CRIT(( "txncache group in blockcache %lu exceeds its counted transaction count while snapshot of slot %lu is in progress", writer->blockhash_descs[ blockhash_idx ].blockcache_idx, writer->snapshot_slot )); } + memcpy( writer->arena[ group->arena_entry_off+group->filled_entry_cnt ], txn->txnhash, sizeof(writer->arena[0]) ); + group->filled_entry_cnt++; } + fd_txncache_blockcache_iter_fini( iter ); } /* Walks every rooted blockcache once to count the transactions of each @@ -366,14 +233,11 @@ writer_fill_batch( fd_txncache_writer_t * writer ) { fd_txncache_writer_t * fd_txncache_writer_init( fd_txncache_writer_t * writer, fd_txncache_t * tc, - fd_txncache_fork_id_t fork_id, ulong snapshot_slot, uchar const * slot_history, ulong slot_history_sz, void * arena, ulong arena_sz ) { - fd_rwlock_t * lock = tc->shmem->lock; - if( FD_UNLIKELY( !arena || !fd_ulong_is_aligned( (ulong)arena, fd_txncache_writer_arena_align() ) || arena_sz<2UL*2UL*sizeof(fd_txnhash_t) ) ) { FD_LOG_CRIT(( "invalid txncache writer arena (%p, %lu bytes)", arena, arena_sz )); } @@ -393,38 +257,20 @@ fd_txncache_writer_init( fd_txncache_writer_t * writer, writer_slots_init( writer, slot_history, slot_history_sz ); - /* Snapshot the root list. The snapshot root must be the newest root. */ + /* Snapshot the root list. The snapshot root is the newest root. */ - fd_rwlock_read( lock ); - if( FD_UNLIKELY( fork_id.val>=tc->shmem->active_slots_max || - root_slist_is_empty( tc->shmem->root_ll, tc->blockcache_shmem_pool ) || - root_slist_idx_peek_tail( tc->shmem->root_ll, tc->blockcache_shmem_pool )!=fork_id.val || - tc->blockcache_shmem_pool[ fork_id.val ].frozen!=2 ) ) { - fd_rwlock_unread( lock ); - return NULL; - } - writer->root_gen = __atomic_load_n( &tc->shmem->root_gen, __ATOMIC_RELAXED ); - - ulong blockhash_cnt = 0UL; - for( ulong it = root_slist_iter_init( tc->shmem->root_ll, tc->blockcache_shmem_pool ); - !root_slist_iter_done( it, tc->shmem->root_ll, tc->blockcache_shmem_pool ); - it = root_slist_iter_next( it, tc->shmem->root_ll, tc->blockcache_shmem_pool ) ) { - ulong blockcache_idx = root_slist_iter_idx( it, tc->shmem->root_ll, tc->blockcache_shmem_pool ); - if( FD_UNLIKELY( blockhash_cnt>=FD_TXNCACHE_WRITER_MAX_BLOCKHASHES ) ) { - FD_LOG_CRIT(( "txncache has more than %lu roots", FD_TXNCACHE_WRITER_MAX_BLOCKHASHES )); - } - fd_txncache_blockcache_shmem_t const * blockcache_shmem = &tc->blockcache_shmem_pool[ blockcache_idx ]; - fd_txncache_writer_blockhash_desc_t * blockhash_desc = &writer->blockhash_descs[ blockhash_cnt ]; - blockhash_desc->blockcache_idx = blockcache_idx; - blockhash_desc->generation = blockcache_shmem->generation; - blockhash_desc->txnhash_offset = blockcache_shmem->txnhash_offset; - memcpy( blockhash_desc->blockhash, blockcache_shmem->blockhash.uc, 32UL ); - blockhash_cnt++; + fd_txncache_root_t roots[ FD_TXNCACHE_MAX_ROOTS ]; + ulong root_cnt = fd_txncache_snapshot_roots( tc, roots ); + for( ulong i=0UL; iblockhash_descs[ i ]; + blockhash_desc->blockcache_idx = roots[ i ].blockcache_idx; + blockhash_desc->generation = roots[ i ].generation; + blockhash_desc->txnhash_offset = roots[ i ].txnhash_offset; + memcpy( blockhash_desc->blockhash, roots[ i ].blockhash, 32UL ); } - fd_rwlock_unread( lock ); - writer->blockhash_cnt = blockhash_cnt; + writer->blockhash_cnt = root_cnt; - if( FD_UNLIKELY( !writer_map_execution_slots( writer ) ) ) return NULL; + writer_map_execution_slots( writer ); writer_groups_init( writer ); return writer; diff --git a/src/discof/backup/fd_txncache_writer.h b/src/discof/backup/fd_txncache_writer.h index 1d17a5b6a9c..ba6bd0b90e5 100644 --- a/src/discof/backup/fd_txncache_writer.h +++ b/src/discof/backup/fd_txncache_writer.h @@ -6,9 +6,7 @@ Vec), shaped the way Agave writes it: one slot delta per recent rooted slot with a block, holding the transactions that executed in that slot, grouped by the blockhash they referenced. - Takes the txncache read lock only during initial setup. The full - scale walks over the txncache are lock free and retry when they race - a mutation. */ + The txncache bucket iterator handles locking and paging. */ #include "../../flamenco/runtime/fd_txncache.h" #include "../../flamenco/runtime/fd_txncache_shmem.h" @@ -36,13 +34,6 @@ #define FD_TXNCACHE_WRITER_BUF_MIN (4096UL) -/* How many chain entries a walk visits between re-reads of the txncache - mutation generation. The walks hold no lock. A concurrent - compaction or root advancement is detected through the generation - number. */ - -#define FD_TXNCACHE_WRITER_CHECK_INTERVAL (1024UL) - struct fd_txncache_writer_blockhash_desc { ulong blockcache_idx; /* blockcache pool index */ uint generation; /* blockcache generation at init, detects reuse */ @@ -84,10 +75,6 @@ struct fd_txncache_writer { ulong blockhash_cnt; ushort fork_id_to_blockhash_i[ USHORT_MAX+1UL ]; - /* The txncache's root generation captured at init. Must not move - during snapshot production. */ - ulong root_gen; - /* Groups in wire order (slot ascending, then blockhash descriptor ascending). */ uint entry_cnt_by_key[ FD_TXNCACHE_WRITER_MAX_GROUPS ]; uint group_i_by_key[ FD_TXNCACHE_WRITER_MAX_GROUPS ]; @@ -131,25 +118,22 @@ FD_FN_CONST ulong fd_txncache_writer_arena_align( void ); /* fd_txncache_writer_init prepares writer to serialize the status - cache of tc as of the rooted fork fork_id, which must be the most - recent root. snapshot_slot is the fork's slot. slot_history points - to the slot_history_sz byte bincode encoding of the bank's - SlotHistory sysvar account, which decides which slots get a - slot delta and names each execution slot. arena is the arena_sz byte - staging area. + cache of tc as of its newest root. The caller must keep the root + from advancing until the writer is done. snapshot_slot is the newest + root's slot. slot_history points to the slot_history_sz byte bincode + encoding of the bank's SlotHistory sysvar account, which decides + which slots get a slot delta and names each execution slot. arena + is the arena_sz byte staging area. Walks the whole txncache once to count the transactions of every (slot, blockhash) group, so the serialized size is known on return. - Returns writer on success. Returns NULL if fork_id is not the most - recent root or if more than one rooted blockhash descriptor lacks a - named execution slot. Replay should not be advancing the root while - a snapshot is in progress, so snapmk treats NULL as fatal. */ + Returns writer. Crashes if more than one rooted blockhash + descriptor lacks a named execution slot. */ fd_txncache_writer_t * fd_txncache_writer_init( fd_txncache_writer_t * writer, fd_txncache_t * tc, - fd_txncache_fork_id_t fork_id, ulong snapshot_slot, uchar const * slot_history, ulong slot_history_sz, diff --git a/src/discof/backup/generated/fd_snapmk_tile_seccomp.h b/src/discof/backup/generated/fd_snapmk_tile_seccomp.h index f5d74d9aa98..b4a9afa7d73 100644 --- a/src/discof/backup/generated/fd_snapmk_tile_seccomp.h +++ b/src/discof/backup/generated/fd_snapmk_tile_seccomp.h @@ -33,11 +33,11 @@ #define FD_SECCOMP_ARG_LO(x) ((uint)(((ulong)(uint)(int)(x) ) & 0xffffffffUL)) #define FD_SECCOMP_ARG_HI(x) ((uint)(((ulong)(x) >> 32) & 0xffffffffUL)) -static const uint sock_filter_policy_fd_snapmk_tile_instr_cnt = 120; +static const uint sock_filter_policy_fd_snapmk_tile_instr_cnt = 124; -static void populate_sock_filter_policy_fd_snapmk_tile( ulong out_cnt, struct sock_filter out[ static 120 ], uint logfile_fd, uint dir_fd, uint pool_min_fd, uint pool_max_fd, uint accdb_ro_fd, uint vote_stakes_fd ) { - FD_TEST( out_cnt >= 120 ); - struct sock_filter filter[120] = { +static void populate_sock_filter_policy_fd_snapmk_tile( ulong out_cnt, struct sock_filter out[ static 124 ], uint logfile_fd, uint dir_fd, uint pool_min_fd, uint pool_max_fd, uint accdb_ro_fd, uint vote_stakes_fd, uint txncache_fd ) { + FD_TEST( out_cnt >= 124 ); + struct sock_filter filter[124] = { /* validate architecture */ BPF_STMT( BPF_LD | BPF_W | BPF_ABS, ( offsetof( struct seccomp_data, arch ) )), BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ARCH_NR, 0, /* RET_KILL_PROCESS */ 18 ), @@ -66,15 +66,15 @@ static void populate_sock_filter_policy_fd_snapmk_tile( ulong out_cnt, struct so /* check pread64 */ BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, SYS_pread64, /* check_pread64 */ 80, 0 ), /* check pwrite64 */ - BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, SYS_pwrite64, /* check_pwrite64 */ 83, 0 ), + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, SYS_pwrite64, /* check_pwrite64 */ 85, 0 ), /* allow sched_yield */ BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, SYS_sched_yield, /* RET_ALLOW */ 5, 0 ), /* check clock_nanosleep */ - BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, SYS_clock_nanosleep, /* check_clock_nanosleep */ 85, 0 ), + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, SYS_clock_nanosleep, /* check_clock_nanosleep */ 89, 0 ), /* check exit */ - BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, SYS_exit, /* check_exit */ 88, 0 ), + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, SYS_exit, /* check_exit */ 92, 0 ), /* check futex */ - BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, SYS_futex, /* check_futex */ 91, 0 ), + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, SYS_futex, /* check_futex */ 95, 0 ), /* allow restart_syscall */ BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, SYS_restart_syscall, /* RET_ALLOW */ 1, 0 ), // RET_KILL_PROCESS: @@ -230,7 +230,11 @@ static void populate_sock_filter_policy_fd_snapmk_tile( ulong out_cnt, struct so // check_pread64: /* arg 0 low 32 bits */ BPF_STMT( BPF_LD | BPF_W | BPF_ABS, FD_SECCOMP_ARG_LO_OFFSET(0)), - BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ((uint)(vote_stakes_fd)), /* pread64_ALLOW */ 1, /* pread64_KILL */ 0 ), + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ((uint)(vote_stakes_fd)), /* pread64_ALLOW */ 3, /* or_17 */ 0 ), +// or_17: + /* arg 0 low 32 bits */ + BPF_STMT( BPF_LD | BPF_W | BPF_ABS, FD_SECCOMP_ARG_LO_OFFSET(0)), + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ((uint)(txncache_fd)), /* pread64_ALLOW */ 1, /* pread64_KILL */ 0 ), // pread64_KILL: BPF_STMT( BPF_RET | BPF_K, SECCOMP_RET_KILL_PROCESS ), // pread64_ALLOW: @@ -238,7 +242,11 @@ static void populate_sock_filter_policy_fd_snapmk_tile( ulong out_cnt, struct so // check_pwrite64: /* arg 0 low 32 bits */ BPF_STMT( BPF_LD | BPF_W | BPF_ABS, FD_SECCOMP_ARG_LO_OFFSET(0)), - BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ((uint)(vote_stakes_fd)), /* pwrite64_ALLOW */ 1, /* pwrite64_KILL */ 0 ), + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ((uint)(vote_stakes_fd)), /* pwrite64_ALLOW */ 3, /* or_18 */ 0 ), +// or_18: + /* arg 0 low 32 bits */ + BPF_STMT( BPF_LD | BPF_W | BPF_ABS, FD_SECCOMP_ARG_LO_OFFSET(0)), + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ((uint)(txncache_fd)), /* pwrite64_ALLOW */ 1, /* pwrite64_KILL */ 0 ), // pwrite64_KILL: BPF_STMT( BPF_RET | BPF_K, SECCOMP_RET_KILL_PROCESS ), // pwrite64_ALLOW: @@ -262,16 +270,16 @@ static void populate_sock_filter_policy_fd_snapmk_tile( ulong out_cnt, struct so // check_futex: /* arg 1 low 32 bits */ BPF_STMT( BPF_LD | BPF_W | BPF_ABS, FD_SECCOMP_ARG_LO_OFFSET(1)), - BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, FD_SECCOMP_ARG_LO(FUTEX_WAIT), /* futex_ALLOW */ 7, /* or_17 */ 0 ), -// or_17: + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, FD_SECCOMP_ARG_LO(FUTEX_WAIT), /* futex_ALLOW */ 7, /* or_19 */ 0 ), +// or_19: /* arg 1 low 32 bits */ BPF_STMT( BPF_LD | BPF_W | BPF_ABS, FD_SECCOMP_ARG_LO_OFFSET(1)), - BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, FD_SECCOMP_ARG_LO(FUTEX_WAKE), /* futex_ALLOW */ 5, /* or_18 */ 0 ), -// or_18: + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, FD_SECCOMP_ARG_LO(FUTEX_WAKE), /* futex_ALLOW */ 5, /* or_20 */ 0 ), +// or_20: /* arg 1 low 32 bits */ BPF_STMT( BPF_LD | BPF_W | BPF_ABS, FD_SECCOMP_ARG_LO_OFFSET(1)), - BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, FD_SECCOMP_ARG_LO(FUTEX_WAIT_BITSET), /* and_19 */ 0, /* futex_KILL */ 2 ), -// and_19: + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, FD_SECCOMP_ARG_LO(FUTEX_WAIT_BITSET), /* and_21 */ 0, /* futex_KILL */ 2 ), +// and_21: /* arg 2 low 32 bits */ BPF_STMT( BPF_LD | BPF_W | BPF_ABS, FD_SECCOMP_ARG_LO_OFFSET(2)), BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, 0x00000000U, /* futex_ALLOW */ 1, /* futex_KILL */ 0 ), diff --git a/src/discof/backup/test_snap_roundtrip.c b/src/discof/backup/test_snap_roundtrip.c index f549ae0c2ee..f805d1c21ac 100644 --- a/src/discof/backup/test_snap_roundtrip.c +++ b/src/discof/backup/test_snap_roundtrip.c @@ -1,3 +1,4 @@ +#define _GNU_SOURCE /* O_DIRECT */ #include "fd_ssmanifest_writer.h" #include "fd_txncache_writer.h" #include "../restore/utils/fd_ssmanifest_parser.h" @@ -15,13 +16,17 @@ #include "../../util/racesan/fd_racesan.h" #endif +#include #include #include #include +#include #include #include #include +static int spill_test_fd = -1; + #define MAX_LIVE_SLOTS 16UL #define MAX_TXN_PER_SLOT 4096UL #define VALIDATOR_CNT 3UL @@ -116,14 +121,15 @@ static test_txncache_t create_txncache_sized( fd_wksp_t * wksp, ulong max_live_slots, ulong max_txn_per_slot ) { - ulong shmem_fp = fd_txncache_shmem_footprint( max_live_slots, max_txn_per_slot ); + ulong cache_footprint = spill_test_fd>=0 ? sizeof(fd_txncache_txnpage_t) : ULONG_MAX; + ulong shmem_fp = fd_txncache_shmem_footprint( max_live_slots, max_txn_per_slot, cache_footprint ); void * shmem_raw = test_alloc( wksp, fd_txncache_shmem_align(), shmem_fp ); - fd_txncache_shmem_t * shmem = fd_txncache_shmem_join( fd_txncache_shmem_new( shmem_raw, max_live_slots, max_txn_per_slot, 1UL ) ); + fd_txncache_shmem_t * shmem = fd_txncache_shmem_join( fd_txncache_shmem_new( shmem_raw, max_live_slots, max_txn_per_slot, cache_footprint, 1UL ) ); FD_TEST( shmem ); ulong ljoin_fp = fd_txncache_footprint( max_live_slots ); void * ljoin_raw = test_alloc( wksp, fd_txncache_align(), ljoin_fp ); - fd_txncache_t * tc = fd_txncache_join( fd_txncache_new( ljoin_raw, shmem ) ); + fd_txncache_t * tc = fd_txncache_join( fd_txncache_new( ljoin_raw, shmem, spill_test_fd ) ); FD_TEST( tc ); return (test_txncache_t){ .tc = tc, .shmem = shmem }; } @@ -1043,8 +1049,7 @@ expect_chain( expect_t exp[ static 6 ], #define SINGLE_BUCKET_STALE_CNT (FD_TXNCACHE_TXNS_PER_PAGE-2UL) /* A full page in one bucket: two rooted transactions separated by stale - entries. Walking it crosses the periodic generation check; inserting - another transaction triggers compaction. */ + entries. Inserting another transaction triggers compaction. */ struct single_bucket { test_txncache_t test_tc; fd_txncache_fork_id_t winner; @@ -1104,7 +1109,7 @@ single_bucket_init( single_bucket_t * sb, static fd_txncache_writer_t * single_bucket_writer_init( single_bucket_t * sb ) { - FD_TEST( fd_txncache_writer_init( sb->writer, sb->test_tc.tc, sb->winner, 1UL, sb->slot_history, FD_SYSVAR_SLOT_HISTORY_BINCODE_SZ, sb->arena, sb->arena_sz ) ); + FD_TEST( fd_txncache_writer_init( sb->writer, sb->test_tc.tc, 1UL, sb->slot_history, FD_SYSVAR_SLOT_HISTORY_BINCODE_SZ, sb->arena, sb->arena_sz ) ); return sb->writer; } @@ -1200,6 +1205,101 @@ test_txncache_writer_arena_sz( void ) { FD_TEST( fd_txncache_writer_arena_sz( 838861UL )==67108880UL ); } +/* One bucket spans two pages and one page is resident, so both walks + miss on the bucket and iterate it under the write lock. */ +struct cache_miss { + test_txncache_t test_tc; + fd_txncache_fork_id_t root; + fd_txncache_fork_id_t winner; + uchar blockhash[ 32 ]; + uchar txnhashes[ 2 ][ 32 ]; + expect_t exp[ 2 ]; + uchar * slot_history; + fd_txncache_writer_t * writer; + void * arena; + ulong arena_sz; +}; +typedef struct cache_miss cache_miss_t; + +static void +cache_miss_init( cache_miss_t * cm, + fd_wksp_t * wksp ) { + cm->test_tc = create_txncache_sized( wksp, 1UL, 64UL ); + fd_txncache_t * tc = cm->test_tc.tc; + FD_TEST( tc->shmem->frames_max==1UL ); + FD_TEST( tc->shmem->txnpages_per_blockhash_max==2UL ); + + memset( cm->blockhash, 0, 32UL ); + cm->blockhash[ 0 ] = 1; + uchar loser_hash[ 32 ] = {2}; + uchar winner_hash[ 32 ] = {3}; + cm->root = fd_txncache_attach_child( tc, NULL_FORK ); + fd_txncache_finalize_fork( tc, cm->root, 0UL, cm->blockhash ); + fd_txncache_fork_id_t loser = fd_txncache_attach_child( tc, cm->root ); + cm->winner = fd_txncache_attach_child( tc, cm->root ); + + for( ulong i=0UL; i<2UL; i++ ) { + memset( cm->txnhashes[ i ], 0, 32UL ); + cm->txnhashes[ i ][ 0 ] = 1; + cm->txnhashes[ i ][ 8 ] = 0xA5; + cm->txnhashes[ i ][ 9 ] = (uchar)i; + memcpy( cm->exp[ i ].txnhash, cm->txnhashes[ i ], 20UL ); + memcpy( cm->exp[ i ].blockhash, cm->blockhash, 32UL ); + cm->exp[ i ].exec_slot = 1UL; + cm->exp[ i ].blockhash_i = 0UL; + } + + fd_txncache_insert( tc, cm->winner, cm->blockhash, cm->txnhashes[ 0 ] ); + for( ulong i=0UL; iblockhash, txnhash ); + } + fd_txncache_insert( tc, cm->winner, cm->blockhash, cm->txnhashes[ 1 ] ); + fd_txncache_finalize_fork( tc, loser, 0UL, loser_hash ); + fd_txncache_finalize_fork( tc, cm->winner, 0UL, winner_hash ); + fd_txncache_advance_root( tc, cm->winner ); + + cm->slot_history = test_alloc( wksp, alignof(uchar), FD_SYSVAR_SLOT_HISTORY_BINCODE_SZ ); + mock_slot_history( cm->slot_history, 1UL, 1UL, NULL, 0UL ); + cm->writer = new_writer( wksp ); + cm->arena = new_arena( wksp, 4UL, &cm->arena_sz ); +} + +static void +cache_miss_writer_init( cache_miss_t * cm ) { + FD_TEST( fd_txncache_writer_init( cm->writer, cm->test_tc.tc, 1UL, cm->slot_history, FD_SYSVAR_SLOT_HISTORY_BINCODE_SZ, cm->arena, cm->arena_sz ) ); + FD_TEST( fd_txncache_writer_serialized_sz( cm->writer )==121UL ); +} + +/* Restores the head page before the fill walk. */ +static void +cache_miss_restore_head( cache_miss_t * cm ) { + FD_TEST( fd_txncache_query( cm->test_tc.tc, cm->winner, cm->blockhash, cm->txnhashes[ 1 ] ) ); +} + +static void +cache_miss_check( cache_miss_t * cm, + fd_wksp_t * wksp, + uchar const * buf, + ulong sz ) { + FD_TEST( sz==121UL ); + FD_TEST( parse_and_check( wksp, buf, sz, cm->exp, 2UL, 1UL, 0UL, NULL )==1UL ); +} + +static void +test_txncache_writer_cache_miss( fd_wksp_t * wksp ) { + FD_LOG_NOTICE(( "test_txncache_writer_cache_miss" )); + cache_miss_t cm[ 1 ]; + cache_miss_init( cm, wksp ); + cache_miss_writer_init( cm ); + cache_miss_restore_head( cm ); + ulong sz; + uchar * buf = serialize_all( wksp, cm->writer, FD_TXNCACHE_WRITER_BUF_MIN, &sz ); + cache_miss_check( cm, wksp, buf, sz ); +} + static void test_txncache_roundtrip_empty( fd_wksp_t * wksp ) { FD_LOG_NOTICE(( "test_txncache_roundtrip_empty" )); @@ -1217,7 +1317,7 @@ test_txncache_roundtrip_empty( fd_wksp_t * wksp ) { fd_txncache_writer_t * writer = new_writer( wksp ); ulong arena_sz; void * arena = new_arena( wksp, 4UL, &arena_sz ); - FD_TEST( fd_txncache_writer_init( writer, tc, root, ROOT_SLOT, slot_history, FD_SYSVAR_SLOT_HISTORY_BINCODE_SZ, arena, arena_sz ) ); + FD_TEST( fd_txncache_writer_init( writer, tc, ROOT_SLOT, slot_history, FD_SYSVAR_SLOT_HISTORY_BINCODE_SZ, arena, arena_sz ) ); FD_TEST( fd_txncache_writer_serialized_sz( writer )==25UL ); ulong total_written; @@ -1261,7 +1361,7 @@ test_txncache_roundtrip_genesis_blockhash( fd_wksp_t * wksp ) { fd_txncache_writer_t * writer = new_writer( wksp ); ulong arena_sz; void * arena = new_arena( wksp, 4UL, &arena_sz ); - FD_TEST( fd_txncache_writer_init( writer, tc, child, snapshot_slot, slot_history, FD_SYSVAR_SLOT_HISTORY_BINCODE_SZ, arena, arena_sz ) ); + FD_TEST( fd_txncache_writer_init( writer, tc, snapshot_slot, slot_history, FD_SYSVAR_SLOT_HISTORY_BINCODE_SZ, arena, arena_sz ) ); FD_TEST( fd_txncache_writer_serialized_sz( writer )==97UL ); ulong total_written; @@ -1291,7 +1391,7 @@ test_txncache_roundtrip( fd_wksp_t * wksp ) { uchar blockhashes[4][32]; uchar txnhashes[6][20]; - fd_txncache_fork_id_t root = populate_txncache( tc, blockhashes, txnhashes ); + populate_txncache( tc, blockhashes, txnhashes ); expect_t exp[ 6 ]; expect_chain( exp, blockhashes, txnhashes ); @@ -1303,7 +1403,7 @@ test_txncache_roundtrip( fd_wksp_t * wksp ) { FD_TEST( arena_sz>=4UL*MAX_TXN_PER_SLOT*20UL ); void * arena = test_alloc( wksp, fd_txncache_writer_arena_align(), arena_sz ); - FD_TEST( fd_txncache_writer_init( writer, tc, root, ROOT_SLOT, slot_history, FD_SYSVAR_SLOT_HISTORY_BINCODE_SZ, arena, arena_sz ) ); + FD_TEST( fd_txncache_writer_init( writer, tc, ROOT_SLOT, slot_history, FD_SYSVAR_SLOT_HISTORY_BINCODE_SZ, arena, arena_sz ) ); /* 1 slot_deltas_len + 4 slot deltas + 3 groups + 6 txns */ ulong expected_sz = 8UL + 4UL*17UL + 3UL*48UL + 6UL*24UL; FD_TEST( fd_txncache_writer_serialized_sz( writer )==expected_sz ); @@ -1315,7 +1415,7 @@ test_txncache_roundtrip( fd_wksp_t * wksp ) { FD_TEST( parse_and_check( wksp, buf, total_written, exp, 6UL, 4UL, 0UL, NULL )==3UL ); /* The output does not depend on the chunking. */ - FD_TEST( fd_txncache_writer_init( writer, tc, root, ROOT_SLOT, slot_history, FD_SYSVAR_SLOT_HISTORY_BINCODE_SZ, arena, arena_sz ) ); + FD_TEST( fd_txncache_writer_init( writer, tc, ROOT_SLOT, slot_history, FD_SYSVAR_SLOT_HISTORY_BINCODE_SZ, arena, arena_sz ) ); ulong total2; uchar * buf2 = serialize_all( wksp, writer, 1UL<<20, &total2 ); FD_TEST( total2==total_written && !memcmp( buf, buf2, total_written ) ); @@ -1337,7 +1437,7 @@ test_txncache_roundtrip_slot_history( fd_wksp_t * wksp ) { uchar blockhashes[4][32]; uchar txnhashes[6][20]; - fd_txncache_fork_id_t root = populate_txncache( tc, blockhashes, txnhashes ); + populate_txncache( tc, blockhashes, txnhashes ); expect_t exp[ 6 ]; expect_chain( exp, blockhashes, txnhashes ); @@ -1355,7 +1455,7 @@ test_txncache_roundtrip_slot_history( fd_wksp_t * wksp ) { fd_txncache_writer_t * writer = new_writer( wksp ); ulong arena_sz = fd_txncache_writer_arena_sz( MAX_TXN_PER_SLOT ); void * arena = test_alloc( wksp, fd_txncache_writer_arena_align(), arena_sz ); - FD_TEST( fd_txncache_writer_init( writer, tc, root, ROOT_SLOT, slot_history, FD_SYSVAR_SLOT_HISTORY_BINCODE_SZ, arena, arena_sz ) ); + FD_TEST( fd_txncache_writer_init( writer, tc, ROOT_SLOT, slot_history, FD_SYSVAR_SLOT_HISTORY_BINCODE_SZ, arena, arena_sz ) ); ulong total_written; uchar * buf = serialize_all( wksp, writer, FD_TXNCACHE_WRITER_BUF_MIN, &total_written ); FD_TEST( total_written==8UL + FD_TXNCACHE_WRITER_MAX_SLOT_DELTAS*17UL + 3UL*48UL + 6UL*24UL ); @@ -1369,7 +1469,7 @@ test_txncache_roundtrip_slot_history( fd_wksp_t * wksp ) { writer must force the snapshot slot back in. */ mock_slot_history( slot_history, 0UL, ROOT_SLOT-1UL, skipped, skipped_cnt ); - FD_TEST( fd_txncache_writer_init( writer, tc, root, ROOT_SLOT, slot_history, FD_SYSVAR_SLOT_HISTORY_BINCODE_SZ, arena, arena_sz ) ); + FD_TEST( fd_txncache_writer_init( writer, tc, ROOT_SLOT, slot_history, FD_SYSVAR_SLOT_HISTORY_BINCODE_SZ, arena, arena_sz ) ); ulong stale_written; uchar * stale_buf = serialize_all( wksp, writer, FD_TXNCACHE_WRITER_BUF_MIN, &stale_written ); FD_TEST( stale_written==total_written && !memcmp( stale_buf, buf, total_written ) ); @@ -1389,13 +1489,6 @@ test_txncache_roundtrip_slot_history( fd_wksp_t * wksp ) { FD_TEST( !slot_set_ele_query( slot_set.map, &skipped[ 1 ], NULL, slot_set.pool ) ); FD_TEST( !slot_set_ele_query( slot_set.map, &before, NULL, slot_set.pool ) ); - /* Any fork but the newest root means replay moved on from the bank - being snapshotted, which the writer must refuse. */ - - fd_txncache_fork_id_t stale = { .val = (ushort)(root.val+1U) }; - FD_TEST( !fd_txncache_writer_init( writer, tc, stale, ROOT_SLOT, slot_history, FD_SYSVAR_SLOT_HISTORY_BINCODE_SZ, arena, arena_sz ) ); - FD_TEST( !fd_txncache_writer_init( writer, tc, NULL_FORK, ROOT_SLOT, slot_history, FD_SYSVAR_SLOT_HISTORY_BINCODE_SZ, arena, arena_sz ) ); - fd_wksp_free_laddr( fd_slot_delta_parser_delete( fd_slot_delta_parser_leave( parser ) ) ); } @@ -1453,7 +1546,7 @@ test_txncache_roundtrip_large( fd_wksp_t * wksp ) { fd_txncache_writer_t * writer = new_writer( wksp ); ulong arena_sz = fd_txncache_writer_arena_sz( MAX_TXN_PER_SLOT ); void * arena = test_alloc( wksp, fd_txncache_writer_arena_align(), arena_sz ); - FD_TEST( fd_txncache_writer_init( writer, tc, fork, ROOT_SLOT, slot_history, FD_SYSVAR_SLOT_HISTORY_BINCODE_SZ, arena, arena_sz ) ); + FD_TEST( fd_txncache_writer_init( writer, tc, ROOT_SLOT, slot_history, FD_SYSVAR_SLOT_HISTORY_BINCODE_SZ, arena, arena_sz ) ); ulong expected_sz = 8UL + LARGE_SLOT_CNT*17UL + group_cnt*48UL + LARGE_EXP_CNT*24UL; FD_TEST( fd_txncache_writer_serialized_sz( writer )==expected_sz ); ulong total_written; @@ -1463,14 +1556,14 @@ test_txncache_roundtrip_large( fd_wksp_t * wksp ) { ulong small_sz; void * small = new_arena( wksp, LARGE_NEAR_CNT+LARGE_FAR_CNT, &small_sz ); - FD_TEST( fd_txncache_writer_init( writer, tc, fork, ROOT_SLOT, slot_history, FD_SYSVAR_SLOT_HISTORY_BINCODE_SZ, small, small_sz ) ); + FD_TEST( fd_txncache_writer_init( writer, tc, ROOT_SLOT, slot_history, FD_SYSVAR_SLOT_HISTORY_BINCODE_SZ, small, small_sz ) ); ulong total_small; uchar * buf_small = serialize_all( wksp, writer, 1UL<<20, &total_small ); FD_TEST( total_small==total_written && !memcmp( buf, buf_small, total_written ) ); ulong exact_sz; void * exact = new_arena( wksp, LARGE_NEAR_CNT, &exact_sz ); - FD_TEST( fd_txncache_writer_init( writer, tc, fork, ROOT_SLOT, slot_history, FD_SYSVAR_SLOT_HISTORY_BINCODE_SZ, exact, exact_sz ) ); + FD_TEST( fd_txncache_writer_init( writer, tc, ROOT_SLOT, slot_history, FD_SYSVAR_SLOT_HISTORY_BINCODE_SZ, exact, exact_sz ) ); ulong total_exact; uchar * buf_exact = serialize_all( wksp, writer, FD_TXNCACHE_WRITER_BUF_MIN, &total_exact ); FD_TEST( total_exact==total_written && !memcmp( buf, buf_exact, total_written ) ); @@ -1519,7 +1612,7 @@ test_txncache_roundtrip_many_roots( fd_wksp_t * wksp ) { /* Serialize the steady state with a large chunk and parse it back. */ mock_slot_history( slot_history, 0UL, slot, NULL, 0UL ); - FD_TEST( fd_txncache_writer_init( writer, tc, fork, slot, slot_history, FD_SYSVAR_SLOT_HISTORY_BINCODE_SZ, arena, arena_sz ) ); + FD_TEST( fd_txncache_writer_init( writer, tc, slot, slot_history, FD_SYSVAR_SLOT_HISTORY_BINCODE_SZ, arena, arena_sz ) ); ulong expected_sz = 8UL + 300UL*17UL + 48UL + hold_cnt*24UL; FD_TEST( fd_txncache_writer_serialized_sz( writer )==expected_sz ); ulong total_written; @@ -1561,72 +1654,6 @@ test_txncache_roundtrip_many_roots( fd_wksp_t * wksp ) { fd_wksp_free_laddr( fd_slot_delta_parser_delete( fd_slot_delta_parser_leave( parser ) ) ); } -static void -test_txncache_writer_rejects_excess_descriptors( fd_wksp_t * wksp ) { - FD_LOG_NOTICE(( "test_txncache_writer_rejects_excess_descriptors" )); - - test_txncache_t test_tc = create_txncache( wksp ); - fd_txncache_t * tc = test_tc.tc; - - uchar blockhashes[ 3UL ][ 32UL ] = {{0}}; - for( ulong i=0UL; i<3UL; i++ ) blockhashes[ i ][ 0 ] = (uchar)( 0xA0UL+i ); - - fd_txncache_fork_id_t fork = fd_txncache_attach_child( tc, NULL_FORK ); - fd_txncache_finalize_fork( tc, fork, 0UL, blockhashes[ 0UL ] ); - for( ulong i=1UL; i<3UL; i++ ) { - fd_txncache_fork_id_t child = fd_txncache_attach_child( tc, fork ); - fd_txncache_finalize_fork( tc, child, 0UL, blockhashes[ i ] ); - fd_txncache_advance_root( tc, child ); - fork = child; - } - - static ulong const snapshot_slot = 2UL; - uchar * slot_history = test_alloc( wksp, alignof(uchar), FD_SYSVAR_SLOT_HISTORY_BINCODE_SZ ); - mock_slot_history( slot_history, snapshot_slot, snapshot_slot, NULL, 0UL ); - - fd_txncache_writer_t * writer = new_writer( wksp ); - ulong arena_sz; - void * arena = new_arena( wksp, 4UL, &arena_sz ); - FD_TEST( !fd_txncache_writer_init( writer, tc, fork, snapshot_slot, slot_history, FD_SYSVAR_SLOT_HISTORY_BINCODE_SZ, arena, arena_sz ) ); -} - -static void -test_txncache_writer_root_advance_is_fatal( fd_wksp_t * wksp ) { - FD_LOG_NOTICE(( "test_txncache_writer_root_advance_is_fatal" )); - single_bucket_t fixture[ 1 ]; - single_bucket_init( fixture, wksp ); - fd_txncache_t * tc = fixture->test_tc.tc; - fd_txncache_writer_t * writer = single_bucket_writer_init( fixture ); - uchar next_hash[ 32 ] = {4}; - fd_txncache_fork_id_t next = fd_txncache_attach_child( tc, fixture->winner ); - fd_txncache_finalize_fork( tc, next, 0UL, next_hash ); - fd_txncache_advance_root( tc, next ); - - FILE * diagnostic = tmpfile(); - FD_TEST( diagnostic ); - pid_t pid = fork(); - FD_TEST( pid>=0 ); - if( pid==0 ) { - FD_TEST( dup2( fileno( diagnostic ), STDERR_FILENO )==STDERR_FILENO ); - fd_log_level_logfile_set( 5 ); - fd_log_level_stderr_set( 5 ); - fd_log_level_core_set( 6 ); - uchar buf[ FD_TXNCACHE_WRITER_BUF_MIN ]; - while( fd_txncache_writer_serialize( writer, buf, sizeof(buf) ) ) {} - _exit( 0 ); - } - int status = 0; - FD_TEST( waitpid( pid, &status, 0 )==pid ); - FD_TEST( WIFEXITED( status ) && WEXITSTATUS( status )==1 ); - rewind( diagnostic ); - char message[ 1024 ]; - ulong message_sz = fread( message, 1UL, sizeof(message)-1UL, diagnostic ); - FD_TEST( !ferror( diagnostic ) ); - message[ message_sz ] = '\0'; - FD_TEST( strstr( message, "txncache root advanced while snapshot of slot 1 was in progress" ) ); - FD_TEST( !fclose( diagnostic ) ); -} - #define STRESS_CANCEL_CYCLES (1024UL) /* Repeatedly insert transactions from a temporary child into the @@ -1674,7 +1701,7 @@ stress_txncache_writer_concurrent_cancels( fd_wksp_t * wksp ) { ulong arena_sz; void * arena = new_arena( wksp, 64UL, &arena_sz ); - FD_TEST( fd_txncache_writer_init( writer, tc, root, ROOT_SLOT, slot_history, FD_SYSVAR_SLOT_HISTORY_BINCODE_SZ, arena, arena_sz ) ); + FD_TEST( fd_txncache_writer_init( writer, tc, ROOT_SLOT, slot_history, FD_SYSVAR_SLOT_HISTORY_BINCODE_SZ, arena, arena_sz ) ); ulong base_sz; uchar * base = serialize_all( wksp, writer, FD_TXNCACHE_WRITER_BUF_MIN, &base_sz ); struct cancel_mutator m = { .tc = tc, .root = root, .cancel_cnt = 0UL }; @@ -1684,14 +1711,19 @@ stress_txncache_writer_concurrent_cancels( fd_wksp_t * wksp ) { while( !atomic_load_explicit( &m.cancel_cnt, memory_order_acquire ) ) FD_YIELD(); ulong iter_cnt = 0UL; + ulong cancel_cnt; do { - FD_TEST( fd_txncache_writer_init( writer, tc, root, ROOT_SLOT, slot_history, FD_SYSVAR_SLOT_HISTORY_BINCODE_SZ, arena, arena_sz ) ); + ulong cancel_cnt0 = atomic_load_explicit( &m.cancel_cnt, memory_order_acquire ); + FD_TEST( fd_txncache_writer_init( writer, tc, ROOT_SLOT, slot_history, FD_SYSVAR_SLOT_HISTORY_BINCODE_SZ, arena, arena_sz ) ); ulong sz; uchar * buf = serialize_all( wksp, writer, FD_TXNCACHE_WRITER_BUF_MIN, &sz ); FD_TEST( sz==base_sz && !memcmp( buf, base, sz ) ); fd_wksp_free_laddr( buf ); iter_cnt++; - } while( atomic_load_explicit( &m.cancel_cnt, memory_order_acquire )test_tc.shmem->mutation_gen; - struct compact_mutator mutator = { .tc = sb->test_tc.tc, .root = sb->winner, .cycle_cnt = 0UL }; memcpy( mutator.blockhash, sb->blockhash, 32UL ); int log_level_stderr = fd_log_level_stderr(); @@ -1756,7 +1786,6 @@ stress_txncache_writer_concurrent_compaction( fd_wksp_t * wksp ) { FD_TEST( !pthread_join( thread, NULL ) ); fd_log_level_stderr_set( log_level_stderr ); - FD_TEST( sb->test_tc.shmem->mutation_gen==mutation_gen+2UL*STRESS_COMPACT_CYCLES ); FD_LOG_NOTICE(( "concurrency smoke test: %lu serializations, %lu compactions (overlap not guaranteed)", iter_cnt, STRESS_COMPACT_CYCLES )); } @@ -1782,9 +1811,9 @@ inject_cancel( void * ctx, fd_txncache_cancel_fork( inject->tc, child ); } -/* Force an unrooted cancellation between a bucket's generation reads. - Since cancellation does not change the rooted transaction set, the - writer must finish even when this happens at every retry point. */ +/* Force an unrooted cancellation before every bucket walk. Since + cancellation does not change the rooted transaction set, the output + must not change. */ static void test_txncache_writer_ignores_unrooted_cancellation( fd_wksp_t * wksp ) { FD_LOG_NOTICE(( "test_txncache_writer_ignores_unrooted_cancellation" )); @@ -1813,9 +1842,9 @@ test_txncache_writer_ignores_unrooted_cancellation( fd_wksp_t * wksp ) { fd_racesan_t racesan[ 1 ]; FD_TEST( fd_racesan_new( racesan, &inject ) ); - fd_racesan_inject( racesan, "txncache_writer:bucket_started", inject_cancel ); + fd_racesan_inject( racesan, "txncache:bucket_started", inject_cancel ); FD_RACESAN_INJECT_BEGIN( racesan ) { - FD_TEST( fd_txncache_writer_init( writer, tc, root, ROOT_SLOT, slot_history, FD_SYSVAR_SLOT_HISTORY_BINCODE_SZ, arena, arena_sz ) ); + FD_TEST( fd_txncache_writer_init( writer, tc, ROOT_SLOT, slot_history, FD_SYSVAR_SLOT_HISTORY_BINCODE_SZ, arena, arena_sz ) ); } FD_RACESAN_INJECT_END; FD_TEST( inject.cancel_cnt ); @@ -1826,294 +1855,58 @@ test_txncache_writer_ignores_unrooted_cancellation( fd_wksp_t * wksp ) { fd_racesan_delete( racesan ); } -struct spin_inject { - fd_txncache_shmem_t * shmem; - int begun; - int ended; -}; - -static void -inject_spin_begin( void * ctx, - ulong name_hash ) { - (void)name_hash; - struct spin_inject * inject = ctx; - if( inject->begun ) return; - inject->begun = 1; - fd_txncache_mutation_begin( inject->shmem ); -} - -static void -inject_spin_end( void * ctx, - ulong name_hash ) { - (void)name_hash; - struct spin_inject * inject = ctx; - FD_TEST( inject->begun && !inject->ended ); - inject->ended = 1; - fd_txncache_mutation_end( inject->shmem ); -} - -/* A mutation begins right after a walk attempt read an even - generation. That attempt must fail its final check, and the retry - must wait for the mutation to end (odd generation) instead of - reading the storage. */ -static void -test_txncache_writer_waits_for_mutation_end( fd_wksp_t * wksp ) { - FD_LOG_NOTICE(( "test_txncache_writer_waits_for_mutation_end" )); - test_txncache_t test_tc = create_txncache( wksp ); - fd_txncache_t * tc = test_tc.tc; - - uchar blockhashes[ 4 ][ 32 ]; - uchar txnhashes [ 6 ][ 20 ]; - fd_txncache_fork_id_t root = populate_txncache( tc, blockhashes, txnhashes ); - expect_t exp[ 6 ]; - expect_chain( exp, blockhashes, txnhashes ); - - uchar * slot_history = test_alloc( wksp, alignof(uchar), FD_SYSVAR_SLOT_HISTORY_BINCODE_SZ ); - mock_slot_history( slot_history, ROOT_SLOT-3UL, ROOT_SLOT, NULL, 0UL ); - fd_txncache_writer_t * writer = new_writer( wksp ); - ulong arena_sz; - void * arena = new_arena( wksp, 64UL, &arena_sz ); - - struct spin_inject inject = { .shmem = test_tc.shmem }; - fd_racesan_t racesan[ 1 ]; - FD_TEST( fd_racesan_new( racesan, &inject ) ); - fd_racesan_inject( racesan, "txncache_writer:bucket_started", inject_spin_begin ); - fd_racesan_inject( racesan, "txncache_writer:mutation_in_progress", inject_spin_end ); - ulong mutation_gen = test_tc.shmem->mutation_gen; - FD_RACESAN_INJECT_BEGIN( racesan ) { - FD_TEST( fd_txncache_writer_init( writer, tc, root, ROOT_SLOT, slot_history, FD_SYSVAR_SLOT_HISTORY_BINCODE_SZ, arena, arena_sz ) ); - } FD_RACESAN_INJECT_END; - FD_TEST( inject.begun && inject.ended ); - FD_TEST( test_tc.shmem->mutation_gen==mutation_gen+2UL ); - - ulong sz; - uchar * buf = serialize_all( wksp, writer, FD_TXNCACHE_WRITER_BUF_MIN, &sz ); - FD_TEST( parse_and_check( wksp, buf, sz, exp, 6UL, 4UL, 0UL, NULL )==3UL ); - - fd_racesan_delete( racesan ); -} - -struct compact_inject { +struct miss_compact_inject { fd_txncache_t * tc; fd_txncache_fork_id_t fork; uchar blockhash[ 32 ]; - uchar txnhash[ 32 ]; - int fired; + ulong miss_cnt; }; +/* At the first miss, fills the rooted blockcache from an unrooted child + until the next insert compacts it, which moves the rooted + transactions. */ static void -inject_compaction( void * ctx, - ulong name_hash ) { +inject_miss_compaction( void * ctx, + ulong name_hash ) { (void)name_hash; - struct compact_inject * inject = ctx; - if( inject->fired ) return; - inject->fired = 1; - fd_txncache_insert( inject->tc, inject->fork, inject->blockhash, inject->txnhash ); + struct miss_compact_inject * inject = ctx; + if( inject->miss_cnt++ ) return; + for( ulong i=0UL; itc, inject->fork, inject->blockhash, txnhash ); + } } +/* The bucket is compacted between the two locks of the first miss, so + the iterator must reload the chain head under the write lock. */ static void -test_txncache_writer_retries_compaction( fd_wksp_t * wksp ) { - FD_LOG_NOTICE(( "test_txncache_writer_retries_compaction" )); - single_bucket_t sb[ 1 ]; - single_bucket_init( sb, wksp ); - fd_txncache_t * tc = sb->test_tc.tc; - fd_txncache_fork_id_t child = fd_txncache_attach_child( tc, sb->winner ); - fd_txncache_writer_t * writer = single_bucket_writer_init( sb ); - - struct compact_inject inject = { - .tc = tc, - .fork = child, - .fired = 0 - }; - memcpy( inject.blockhash, sb->blockhash, 32UL ); - memset( inject.txnhash, 0xEE, 32UL ); - +test_txncache_writer_cache_miss_compaction( fd_wksp_t * wksp ) { + FD_LOG_NOTICE(( "test_txncache_writer_cache_miss_compaction" )); + cache_miss_t cm[ 1 ]; + cache_miss_init( cm, wksp ); + fd_txncache_t * tc = cm->test_tc.tc; + + struct miss_compact_inject inject = { .tc = tc, .fork = fd_txncache_attach_child( tc, cm->winner ) }; + memcpy( inject.blockhash, cm->blockhash, 32UL ); fd_racesan_t racesan[ 1 ]; FD_TEST( fd_racesan_new( racesan, &inject ) ); - fd_racesan_inject( racesan, "txncache_writer:txnhash_staged", inject_compaction ); - ulong mutation_gen = sb->test_tc.shmem->mutation_gen; - ulong sz = 0UL; - uchar * buf = NULL; + fd_racesan_inject( racesan, "txncache:bucket_iter_miss", inject_miss_compaction ); FD_RACESAN_INJECT_BEGIN( racesan ) { - buf = serialize_all( wksp, writer, FD_TXNCACHE_WRITER_BUF_MIN, &sz ); + cache_miss_writer_init( cm ); } FD_RACESAN_INJECT_END; - FD_TEST( inject.fired ); - FD_TEST( sb->test_tc.shmem->mutation_gen==mutation_gen+2UL ); - FD_TEST( parse_and_check( wksp, buf, sz, sb->exp, 2UL, 1UL, 0UL, NULL )==1UL ); - - fd_racesan_delete( racesan ); -} - -struct bracket_inject { - fd_txncache_shmem_t * shmem; - ulong attempt_cnt; - ulong staged_cnt; - ulong bracket_max; - int bracketed; -}; - -static void -inject_bracket_attempt( void * ctx, - ulong name_hash ) { - (void)name_hash; - struct bracket_inject * inject = ctx; - inject->attempt_cnt++; - inject->bracketed = 0; -} + FD_TEST( inject.miss_cnt==1UL ); + FD_TEST( tc->blockcache_pool[ cm->root.val ].shmem->pages_cnt==2UL ); -/* Brackets an empty mutation at the first staged transaction of each - of the first bracket_max walk attempts, as a compaction of some other - blockcache would. */ -static void -inject_bracket_staged( void * ctx, - ulong name_hash ) { - (void)name_hash; - struct bracket_inject * inject = ctx; - inject->staged_cnt++; - if( inject->bracketed || inject->attempt_cnt>inject->bracket_max ) return; - inject->bracketed = 1; - fd_txncache_mutation_begin( inject->shmem ); - fd_txncache_mutation_end ( inject->shmem ); -} - -/* Each interrupted attempt stages only the head transaction before the - periodic check aborts the walk. The successful retry stages both. */ -static void -test_txncache_writer_retries_at_periodic_check( fd_wksp_t * wksp ) { - FD_LOG_NOTICE(( "test_txncache_writer_retries_at_periodic_check" )); - single_bucket_t sb[ 1 ]; - single_bucket_init( sb, wksp ); - fd_txncache_writer_t * writer = single_bucket_writer_init( sb ); - - ulong const bracket_max = 3UL; - struct bracket_inject inject = { .shmem = sb->test_tc.shmem, .bracket_max = bracket_max }; - fd_racesan_t racesan[ 1 ]; - FD_TEST( fd_racesan_new( racesan, &inject ) ); - fd_racesan_inject( racesan, "txncache_writer:bucket_started", inject_bracket_attempt ); - fd_racesan_inject( racesan, "txncache_writer:txnhash_staged", inject_bracket_staged ); - ulong mutation_gen = sb->test_tc.shmem->mutation_gen; - ulong sz = 0UL; - uchar * buf = NULL; + cache_miss_restore_head( cm ); + ulong sz = 0UL; + uchar * buf = NULL; FD_RACESAN_INJECT_BEGIN( racesan ) { - buf = serialize_all( wksp, writer, FD_TXNCACHE_WRITER_BUF_MIN, &sz ); + buf = serialize_all( wksp, cm->writer, FD_TXNCACHE_WRITER_BUF_MIN, &sz ); } FD_RACESAN_INJECT_END; - - FD_TEST( inject.attempt_cnt==bracket_max+1UL ); - FD_TEST( inject.staged_cnt ==bracket_max+2UL ); - FD_TEST( sb->test_tc.shmem->mutation_gen==mutation_gen+2UL*bracket_max ); - FD_TEST( parse_and_check( wksp, buf, sz, sb->exp, 2UL, 1UL, 0UL, NULL )==1UL ); - - fd_racesan_delete( racesan ); -} - -struct cycle_inject { - fd_txncache_shmem_t * shmem; - fd_txncache_single_txn_t * txn; - uint head; - uint next; - int installed; - int repaired; - ulong staged_cnt; -}; - -static void -inject_cycle( void * ctx, - ulong name_hash ) { - (void)name_hash; - struct cycle_inject * inject = ctx; - if( inject->installed ) return; - inject->txn->blockcache_next = inject->head; - inject->installed = 1; -} - -static void -count_staged( void * ctx, - ulong name_hash ) { - (void)name_hash; - struct cycle_inject * inject = ctx; - inject->staged_cnt++; -} - -static void -repair_cycle( void * ctx, - ulong name_hash ) { - (void)name_hash; - struct cycle_inject * inject = ctx; - FD_TEST( inject->installed && !inject->repaired ); - fd_txncache_mutation_begin( inject->shmem ); - inject->txn->blockcache_next = inject->next; - fd_txncache_mutation_end( inject->shmem ); - inject->repaired = 1; -} - -/* Count detects the self-loop at the chain-length bound; fill detects it - at the group-size bound. Both must retry when the generation changes. */ -static void -test_txncache_writer_retries_transient_cycle( fd_wksp_t * wksp, - int during_fill ) { - FD_LOG_NOTICE(( "test_txncache_writer_retries_transient_cycle (%s)", during_fill ? "fill" : "count" )); - test_txncache_t test_tc = create_txncache_sized( wksp, 1UL, 1UL ); - fd_txncache_t * tc = test_tc.tc; - - uchar blockhash[ 32 ] = {1}; - uchar winner_hash[ 32 ] = {2}; - uchar txnhash[ 32 ] = {0}; - txnhash[ 0 ] = 7U; - txnhash[ 8 ] = 0xA7; - fd_txncache_fork_id_t root = fd_txncache_attach_child( tc, NULL_FORK ); - fd_txncache_finalize_fork( tc, root, 0UL, blockhash ); - fd_txncache_fork_id_t winner = fd_txncache_attach_child( tc, root ); - fd_txncache_insert( tc, winner, blockhash, txnhash ); - fd_txncache_finalize_fork( tc, winner, 0UL, winner_hash ); - fd_txncache_advance_root( tc, winner ); - - uint head = tc->blockcache_pool[ root.val ].heads[ 0 ]; - FD_TEST( head!=UINT_MAX ); - fd_txncache_single_txn_t * txn = tc->txnpages[ head/FD_TXNCACHE_TXNS_PER_PAGE ].txns[ head%FD_TXNCACHE_TXNS_PER_PAGE ]; - - struct cycle_inject inject = { - .shmem = test_tc.shmem, - .txn = txn, - .head = head, - .next = txn->blockcache_next - }; - - uchar * slot_history = test_alloc( wksp, alignof(uchar), FD_SYSVAR_SLOT_HISTORY_BINCODE_SZ ); - mock_slot_history( slot_history, 1UL, 1UL, NULL, 0UL ); - fd_txncache_writer_t * writer = new_writer( wksp ); - ulong arena_sz; - void * arena = new_arena( wksp, 4UL, &arena_sz ); - - fd_racesan_t racesan[ 1 ]; - FD_TEST( fd_racesan_new( racesan, &inject ) ); - fd_racesan_inject( racesan, "txncache_writer:bucket_started", inject_cycle ); - fd_racesan_inject( racesan, "txncache_writer:txnhash_staged", count_staged ); - fd_racesan_inject( racesan, "txncache_writer:anomaly_detected", repair_cycle ); - ulong mutation_gen = test_tc.shmem->mutation_gen; - ulong sz = 0UL; - uchar * buf = NULL; - if( !during_fill ) { - FD_RACESAN_INJECT_BEGIN( racesan ) { - FD_TEST( fd_txncache_writer_init( writer, tc, winner, 1UL, slot_history, FD_SYSVAR_SLOT_HISTORY_BINCODE_SZ, arena, arena_sz ) ); - } FD_RACESAN_INJECT_END; - buf = serialize_all( wksp, writer, FD_TXNCACHE_WRITER_BUF_MIN, &sz ); - } else { - FD_TEST( fd_txncache_writer_init( writer, tc, winner, 1UL, slot_history, FD_SYSVAR_SLOT_HISTORY_BINCODE_SZ, arena, arena_sz ) ); - FD_RACESAN_INJECT_BEGIN( racesan ) { - buf = serialize_all( wksp, writer, FD_TXNCACHE_WRITER_BUF_MIN, &sz ); - } FD_RACESAN_INJECT_END; - FD_TEST( inject.staged_cnt==2UL ); - } - FD_TEST( inject.installed && inject.repaired ); - FD_TEST( test_tc.shmem->mutation_gen==mutation_gen+2UL ); - - expect_t exp[ 1 ] = {{ - .exec_slot = 1UL, - .blockhash_i = 0UL - }}; - memcpy( exp->txnhash, txnhash, 20UL ); - memcpy( exp->blockhash, blockhash, 32UL ); - FD_TEST( parse_and_check( wksp, buf, sz, exp, 1UL, 1UL, 0UL, NULL )==1UL ); + FD_TEST( inject.miss_cnt==2UL ); + cache_miss_check( cm, wksp, buf, sz ); fd_racesan_delete( racesan ); } @@ -2172,21 +1965,41 @@ main( int argc, #if FD_HAS_RACESAN test_txncache_writer_ignores_unrooted_cancellation( wksp ); test_allocs_reclaim( wksp ); - test_txncache_writer_waits_for_mutation_end( wksp ); +#endif test_allocs_reclaim( wksp ); - test_txncache_writer_retries_compaction( wksp ); + + /* Repeat serialization/parse checks with a one-page cache and + direct I/O to the backing file. */ + char spill_path[] = "/tmp/fd-txncache-roundtrip-XXXXXX"; + spill_test_fd = mkstemp( spill_path ); + FD_TEST( spill_test_fd>=0 ); + FD_TEST( !unlink( spill_path ) ); + int spill_flags = fcntl( spill_test_fd, F_GETFL ); + FD_TEST( spill_flags>=0 ); + FD_TEST( !fcntl( spill_test_fd, F_SETFL, spill_flags|O_DIRECT ) ); + test_txncache_writer_cache_miss( wksp ); test_allocs_reclaim( wksp ); - test_txncache_writer_retries_at_periodic_check( wksp ); +#if FD_HAS_RACESAN + test_txncache_writer_cache_miss_compaction( wksp ); test_allocs_reclaim( wksp ); - test_txncache_writer_retries_transient_cycle( wksp, 0 ); +#endif + test_txncache_roundtrip_empty( wksp ); test_allocs_reclaim( wksp ); - test_txncache_writer_retries_transient_cycle( wksp, 1 ); + test_txncache_roundtrip_genesis_blockhash( wksp ); test_allocs_reclaim( wksp ); -#endif - test_txncache_writer_rejects_excess_descriptors( wksp ); + test_txncache_roundtrip( wksp ); + test_allocs_reclaim( wksp ); + test_txncache_roundtrip_slot_history( wksp ); + test_allocs_reclaim( wksp ); + test_txncache_roundtrip_large( wksp ); test_allocs_reclaim( wksp ); - test_txncache_writer_root_advance_is_fatal( wksp ); + test_txncache_roundtrip_many_roots( wksp ); + test_allocs_reclaim( wksp ); + stress_txncache_writer_concurrent_cancels( wksp ); + test_allocs_reclaim( wksp ); + stress_txncache_writer_concurrent_compaction( wksp ); test_allocs_reclaim( wksp ); + FD_TEST( !close( spill_test_fd ) ); FD_LOG_NOTICE(( "pass" )); fd_svm_test_halt( mini ); diff --git a/src/discof/execle/fd_execle_tile.c b/src/discof/execle/fd_execle_tile.c index f4573578c86..f443b87a958 100644 --- a/src/discof/execle/fd_execle_tile.c +++ b/src/discof/execle/fd_execle_tile.c @@ -146,6 +146,13 @@ metrics_write( fd_execle_tile_t * ctx ) { FD_PROGCACHE_METRICS_WRITE( EXECLE, ctx->progcache->metrics ); + fd_txncache_join_metrics_t const * txncache = fd_txncache_join_metrics( ctx->runtime->status_cache ); + FD_MCNT_SET ( EXECLE, TXNCACHE_QUERY_MISS, txncache->query_miss_cnt ); + FD_MCNT_SET ( EXECLE, TXNCACHE_INSERT_MISS, txncache->insert_miss_cnt ); + FD_MCNT_SET ( EXECLE, TXNCACHE_OPERATION_STALLED, txncache->stalled_cnt ); + FD_MHIST_COPY( EXECLE, TXNCACHE_QUERY_DURATION_NANOS, txncache->query_ticks ); + FD_MHIST_COPY( EXECLE, TXNCACHE_INSERT_DURATION_NANOS, txncache->insert_ticks ); + FD_ACCDB_METRICS_WRITE( EXECLE, fd_accdb_metrics( ctx->accdb ) ); } @@ -807,8 +814,11 @@ unprivileged_init( fd_topo_t const * topo, void * _txncache_shmem = fd_topo_obj_laddr( topo, tile->execle.txncache_obj_id ); fd_txncache_shmem_t * txncache_shmem = fd_txncache_shmem_join( _txncache_shmem ); FD_TEST( txncache_shmem ); - fd_txncache_t * txncache = fd_txncache_join( fd_txncache_new( _txncache, txncache_shmem ) ); + fd_txncache_t * txncache = fd_txncache_join( fd_txncache_new( _txncache, txncache_shmem, FD_TXNCACHE_FD ) ); FD_TEST( txncache ); + fd_txncache_join_metrics_t * txncache_metrics = fd_txncache_join_metrics( txncache ); + fd_histf_join( fd_metrics_histf_new_ns( txncache_metrics->query_ticks, FD_MHIST_EDGES_NS( EXECLE, TXNCACHE_QUERY_DURATION_NANOS ) ) ); + fd_histf_join( fd_metrics_histf_new_ns( txncache_metrics->insert_ticks, FD_MHIST_EDGES_NS( EXECLE, TXNCACHE_INSERT_DURATION_NANOS ) ) ); void * _accdb_shmem = fd_topo_obj_laddr( topo, tile->execle.accdb_obj_id ); fd_accdb_shmem_t * accdb_shmem = fd_accdb_shmem_join( _accdb_shmem ); @@ -872,7 +882,7 @@ populate_allowed_seccomp( fd_topo_t const * topo, (void)topo; (void)tile; - populate_sock_filter_policy_fd_execle_tile( out_cnt, out, (uint)fd_log_private_logfile_fd(), FD_ACCDB_FD_RW, FD_STAKE_DELEGATIONS_FD, FD_VOTE_STAKES_FD ); + populate_sock_filter_policy_fd_execle_tile( out_cnt, out, (uint)fd_log_private_logfile_fd(), FD_ACCDB_FD_RW, FD_STAKE_DELEGATIONS_FD, FD_VOTE_STAKES_FD, (uint)FD_TXNCACHE_FD ); return sock_filter_policy_fd_execle_tile_instr_cnt; } @@ -884,7 +894,7 @@ populate_allowed_fds( fd_topo_t const * topo, (void)topo; (void)tile; - if( FD_UNLIKELY( out_fds_cnt<5UL ) ) FD_LOG_ERR(( "out_fds_cnt %lu", out_fds_cnt )); + if( FD_UNLIKELY( out_fds_cnt<6UL ) ) FD_LOG_ERR(( "out_fds_cnt %lu", out_fds_cnt )); ulong out_cnt = 0UL; out_fds[ out_cnt++ ] = 2; /* stderr */ @@ -893,6 +903,7 @@ populate_allowed_fds( fd_topo_t const * topo, out_fds[ out_cnt++ ] = FD_ACCDB_FD_RW; /* accounts db */ out_fds[ out_cnt++ ] = FD_STAKE_DELEGATIONS_FD; /* stake delegation disk spill */ out_fds[ out_cnt++ ] = FD_VOTE_STAKES_FD; /* vote stakes disk spill */ + out_fds[ out_cnt++ ] = FD_TXNCACHE_FD; /* transaction cache spill */ return out_cnt; } diff --git a/src/discof/execle/fd_execle_tile.seccomppolicy b/src/discof/execle/fd_execle_tile.seccomppolicy index 88369d2e506..344830e42f4 100644 --- a/src/discof/execle/fd_execle_tile.seccomppolicy +++ b/src/discof/execle/fd_execle_tile.seccomppolicy @@ -9,7 +9,10 @@ # vote_stakes_fd: Disk backing for t-1 vote stakes sets beyond the # in-memory cache. Any tile reading a set that is not # cached loads it, writing back the evicted set. -uint logfile_fd, uint accounts_fd, uint stake_spill_fd, uint vote_stakes_fd +# +# txncache_fd: Disk backing for transaction cache pages beyond the +# in-memory cache. +uint logfile_fd, uint accounts_fd, uint stake_spill_fd, uint vote_stakes_fd, uint txncache_fd # logging: all log messages are written to a file and/or pipe # @@ -55,14 +58,18 @@ fallocate: (and (eq (arg 0) accounts_fd) # the in-memory delegation pools. # # vote stakes: load a spilled t-1 set into the cache. +# transaction cache: read spilled pages pread64: (or (eq (arg 0) stake_spill_fd) - (eq (arg 0) vote_stakes_fd)) + (eq (arg 0) vote_stakes_fd) + (eq (arg 0) txncache_fd)) # stake delegations: full-record disk spill # # vote stakes: write back the t-1 set evicted from the cache. +# transaction cache: spill pages to disk pwrite64: (or (eq (arg 0) stake_spill_fd) - (eq (arg 0) vote_stakes_fd)) + (eq (arg 0) vote_stakes_fd) + (eq (arg 0) txncache_fd)) # power saving: sleep when there is no work to do # diff --git a/src/discof/execle/generated/fd_execle_tile_seccomp.h b/src/discof/execle/generated/fd_execle_tile_seccomp.h index 0d7ffb91726..45cca7a2177 100644 --- a/src/discof/execle/generated/fd_execle_tile_seccomp.h +++ b/src/discof/execle/generated/fd_execle_tile_seccomp.h @@ -33,11 +33,11 @@ #define FD_SECCOMP_ARG_LO(x) ((uint)(((ulong)(uint)(int)(x) ) & 0xffffffffUL)) #define FD_SECCOMP_ARG_HI(x) ((uint)(((ulong)(x) >> 32) & 0xffffffffUL)) -static const uint sock_filter_policy_fd_execle_tile_instr_cnt = 61; +static const uint sock_filter_policy_fd_execle_tile_instr_cnt = 65; -static void populate_sock_filter_policy_fd_execle_tile( ulong out_cnt, struct sock_filter out[ static 61 ], uint logfile_fd, uint accounts_fd, uint stake_spill_fd, uint vote_stakes_fd ) { - FD_TEST( out_cnt >= 61 ); - struct sock_filter filter[61] = { +static void populate_sock_filter_policy_fd_execle_tile( ulong out_cnt, struct sock_filter out[ static 65 ], uint logfile_fd, uint accounts_fd, uint stake_spill_fd, uint vote_stakes_fd, uint txncache_fd ) { + FD_TEST( out_cnt >= 65 ); + struct sock_filter filter[65] = { /* validate architecture */ BPF_STMT( BPF_LD | BPF_W | BPF_ABS, ( offsetof( struct seccomp_data, arch ) )), BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ARCH_NR, 0, /* RET_KILL_PROCESS */ 11 ), @@ -58,9 +58,9 @@ static void populate_sock_filter_policy_fd_execle_tile( ulong out_cnt, struct so /* check pread64 */ BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, SYS_pread64, /* check_pread64 */ 33, 0 ), /* check pwrite64 */ - BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, SYS_pwrite64, /* check_pwrite64 */ 38, 0 ), + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, SYS_pwrite64, /* check_pwrite64 */ 40, 0 ), /* check futex */ - BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, SYS_futex, /* check_futex */ 43, 0 ), + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, SYS_futex, /* check_futex */ 47, 0 ), /* allow restart_syscall */ BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, SYS_restart_syscall, /* RET_ALLOW */ 1, 0 ), // RET_KILL_PROCESS: @@ -128,11 +128,15 @@ static void populate_sock_filter_policy_fd_execle_tile( ulong out_cnt, struct so // check_pread64: /* arg 0 low 32 bits */ BPF_STMT( BPF_LD | BPF_W | BPF_ABS, FD_SECCOMP_ARG_LO_OFFSET(0)), - BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ((uint)(stake_spill_fd)), /* pread64_ALLOW */ 3, /* or_3 */ 0 ), + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ((uint)(stake_spill_fd)), /* pread64_ALLOW */ 5, /* or_3 */ 0 ), // or_3: /* arg 0 low 32 bits */ BPF_STMT( BPF_LD | BPF_W | BPF_ABS, FD_SECCOMP_ARG_LO_OFFSET(0)), - BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ((uint)(vote_stakes_fd)), /* pread64_ALLOW */ 1, /* pread64_KILL */ 0 ), + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ((uint)(vote_stakes_fd)), /* pread64_ALLOW */ 3, /* or_4 */ 0 ), +// or_4: + /* arg 0 low 32 bits */ + BPF_STMT( BPF_LD | BPF_W | BPF_ABS, FD_SECCOMP_ARG_LO_OFFSET(0)), + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ((uint)(txncache_fd)), /* pread64_ALLOW */ 1, /* pread64_KILL */ 0 ), // pread64_KILL: BPF_STMT( BPF_RET | BPF_K, SECCOMP_RET_KILL_PROCESS ), // pread64_ALLOW: @@ -140,11 +144,15 @@ static void populate_sock_filter_policy_fd_execle_tile( ulong out_cnt, struct so // check_pwrite64: /* arg 0 low 32 bits */ BPF_STMT( BPF_LD | BPF_W | BPF_ABS, FD_SECCOMP_ARG_LO_OFFSET(0)), - BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ((uint)(stake_spill_fd)), /* pwrite64_ALLOW */ 3, /* or_4 */ 0 ), -// or_4: + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ((uint)(stake_spill_fd)), /* pwrite64_ALLOW */ 5, /* or_5 */ 0 ), +// or_5: + /* arg 0 low 32 bits */ + BPF_STMT( BPF_LD | BPF_W | BPF_ABS, FD_SECCOMP_ARG_LO_OFFSET(0)), + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ((uint)(vote_stakes_fd)), /* pwrite64_ALLOW */ 3, /* or_6 */ 0 ), +// or_6: /* arg 0 low 32 bits */ BPF_STMT( BPF_LD | BPF_W | BPF_ABS, FD_SECCOMP_ARG_LO_OFFSET(0)), - BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ((uint)(vote_stakes_fd)), /* pwrite64_ALLOW */ 1, /* pwrite64_KILL */ 0 ), + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ((uint)(txncache_fd)), /* pwrite64_ALLOW */ 1, /* pwrite64_KILL */ 0 ), // pwrite64_KILL: BPF_STMT( BPF_RET | BPF_K, SECCOMP_RET_KILL_PROCESS ), // pwrite64_ALLOW: @@ -152,8 +160,8 @@ static void populate_sock_filter_policy_fd_execle_tile( ulong out_cnt, struct so // check_futex: /* arg 1 low 32 bits */ BPF_STMT( BPF_LD | BPF_W | BPF_ABS, FD_SECCOMP_ARG_LO_OFFSET(1)), - BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, FD_SECCOMP_ARG_LO(FUTEX_WAIT_BITSET), /* and_5 */ 0, /* futex_KILL */ 2 ), -// and_5: + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, FD_SECCOMP_ARG_LO(FUTEX_WAIT_BITSET), /* and_7 */ 0, /* futex_KILL */ 2 ), +// and_7: /* arg 2 low 32 bits */ BPF_STMT( BPF_LD | BPF_W | BPF_ABS, FD_SECCOMP_ARG_LO_OFFSET(2)), BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, 0x00000000U, /* futex_ALLOW */ 1, /* futex_KILL */ 0 ), diff --git a/src/discof/execle/test_execle_tile.c b/src/discof/execle/test_execle_tile.c index 8e1eb0ae2f1..0054224cefd 100644 --- a/src/discof/execle/test_execle_tile.c +++ b/src/discof/execle/test_execle_tile.c @@ -810,15 +810,16 @@ test_assert_txn_ns_dt_ordered( fd_txn_ns_dt_t const * dt ) { } FD_UNIT_TEST( execle_seccomp ) { - int out_fds[5]; - ulong nfds = populate_allowed_fds( NULL, NULL, 5UL, out_fds ); - FD_TEST( nfds>=4 && nfds<=5 ); + int out_fds[6]; + ulong nfds = populate_allowed_fds( NULL, NULL, 6UL, out_fds ); + FD_TEST( nfds>=5 && nfds<=6 ); FD_TEST( out_fds[0]==STDERR_FILENO ); /* logfile fd is optional; the spill fds are always last */ - FD_TEST( out_fds[ nfds-3UL ]==FD_ACCDB_FD_RW ); - FD_TEST( out_fds[ nfds-2UL ]==FD_STAKE_DELEGATIONS_FD ); - FD_TEST( out_fds[ nfds-1UL ]==FD_VOTE_STAKES_FD ); - if( nfds==5 ) FD_TEST( out_fds[1]==fd_log_private_logfile_fd() ); + FD_TEST( out_fds[ nfds-4UL ]==FD_ACCDB_FD_RW ); + FD_TEST( out_fds[ nfds-3UL ]==FD_STAKE_DELEGATIONS_FD ); + FD_TEST( out_fds[ nfds-2UL ]==FD_VOTE_STAKES_FD ); + FD_TEST( out_fds[ nfds-1UL ]==FD_TXNCACHE_FD ); + if( nfds==6 ) FD_TEST( out_fds[1]==fd_log_private_logfile_fd() ); struct sock_filter filter[ sock_filter_policy_fd_execle_tile_instr_cnt ]; populate_allowed_seccomp( NULL, NULL, sock_filter_policy_fd_execle_tile_instr_cnt, filter ); diff --git a/src/discof/execrp/fd_execrp_tile.c b/src/discof/execrp/fd_execrp_tile.c index ead28b365d1..e74e9367202 100644 --- a/src/discof/execrp/fd_execrp_tile.c +++ b/src/discof/execrp/fd_execrp_tile.c @@ -164,6 +164,13 @@ metrics_write( fd_execrp_tile_t * ctx ) { FD_MCNT_SET( EXECRP, ACCOUNT_LTHASH_SKIPPED, runtime->metrics.lthash_unchanged_cnt ); FD_MCNT_SET( EXECRP, BANK_SWITCHED, runtime->metrics.bank_switch_cnt ); + fd_txncache_join_metrics_t const * txncache = fd_txncache_join_metrics( ctx->txncache ); + FD_MCNT_SET ( EXECRP, TXNCACHE_QUERY_MISS, txncache->query_miss_cnt ); + FD_MCNT_SET ( EXECRP, TXNCACHE_INSERT_MISS, txncache->insert_miss_cnt ); + FD_MCNT_SET ( EXECRP, TXNCACHE_OPERATION_STALLED, txncache->stalled_cnt ); + FD_MHIST_COPY( EXECRP, TXNCACHE_QUERY_DURATION_NANOS, txncache->query_ticks ); + FD_MHIST_COPY( EXECRP, TXNCACHE_INSERT_DURATION_NANOS, txncache->insert_ticks ); + FD_ACCDB_METRICS_WRITE( EXECRP, fd_accdb_metrics( ctx->accdb ) ); } @@ -400,8 +407,11 @@ unprivileged_init( fd_topo_t const * topo, void * _txncache_shmem = fd_topo_obj_laddr( topo, tile->execrp.txncache_obj_id ); fd_txncache_shmem_t * txncache_shmem = fd_txncache_shmem_join( _txncache_shmem ); FD_TEST( txncache_shmem ); - ctx->txncache = fd_txncache_join( fd_txncache_new( _txncache, txncache_shmem ) ); + ctx->txncache = fd_txncache_join( fd_txncache_new( _txncache, txncache_shmem, FD_TXNCACHE_FD ) ); FD_TEST( ctx->txncache ); + fd_txncache_join_metrics_t * txncache_metrics = fd_txncache_join_metrics( ctx->txncache ); + fd_histf_join( fd_metrics_histf_new_ns( txncache_metrics->query_ticks, FD_MHIST_EDGES_NS( EXECRP, TXNCACHE_QUERY_DURATION_NANOS ) ) ); + fd_histf_join( fd_metrics_histf_new_ns( txncache_metrics->insert_ticks, FD_MHIST_EDGES_NS( EXECRP, TXNCACHE_INSERT_DURATION_NANOS ) ) ); void * _accdb_shmem = fd_topo_obj_laddr( topo, tile->execrp.accdb_obj_id ); fd_accdb_shmem_t * accdb_shmem = fd_accdb_shmem_join( _accdb_shmem ); @@ -533,7 +543,7 @@ populate_allowed_seccomp( fd_topo_t const * topo FD_PARAM_UNUSED, fd_topo_tile_t const * tile FD_PARAM_UNUSED, ulong out_cnt, struct sock_filter * out ) { - populate_sock_filter_policy_fd_execrp_tile( out_cnt, out, (uint)fd_log_private_logfile_fd(), (uint)FD_ACCDB_FD_RW, FD_STAKE_DELEGATIONS_FD, FD_VOTE_STAKES_FD ); + populate_sock_filter_policy_fd_execrp_tile( out_cnt, out, (uint)fd_log_private_logfile_fd(), (uint)FD_ACCDB_FD_RW, FD_STAKE_DELEGATIONS_FD, FD_VOTE_STAKES_FD, (uint)FD_TXNCACHE_FD ); return sock_filter_policy_fd_execrp_tile_instr_cnt; } @@ -543,7 +553,7 @@ populate_allowed_fds( fd_topo_t const * topo FD_PARAM_UNUSED, ulong out_fds_cnt, int * out_fds ) { - if( FD_UNLIKELY( out_fds_cnt<5UL ) ) FD_LOG_ERR(( "out_fds_cnt %lu", out_fds_cnt )); + if( FD_UNLIKELY( out_fds_cnt<6UL ) ) FD_LOG_ERR(( "out_fds_cnt %lu", out_fds_cnt )); ulong out_cnt = 0UL; out_fds[ out_cnt++ ] = 2; /* stderr */ @@ -553,6 +563,7 @@ populate_allowed_fds( fd_topo_t const * topo FD_PARAM_UNUSED, out_fds[ out_cnt++ ] = FD_ACCDB_FD_RW; /* accounts db */ out_fds[ out_cnt++ ] = FD_STAKE_DELEGATIONS_FD; /* stake delegation disk spill */ out_fds[ out_cnt++ ] = FD_VOTE_STAKES_FD; /* vote stakes disk spill */ + out_fds[ out_cnt++ ] = FD_TXNCACHE_FD; /* transaction cache spill */ return out_cnt; } diff --git a/src/discof/execrp/fd_execrp_tile.seccomppolicy b/src/discof/execrp/fd_execrp_tile.seccomppolicy index 88369d2e506..344830e42f4 100644 --- a/src/discof/execrp/fd_execrp_tile.seccomppolicy +++ b/src/discof/execrp/fd_execrp_tile.seccomppolicy @@ -9,7 +9,10 @@ # vote_stakes_fd: Disk backing for t-1 vote stakes sets beyond the # in-memory cache. Any tile reading a set that is not # cached loads it, writing back the evicted set. -uint logfile_fd, uint accounts_fd, uint stake_spill_fd, uint vote_stakes_fd +# +# txncache_fd: Disk backing for transaction cache pages beyond the +# in-memory cache. +uint logfile_fd, uint accounts_fd, uint stake_spill_fd, uint vote_stakes_fd, uint txncache_fd # logging: all log messages are written to a file and/or pipe # @@ -55,14 +58,18 @@ fallocate: (and (eq (arg 0) accounts_fd) # the in-memory delegation pools. # # vote stakes: load a spilled t-1 set into the cache. +# transaction cache: read spilled pages pread64: (or (eq (arg 0) stake_spill_fd) - (eq (arg 0) vote_stakes_fd)) + (eq (arg 0) vote_stakes_fd) + (eq (arg 0) txncache_fd)) # stake delegations: full-record disk spill # # vote stakes: write back the t-1 set evicted from the cache. +# transaction cache: spill pages to disk pwrite64: (or (eq (arg 0) stake_spill_fd) - (eq (arg 0) vote_stakes_fd)) + (eq (arg 0) vote_stakes_fd) + (eq (arg 0) txncache_fd)) # power saving: sleep when there is no work to do # diff --git a/src/discof/execrp/generated/fd_execrp_tile_seccomp.h b/src/discof/execrp/generated/fd_execrp_tile_seccomp.h index 7b03e21cebb..449acccc4cb 100644 --- a/src/discof/execrp/generated/fd_execrp_tile_seccomp.h +++ b/src/discof/execrp/generated/fd_execrp_tile_seccomp.h @@ -33,11 +33,11 @@ #define FD_SECCOMP_ARG_LO(x) ((uint)(((ulong)(uint)(int)(x) ) & 0xffffffffUL)) #define FD_SECCOMP_ARG_HI(x) ((uint)(((ulong)(x) >> 32) & 0xffffffffUL)) -static const uint sock_filter_policy_fd_execrp_tile_instr_cnt = 61; +static const uint sock_filter_policy_fd_execrp_tile_instr_cnt = 65; -static void populate_sock_filter_policy_fd_execrp_tile( ulong out_cnt, struct sock_filter out[ static 61 ], uint logfile_fd, uint accounts_fd, uint stake_spill_fd, uint vote_stakes_fd ) { - FD_TEST( out_cnt >= 61 ); - struct sock_filter filter[61] = { +static void populate_sock_filter_policy_fd_execrp_tile( ulong out_cnt, struct sock_filter out[ static 65 ], uint logfile_fd, uint accounts_fd, uint stake_spill_fd, uint vote_stakes_fd, uint txncache_fd ) { + FD_TEST( out_cnt >= 65 ); + struct sock_filter filter[65] = { /* validate architecture */ BPF_STMT( BPF_LD | BPF_W | BPF_ABS, ( offsetof( struct seccomp_data, arch ) )), BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ARCH_NR, 0, /* RET_KILL_PROCESS */ 11 ), @@ -58,9 +58,9 @@ static void populate_sock_filter_policy_fd_execrp_tile( ulong out_cnt, struct so /* check pread64 */ BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, SYS_pread64, /* check_pread64 */ 33, 0 ), /* check pwrite64 */ - BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, SYS_pwrite64, /* check_pwrite64 */ 38, 0 ), + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, SYS_pwrite64, /* check_pwrite64 */ 40, 0 ), /* check futex */ - BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, SYS_futex, /* check_futex */ 43, 0 ), + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, SYS_futex, /* check_futex */ 47, 0 ), /* allow restart_syscall */ BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, SYS_restart_syscall, /* RET_ALLOW */ 1, 0 ), // RET_KILL_PROCESS: @@ -128,11 +128,15 @@ static void populate_sock_filter_policy_fd_execrp_tile( ulong out_cnt, struct so // check_pread64: /* arg 0 low 32 bits */ BPF_STMT( BPF_LD | BPF_W | BPF_ABS, FD_SECCOMP_ARG_LO_OFFSET(0)), - BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ((uint)(stake_spill_fd)), /* pread64_ALLOW */ 3, /* or_3 */ 0 ), + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ((uint)(stake_spill_fd)), /* pread64_ALLOW */ 5, /* or_3 */ 0 ), // or_3: /* arg 0 low 32 bits */ BPF_STMT( BPF_LD | BPF_W | BPF_ABS, FD_SECCOMP_ARG_LO_OFFSET(0)), - BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ((uint)(vote_stakes_fd)), /* pread64_ALLOW */ 1, /* pread64_KILL */ 0 ), + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ((uint)(vote_stakes_fd)), /* pread64_ALLOW */ 3, /* or_4 */ 0 ), +// or_4: + /* arg 0 low 32 bits */ + BPF_STMT( BPF_LD | BPF_W | BPF_ABS, FD_SECCOMP_ARG_LO_OFFSET(0)), + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ((uint)(txncache_fd)), /* pread64_ALLOW */ 1, /* pread64_KILL */ 0 ), // pread64_KILL: BPF_STMT( BPF_RET | BPF_K, SECCOMP_RET_KILL_PROCESS ), // pread64_ALLOW: @@ -140,11 +144,15 @@ static void populate_sock_filter_policy_fd_execrp_tile( ulong out_cnt, struct so // check_pwrite64: /* arg 0 low 32 bits */ BPF_STMT( BPF_LD | BPF_W | BPF_ABS, FD_SECCOMP_ARG_LO_OFFSET(0)), - BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ((uint)(stake_spill_fd)), /* pwrite64_ALLOW */ 3, /* or_4 */ 0 ), -// or_4: + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ((uint)(stake_spill_fd)), /* pwrite64_ALLOW */ 5, /* or_5 */ 0 ), +// or_5: + /* arg 0 low 32 bits */ + BPF_STMT( BPF_LD | BPF_W | BPF_ABS, FD_SECCOMP_ARG_LO_OFFSET(0)), + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ((uint)(vote_stakes_fd)), /* pwrite64_ALLOW */ 3, /* or_6 */ 0 ), +// or_6: /* arg 0 low 32 bits */ BPF_STMT( BPF_LD | BPF_W | BPF_ABS, FD_SECCOMP_ARG_LO_OFFSET(0)), - BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ((uint)(vote_stakes_fd)), /* pwrite64_ALLOW */ 1, /* pwrite64_KILL */ 0 ), + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ((uint)(txncache_fd)), /* pwrite64_ALLOW */ 1, /* pwrite64_KILL */ 0 ), // pwrite64_KILL: BPF_STMT( BPF_RET | BPF_K, SECCOMP_RET_KILL_PROCESS ), // pwrite64_ALLOW: @@ -152,8 +160,8 @@ static void populate_sock_filter_policy_fd_execrp_tile( ulong out_cnt, struct so // check_futex: /* arg 1 low 32 bits */ BPF_STMT( BPF_LD | BPF_W | BPF_ABS, FD_SECCOMP_ARG_LO_OFFSET(1)), - BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, FD_SECCOMP_ARG_LO(FUTEX_WAIT_BITSET), /* and_5 */ 0, /* futex_KILL */ 2 ), -// and_5: + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, FD_SECCOMP_ARG_LO(FUTEX_WAIT_BITSET), /* and_7 */ 0, /* futex_KILL */ 2 ), +// and_7: /* arg 2 low 32 bits */ BPF_STMT( BPF_LD | BPF_W | BPF_ABS, FD_SECCOMP_ARG_LO_OFFSET(2)), BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, 0x00000000U, /* futex_ALLOW */ 1, /* futex_KILL */ 0 ), diff --git a/src/discof/execrp/test_execrp_tile.c b/src/discof/execrp/test_execrp_tile.c index 79c7747894d..a160191cdd6 100644 --- a/src/discof/execrp/test_execrp_tile.c +++ b/src/discof/execrp/test_execrp_tile.c @@ -385,15 +385,16 @@ test_execrp_run( test_env_t * env, } FD_UNIT_TEST( execrp_seccomp ) { - int out_fds[5]; - ulong nfds = populate_allowed_fds( NULL, NULL, 5UL, out_fds ); - FD_TEST( nfds>=4 && nfds<=5 ); + int out_fds[6]; + ulong nfds = populate_allowed_fds( NULL, NULL, 6UL, out_fds ); + FD_TEST( nfds>=5 && nfds<=6 ); FD_TEST( out_fds[0]==STDERR_FILENO ); /* logfile fd is optional; the spill fds are always last */ - FD_TEST( out_fds[ nfds-3UL ]==FD_ACCDB_FD_RW ); - FD_TEST( out_fds[ nfds-2UL ]==FD_STAKE_DELEGATIONS_FD ); - FD_TEST( out_fds[ nfds-1UL ]==FD_VOTE_STAKES_FD ); - if( nfds==5 ) FD_TEST( out_fds[1]==fd_log_private_logfile_fd() ); + FD_TEST( out_fds[ nfds-4UL ]==FD_ACCDB_FD_RW ); + FD_TEST( out_fds[ nfds-3UL ]==FD_STAKE_DELEGATIONS_FD ); + FD_TEST( out_fds[ nfds-2UL ]==FD_VOTE_STAKES_FD ); + FD_TEST( out_fds[ nfds-1UL ]==FD_TXNCACHE_FD ); + if( nfds==6 ) FD_TEST( out_fds[1]==fd_log_private_logfile_fd() ); struct sock_filter filter[ sock_filter_policy_fd_execrp_tile_instr_cnt ]; populate_allowed_seccomp( NULL, NULL, sock_filter_policy_fd_execrp_tile_instr_cnt, filter ); diff --git a/src/discof/replay/fd_replay_tile.c b/src/discof/replay/fd_replay_tile.c index 27579c34083..c1f05181225 100644 --- a/src/discof/replay/fd_replay_tile.c +++ b/src/discof/replay/fd_replay_tile.c @@ -299,6 +299,20 @@ metrics_write( fd_replay_tile_t * ctx ) { FD_MGAUGE_SET( REPLAY, STORE_FEC_CACHE_CAPACITY, store_fec_cache_max ); FD_MGAUGE_SET( REPLAY, STORE_FEC_CACHE_PINNED, store_fec_cache_pinned ); + fd_txncache_metrics_t txncache_metrics[1]; + fd_txncache_metrics_snap( ctx->txncache, txncache_metrics ); + FD_MCNT_SET ( REPLAY, TXNCACHE_PAGE_MISS, txncache_metrics->page_miss_cnt ); + FD_MCNT_SET ( REPLAY, TXNCACHE_PAGE_SPILLED, txncache_metrics->page_spilled_cnt ); + FD_MCNT_SET ( REPLAY, TXNCACHE_PAGE_SPILL_READ, txncache_metrics->page_spill_read_cnt ); + FD_MCNT_SET ( REPLAY, TXNCACHE_FRAME_EVICTED, txncache_metrics->frame_evicted_cnt ); + FD_MCNT_SET ( REPLAY, TXNCACHE_SPILL_DURATION_NANOS, txncache_metrics->spill_ticks ); + FD_MCNT_SET ( REPLAY, TXNCACHE_LOCK_READ_WAIT_DURATION_NANOS, txncache_metrics->lock_read_wait_ticks ); + FD_MCNT_SET ( REPLAY, TXNCACHE_LOCK_WRITE_WAIT_DURATION_NANOS, txncache_metrics->lock_write_wait_ticks ); + FD_MGAUGE_SET( REPLAY, TXNCACHE_PAGE_USED, txncache_metrics->page_used ); + FD_MGAUGE_SET( REPLAY, TXNCACHE_PAGE_CAPACITY, txncache_metrics->page_capacity ); + FD_MGAUGE_SET( REPLAY, TXNCACHE_FRAME_USED, txncache_metrics->frame_used ); + FD_MGAUGE_SET( REPLAY, TXNCACHE_FRAME_CAPACITY, txncache_metrics->frame_capacity ); + FD_MGAUGE_SET( REPLAY, ROOT_SLOT, ctx->consensus_root_slot==ULONG_MAX ? 0UL : ctx->consensus_root_slot ); ulong leader_slot = ctx->leader_bank ? ctx->leader_bank->f.slot : 0UL; @@ -2363,6 +2377,7 @@ boot_genesis( fd_replay_tile_t * ctx, fd_runtime_read_genesis( ctx->banks, bank, ctx->accdb, NULL, &meta->genesis_hash, &meta->lthash, ctx->genesis, genesis_blob, ctx->runtime_stack ); FD_TEST( fd_sysvar_cache_validate( bank, ctx->accdb ) ); + fd_txncache_reset( ctx->txncache ); bank->txncache_fork_id = fd_txncache_attach_child ( ctx->txncache, (fd_txncache_fork_id_t){USHORT_MAX} ); bank->progcache_fork_id = fd_progcache_attach_child( ctx->progcache, fd_progcache_fork_id_initial() ); @@ -4500,7 +4515,7 @@ unprivileged_init( fd_topo_t const * topo, void * _txncache_shmem = fd_topo_obj_laddr( topo, tile->replay.txncache_obj_id ); fd_txncache_shmem_t * txncache_shmem = fd_txncache_shmem_join( _txncache_shmem ); FD_TEST( txncache_shmem ); - ctx->txncache = fd_txncache_join( fd_txncache_new( _txncache, txncache_shmem ) ); + ctx->txncache = fd_txncache_join( fd_txncache_new( _txncache, txncache_shmem, FD_TXNCACHE_FD ) ); FD_TEST( ctx->txncache ); void * _accdb_shmem = fd_topo_obj_laddr( topo, tile->replay.accdb_obj_id ); @@ -4772,7 +4787,7 @@ populate_allowed_seccomp( fd_topo_t const * topo, void * scratch = fd_topo_obj_laddr( topo, tile->tile_obj_id ); FD_SCRATCH_ALLOC_INIT( l, scratch ); fd_replay_tile_t * ctx = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_replay_tile_t), sizeof(fd_replay_tile_t) ); - populate_sock_filter_policy_fd_replay_tile( out_cnt, out, (uint)fd_log_private_logfile_fd(), FD_ACCDB_FD_RW, (uint)ctx->store_disk_fd, FD_STAKE_DELEGATIONS_FD, FD_EPOCH_CREDITS_FD, FD_COST_TRACKER_FD, FD_VOTE_STAKES_FD ); + populate_sock_filter_policy_fd_replay_tile( out_cnt, out, (uint)fd_log_private_logfile_fd(), FD_ACCDB_FD_RW, (uint)ctx->store_disk_fd, FD_STAKE_DELEGATIONS_FD, FD_EPOCH_CREDITS_FD, FD_COST_TRACKER_FD, FD_VOTE_STAKES_FD, (uint)FD_TXNCACHE_FD ); return sock_filter_policy_fd_replay_tile_instr_cnt; } @@ -4784,7 +4799,7 @@ populate_allowed_fds( fd_topo_t const * topo, void * scratch = fd_topo_obj_laddr( topo, tile->tile_obj_id ); FD_SCRATCH_ALLOC_INIT( l, scratch ); fd_replay_tile_t * ctx = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_replay_tile_t), sizeof(fd_replay_tile_t) ); - if( FD_UNLIKELY( out_fds_cnt<8UL ) ) FD_LOG_ERR(( "out_fds_cnt %lu", out_fds_cnt )); + if( FD_UNLIKELY( out_fds_cnt<9UL ) ) FD_LOG_ERR(( "out_fds_cnt %lu", out_fds_cnt )); ulong out_cnt = 0UL; out_fds[ out_cnt++ ] = 2; /* stderr */ @@ -4795,6 +4810,7 @@ populate_allowed_fds( fd_topo_t const * topo, out_fds[ out_cnt++ ] = FD_EPOCH_CREDITS_FD; /* epoch credits disk spill */ out_fds[ out_cnt++ ] = FD_COST_TRACKER_FD; /* cost tracker disk spill */ out_fds[ out_cnt++ ] = FD_VOTE_STAKES_FD; /* vote stakes disk spill */ + out_fds[ out_cnt++ ] = FD_TXNCACHE_FD; /* transaction cache spill */ if( FD_LIKELY( ctx->store_disk_fd>=0 ) ) out_fds[ out_cnt++ ] = ctx->store_disk_fd; diff --git a/src/discof/replay/fd_replay_tile.seccomppolicy b/src/discof/replay/fd_replay_tile.seccomppolicy index fda91a76305..2580eb7f2c5 100644 --- a/src/discof/replay/fd_replay_tile.seccomppolicy +++ b/src/discof/replay/fd_replay_tile.seccomppolicy @@ -14,7 +14,10 @@ # # vote_stakes_fd: Disk backing for t-1 vote stakes sets beyond the # in-memory cache. -uint logfile_fd, uint accounts_fd, uint store_fd, uint stake_spill_fd, uint epoch_credits_fd, uint cost_tracker_fd, uint vote_stakes_fd +# +# txncache_fd: Disk backing for transaction cache pages beyond the +# in-memory cache. +uint logfile_fd, uint accounts_fd, uint store_fd, uint stake_spill_fd, uint epoch_credits_fd, uint cost_tracker_fd, uint vote_stakes_fd, uint txncache_fd # logging: all log messages are written to a file and/or pipe # @@ -79,15 +82,19 @@ restart_syscall # # stake delegations, epoch credits, cost trackers and vote stakes: # records spilled beyond their in-memory caches. +# transaction cache: read spilled pages pread64: (or (eq (arg 0) store_fd) (eq (arg 0) stake_spill_fd) (eq (arg 0) epoch_credits_fd) (eq (arg 0) cost_tracker_fd) - (eq (arg 0) vote_stakes_fd)) + (eq (arg 0) vote_stakes_fd) + (eq (arg 0) txncache_fd)) # stake delegations, epoch credits, cost trackers and vote stakes: disk # spill +# transaction cache: spill pages to disk pwrite64: (or (eq (arg 0) stake_spill_fd) (eq (arg 0) epoch_credits_fd) (eq (arg 0) cost_tracker_fd) - (eq (arg 0) vote_stakes_fd)) + (eq (arg 0) vote_stakes_fd) + (eq (arg 0) txncache_fd)) diff --git a/src/discof/replay/generated/fd_replay_tile_seccomp.h b/src/discof/replay/generated/fd_replay_tile_seccomp.h index 3dc79ad97d5..e49c87b2b45 100644 --- a/src/discof/replay/generated/fd_replay_tile_seccomp.h +++ b/src/discof/replay/generated/fd_replay_tile_seccomp.h @@ -33,11 +33,11 @@ #define FD_SECCOMP_ARG_LO(x) ((uint)(((ulong)(uint)(int)(x) ) & 0xffffffffUL)) #define FD_SECCOMP_ARG_HI(x) ((uint)(((ulong)(x) >> 32) & 0xffffffffUL)) -static const uint sock_filter_policy_fd_replay_tile_instr_cnt = 70; +static const uint sock_filter_policy_fd_replay_tile_instr_cnt = 74; -static void populate_sock_filter_policy_fd_replay_tile( ulong out_cnt, struct sock_filter out[ static 70 ], uint logfile_fd, uint accounts_fd, uint store_fd, uint stake_spill_fd, uint epoch_credits_fd, uint cost_tracker_fd, uint vote_stakes_fd ) { - FD_TEST( out_cnt >= 70 ); - struct sock_filter filter[70] = { +static void populate_sock_filter_policy_fd_replay_tile( ulong out_cnt, struct sock_filter out[ static 74 ], uint logfile_fd, uint accounts_fd, uint store_fd, uint stake_spill_fd, uint epoch_credits_fd, uint cost_tracker_fd, uint vote_stakes_fd, uint txncache_fd ) { + FD_TEST( out_cnt >= 74 ); + struct sock_filter filter[74] = { /* validate architecture */ BPF_STMT( BPF_LD | BPF_W | BPF_ABS, ( offsetof( struct seccomp_data, arch ) )), BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ARCH_NR, 0, /* RET_KILL_PROCESS */ 10 ), @@ -60,7 +60,7 @@ static void populate_sock_filter_policy_fd_replay_tile( ulong out_cnt, struct so /* check pread64 */ BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, SYS_pread64, /* check_pread64 */ 37, 0 ), /* check pwrite64 */ - BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, SYS_pwrite64, /* check_pwrite64 */ 48, 0 ), + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, SYS_pwrite64, /* check_pwrite64 */ 50, 0 ), // RET_KILL_PROCESS: /* default deny */ BPF_STMT( BPF_RET | BPF_K, SECCOMP_RET_KILL_PROCESS ), @@ -138,23 +138,27 @@ static void populate_sock_filter_policy_fd_replay_tile( ulong out_cnt, struct so // check_pread64: /* arg 0 low 32 bits */ BPF_STMT( BPF_LD | BPF_W | BPF_ABS, FD_SECCOMP_ARG_LO_OFFSET(0)), - BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ((uint)(store_fd)), /* pread64_ALLOW */ 9, /* or_6 */ 0 ), + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ((uint)(store_fd)), /* pread64_ALLOW */ 11, /* or_6 */ 0 ), // or_6: /* arg 0 low 32 bits */ BPF_STMT( BPF_LD | BPF_W | BPF_ABS, FD_SECCOMP_ARG_LO_OFFSET(0)), - BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ((uint)(stake_spill_fd)), /* pread64_ALLOW */ 7, /* or_7 */ 0 ), + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ((uint)(stake_spill_fd)), /* pread64_ALLOW */ 9, /* or_7 */ 0 ), // or_7: /* arg 0 low 32 bits */ BPF_STMT( BPF_LD | BPF_W | BPF_ABS, FD_SECCOMP_ARG_LO_OFFSET(0)), - BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ((uint)(epoch_credits_fd)), /* pread64_ALLOW */ 5, /* or_8 */ 0 ), + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ((uint)(epoch_credits_fd)), /* pread64_ALLOW */ 7, /* or_8 */ 0 ), // or_8: /* arg 0 low 32 bits */ BPF_STMT( BPF_LD | BPF_W | BPF_ABS, FD_SECCOMP_ARG_LO_OFFSET(0)), - BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ((uint)(cost_tracker_fd)), /* pread64_ALLOW */ 3, /* or_9 */ 0 ), + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ((uint)(cost_tracker_fd)), /* pread64_ALLOW */ 5, /* or_9 */ 0 ), // or_9: /* arg 0 low 32 bits */ BPF_STMT( BPF_LD | BPF_W | BPF_ABS, FD_SECCOMP_ARG_LO_OFFSET(0)), - BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ((uint)(vote_stakes_fd)), /* pread64_ALLOW */ 1, /* pread64_KILL */ 0 ), + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ((uint)(vote_stakes_fd)), /* pread64_ALLOW */ 3, /* or_10 */ 0 ), +// or_10: + /* arg 0 low 32 bits */ + BPF_STMT( BPF_LD | BPF_W | BPF_ABS, FD_SECCOMP_ARG_LO_OFFSET(0)), + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ((uint)(txncache_fd)), /* pread64_ALLOW */ 1, /* pread64_KILL */ 0 ), // pread64_KILL: BPF_STMT( BPF_RET | BPF_K, SECCOMP_RET_KILL_PROCESS ), // pread64_ALLOW: @@ -162,19 +166,23 @@ static void populate_sock_filter_policy_fd_replay_tile( ulong out_cnt, struct so // check_pwrite64: /* arg 0 low 32 bits */ BPF_STMT( BPF_LD | BPF_W | BPF_ABS, FD_SECCOMP_ARG_LO_OFFSET(0)), - BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ((uint)(stake_spill_fd)), /* pwrite64_ALLOW */ 7, /* or_10 */ 0 ), -// or_10: - /* arg 0 low 32 bits */ - BPF_STMT( BPF_LD | BPF_W | BPF_ABS, FD_SECCOMP_ARG_LO_OFFSET(0)), - BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ((uint)(epoch_credits_fd)), /* pwrite64_ALLOW */ 5, /* or_11 */ 0 ), + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ((uint)(stake_spill_fd)), /* pwrite64_ALLOW */ 9, /* or_11 */ 0 ), // or_11: /* arg 0 low 32 bits */ BPF_STMT( BPF_LD | BPF_W | BPF_ABS, FD_SECCOMP_ARG_LO_OFFSET(0)), - BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ((uint)(cost_tracker_fd)), /* pwrite64_ALLOW */ 3, /* or_12 */ 0 ), + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ((uint)(epoch_credits_fd)), /* pwrite64_ALLOW */ 7, /* or_12 */ 0 ), // or_12: /* arg 0 low 32 bits */ BPF_STMT( BPF_LD | BPF_W | BPF_ABS, FD_SECCOMP_ARG_LO_OFFSET(0)), - BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ((uint)(vote_stakes_fd)), /* pwrite64_ALLOW */ 1, /* pwrite64_KILL */ 0 ), + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ((uint)(cost_tracker_fd)), /* pwrite64_ALLOW */ 5, /* or_13 */ 0 ), +// or_13: + /* arg 0 low 32 bits */ + BPF_STMT( BPF_LD | BPF_W | BPF_ABS, FD_SECCOMP_ARG_LO_OFFSET(0)), + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ((uint)(vote_stakes_fd)), /* pwrite64_ALLOW */ 3, /* or_14 */ 0 ), +// or_14: + /* arg 0 low 32 bits */ + BPF_STMT( BPF_LD | BPF_W | BPF_ABS, FD_SECCOMP_ARG_LO_OFFSET(0)), + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ((uint)(txncache_fd)), /* pwrite64_ALLOW */ 1, /* pwrite64_KILL */ 0 ), // pwrite64_KILL: BPF_STMT( BPF_RET | BPF_K, SECCOMP_RET_KILL_PROCESS ), // pwrite64_ALLOW: diff --git a/src/discof/replay/test_replay_tile.c b/src/discof/replay/test_replay_tile.c index 89ac7eee313..178fffeac6b 100644 --- a/src/discof/replay/test_replay_tile.c +++ b/src/discof/replay/test_replay_tile.c @@ -254,6 +254,7 @@ mock_runtime_block_execute_prepare_fn( fd_banks_t * banks FD_PARAM_UNUSE #define fd_progcache_cancel_fork(c,f) ((void)(c),(void)(f)) #define fd_accdb_purge(a,f) ((void)(a),(void)(f)) #define fd_accdb_flush_metrics(a) do { if( a ) (fd_accdb_flush_metrics)(a); } while(0) +#define fd_txncache_metrics_snap(t,o) do { if( t ) (fd_txncache_metrics_snap)(t,o); else memset( (o), 0, sizeof(*(o)) ); } while(0) #define fd_accdb_metrics(a) ( (a) ? (fd_accdb_metrics)(a) : &mock_accdb_metrics ) #define fd_progcache_cache_class_occupancy(c,u,m) do { if( (c)->shmem ) (fd_progcache_cache_class_occupancy)(c,u,m); else { memset( u, 0, FD_PROGCACHE_CACHE_CLASS_CNT*sizeof(ulong) ); memset( m, 0, FD_PROGCACHE_CACHE_CLASS_CNT*sizeof(ulong) ); } } while(0) #define fd_sched_block_start_pending(s) ( (s) ? (fd_sched_block_start_pending)(s) : 0 ) diff --git a/src/discof/restore/fd_snapin_tile.c b/src/discof/restore/fd_snapin_tile.c index 4f9e1ca0629..092d5ae496c 100644 --- a/src/discof/restore/fd_snapin_tile.c +++ b/src/discof/restore/fd_snapin_tile.c @@ -134,7 +134,7 @@ struct blockhash_group { typedef struct blockhash_group blockhash_group_t; -FD_STATIC_ASSERT( sizeof(blockhash_group_t)==40UL, blockhash_group ); +FD_STATIC_ASSERT( sizeof(blockhash_group_t)==FD_TXNCACHE_SNAPIN_GROUP_SZ, blockhash_group ); /* After filtering with the recent blockhash queue, there is at most one group per retained (slot, recent blockhash) pair. Filtering can only @@ -818,7 +818,7 @@ txncache_staging_scratch( fd_snapin_tile_t * ctx ) { void * scratch = fd_txncache_snapin_scratch( ctx->lead.txncache, &scratch_sz ); ulong groups_max = FD_TXNCACHE_MAX_SLOT_DELTAS*ctx->lead.txncache_max_groups_per_slot; blockhash_group_t * groups = txncache_staging_groups_join( scratch, scratch_sz, groups_max ); - if( FD_UNLIKELY( !groups ) ) FD_LOG_ERR(( "txncache scratch (%lu bytes) too small to stage %lu blockhash groups (%lu bytes)", scratch_sz, groups_max, groups_max*sizeof(blockhash_group_t) )); + if( FD_UNLIKELY( !groups ) ) FD_LOG_ERR(( "txncache scratch (%lu bytes) too small to stage %lu blockhash groups (%lu bytes); increase runtime.transaction_cache_size_mib", scratch_sz, groups_max, groups_max*sizeof(blockhash_group_t) )); return groups; } @@ -2292,7 +2292,7 @@ populate_allowed_fds( fd_topo_t const * topo, fd_topo_tile_t const * tile, ulong out_fds_cnt, int * out_fds ) { - if( FD_UNLIKELY( out_fds_cnt<5UL ) ) FD_LOG_ERR(( "invalid out_fds_cnt %lu", out_fds_cnt )); + if( FD_UNLIKELY( out_fds_cnt<6UL ) ) FD_LOG_ERR(( "invalid out_fds_cnt %lu", out_fds_cnt )); fd_snapin_tile_t const * ctx = fd_topo_obj_laddr( topo, tile->tile_obj_id ); ulong out_cnt = 0; @@ -2303,6 +2303,7 @@ populate_allowed_fds( fd_topo_t const * topo, out_fds[ out_cnt++ ] = FD_ACCDB_FD_RW; /* accounts db */ out_fds[ out_cnt++ ] = FD_STAKE_DELEGATIONS_FD; /* stake delegation disk spill */ out_fds[ out_cnt++ ] = ctx->writer.accdb_direct_fd; /* accounts db, O_DIRECT */ + if( FD_LIKELY( !tile->kind_id ) ) out_fds[ out_cnt++ ] = FD_TXNCACHE_FD; return out_cnt; } @@ -2313,7 +2314,7 @@ populate_allowed_seccomp( fd_topo_t const * topo, ulong out_cnt, struct sock_filter * out ) { fd_snapin_tile_t const * ctx = fd_topo_obj_laddr( topo, tile->tile_obj_id ); - populate_sock_filter_policy_fd_snapin_tile( out_cnt, out, (uint)fd_log_private_logfile_fd(), FD_ACCDB_FD_RW, (uint)ctx->writer.accdb_direct_fd, FD_STAKE_DELEGATIONS_FD ); + populate_sock_filter_policy_fd_snapin_tile( out_cnt, out, (uint)fd_log_private_logfile_fd(), FD_ACCDB_FD_RW, (uint)ctx->writer.accdb_direct_fd, FD_STAKE_DELEGATIONS_FD, (uint)FD_TXNCACHE_FD ); return sock_filter_policy_fd_snapin_tile_instr_cnt; } @@ -2416,7 +2417,7 @@ unprivileged_init( fd_topo_t const * topo, void * _txncache_shmem = fd_topo_obj_laddr( topo, tile->snapin.txncache_obj_id ); fd_txncache_shmem_t * txncache_shmem = fd_txncache_shmem_join( _txncache_shmem ); FD_TEST( txncache_shmem ); - ctx->lead.txncache = fd_txncache_join( fd_txncache_new( _txncache, txncache_shmem ) ); + ctx->lead.txncache = fd_txncache_join( fd_txncache_new( _txncache, txncache_shmem, FD_TXNCACHE_FD ) ); FD_TEST( ctx->lead.txncache ); ctx->lead.blockhash_groups = NULL; diff --git a/src/discof/restore/fd_snapin_tile.seccomppolicy b/src/discof/restore/fd_snapin_tile.seccomppolicy index 8911acdff40..9966bee8e38 100644 --- a/src/discof/restore/fd_snapin_tile.seccomppolicy +++ b/src/discof/restore/fd_snapin_tile.seccomppolicy @@ -5,7 +5,7 @@ # tiles may need to pull accounts into the cache from disk, # or purge out of cache to the disk to make forward # progress. -uint logfile_fd, uint accounts_fd, uint accounts_direct_fd, uint stake_spill_fd +uint logfile_fd, uint accounts_fd, uint accounts_direct_fd, uint stake_spill_fd, uint txncache_fd # logging: all log messages are written to a file and/or pipe # @@ -41,18 +41,22 @@ fallocate: (and (eq (arg 0) accounts_fd) # stake delegations: full root and fork-delta records spilled beyond # the in-memory delegation pools. -pread64: (eq (arg 0) stake_spill_fd) +# transaction cache: read spilled pages +pread64: (or (eq (arg 0) stake_spill_fd) + (eq (arg 0) txncache_fd)) # accounts database: write account records at explicit offsets # # Parallel snapshot loader tiles write the packed account records # directly into their own partitions of the accounts file. # -# Stake delegations may also spill full records to disk. +# Stake delegations may also spill full records to disk, and the +# transaction cache may spill pages to disk. # # arg 0 is the file descriptor to write to. pwrite64: (or (eq (arg 0) accounts_direct_fd) - (eq (arg 0) stake_spill_fd)) + (eq (arg 0) stake_spill_fd) + (eq (arg 0) txncache_fd)) # power saving: sleep when there is no work to do # diff --git a/src/discof/restore/generated/fd_snapin_tile_seccomp.h b/src/discof/restore/generated/fd_snapin_tile_seccomp.h index 80a68399f7e..219be428b1f 100644 --- a/src/discof/restore/generated/fd_snapin_tile_seccomp.h +++ b/src/discof/restore/generated/fd_snapin_tile_seccomp.h @@ -33,11 +33,11 @@ #define FD_SECCOMP_ARG_LO(x) ((uint)(((ulong)(uint)(int)(x) ) & 0xffffffffUL)) #define FD_SECCOMP_ARG_HI(x) ((uint)(((ulong)(x) >> 32) & 0xffffffffUL)) -static const uint sock_filter_policy_fd_snapin_tile_instr_cnt = 54; +static const uint sock_filter_policy_fd_snapin_tile_instr_cnt = 58; -static void populate_sock_filter_policy_fd_snapin_tile( ulong out_cnt, struct sock_filter out[ static 54 ], uint logfile_fd, uint accounts_fd, uint accounts_direct_fd, uint stake_spill_fd ) { - FD_TEST( out_cnt >= 54 ); - struct sock_filter filter[54] = { +static void populate_sock_filter_policy_fd_snapin_tile( ulong out_cnt, struct sock_filter out[ static 58 ], uint logfile_fd, uint accounts_fd, uint accounts_direct_fd, uint stake_spill_fd, uint txncache_fd ) { + FD_TEST( out_cnt >= 58 ); + struct sock_filter filter[58] = { /* validate architecture */ BPF_STMT( BPF_LD | BPF_W | BPF_ABS, ( offsetof( struct seccomp_data, arch ) )), BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ARCH_NR, 0, /* RET_KILL_PROCESS */ 10 ), @@ -56,9 +56,9 @@ static void populate_sock_filter_policy_fd_snapin_tile( ulong out_cnt, struct so /* check pread64 */ BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, SYS_pread64, /* check_pread64 */ 29, 0 ), /* check pwrite64 */ - BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, SYS_pwrite64, /* check_pwrite64 */ 32, 0 ), + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, SYS_pwrite64, /* check_pwrite64 */ 34, 0 ), /* check futex */ - BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, SYS_futex, /* check_futex */ 37, 0 ), + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, SYS_futex, /* check_futex */ 41, 0 ), /* allow restart_syscall */ BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, SYS_restart_syscall, /* RET_ALLOW */ 1, 0 ), // RET_KILL_PROCESS: @@ -118,7 +118,11 @@ static void populate_sock_filter_policy_fd_snapin_tile( ulong out_cnt, struct so // check_pread64: /* arg 0 low 32 bits */ BPF_STMT( BPF_LD | BPF_W | BPF_ABS, FD_SECCOMP_ARG_LO_OFFSET(0)), - BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ((uint)(stake_spill_fd)), /* pread64_ALLOW */ 1, /* pread64_KILL */ 0 ), + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ((uint)(stake_spill_fd)), /* pread64_ALLOW */ 3, /* or_3 */ 0 ), +// or_3: + /* arg 0 low 32 bits */ + BPF_STMT( BPF_LD | BPF_W | BPF_ABS, FD_SECCOMP_ARG_LO_OFFSET(0)), + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ((uint)(txncache_fd)), /* pread64_ALLOW */ 1, /* pread64_KILL */ 0 ), // pread64_KILL: BPF_STMT( BPF_RET | BPF_K, SECCOMP_RET_KILL_PROCESS ), // pread64_ALLOW: @@ -126,11 +130,15 @@ static void populate_sock_filter_policy_fd_snapin_tile( ulong out_cnt, struct so // check_pwrite64: /* arg 0 low 32 bits */ BPF_STMT( BPF_LD | BPF_W | BPF_ABS, FD_SECCOMP_ARG_LO_OFFSET(0)), - BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ((uint)(accounts_direct_fd)), /* pwrite64_ALLOW */ 3, /* or_3 */ 0 ), -// or_3: + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ((uint)(accounts_direct_fd)), /* pwrite64_ALLOW */ 5, /* or_4 */ 0 ), +// or_4: + /* arg 0 low 32 bits */ + BPF_STMT( BPF_LD | BPF_W | BPF_ABS, FD_SECCOMP_ARG_LO_OFFSET(0)), + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ((uint)(stake_spill_fd)), /* pwrite64_ALLOW */ 3, /* or_5 */ 0 ), +// or_5: /* arg 0 low 32 bits */ BPF_STMT( BPF_LD | BPF_W | BPF_ABS, FD_SECCOMP_ARG_LO_OFFSET(0)), - BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ((uint)(stake_spill_fd)), /* pwrite64_ALLOW */ 1, /* pwrite64_KILL */ 0 ), + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ((uint)(txncache_fd)), /* pwrite64_ALLOW */ 1, /* pwrite64_KILL */ 0 ), // pwrite64_KILL: BPF_STMT( BPF_RET | BPF_K, SECCOMP_RET_KILL_PROCESS ), // pwrite64_ALLOW: @@ -138,8 +146,8 @@ static void populate_sock_filter_policy_fd_snapin_tile( ulong out_cnt, struct so // check_futex: /* arg 1 low 32 bits */ BPF_STMT( BPF_LD | BPF_W | BPF_ABS, FD_SECCOMP_ARG_LO_OFFSET(1)), - BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, FD_SECCOMP_ARG_LO(FUTEX_WAIT_BITSET), /* and_4 */ 0, /* futex_KILL */ 2 ), -// and_4: + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, FD_SECCOMP_ARG_LO(FUTEX_WAIT_BITSET), /* and_6 */ 0, /* futex_KILL */ 2 ), +// and_6: /* arg 2 low 32 bits */ BPF_STMT( BPF_LD | BPF_W | BPF_ABS, FD_SECCOMP_ARG_LO_OFFSET(2)), BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, 0x00000000U, /* futex_ALLOW */ 1, /* futex_KILL */ 0 ), diff --git a/src/discof/restore/test_snapin_tile.c b/src/discof/restore/test_snapin_tile.c index b638933b4d3..161d73887ba 100644 --- a/src/discof/restore/test_snapin_tile.c +++ b/src/discof/restore/test_snapin_tile.c @@ -222,6 +222,7 @@ test_padded_sz( ulong used ) { #undef fd_accdb_reset #include +#include #include "../../flamenco/stakes/test_stake_delegations_util.h" static fd_snapin_tile_t * test_ctx; @@ -1769,14 +1770,15 @@ test_txncache_staging_ctx_init( fd_snapin_tile_t * ctx, static fd_txncache_t * new_txncache( fd_wksp_t * wksp, ulong max_txn_per_slot ) { - void * shmem = fd_wksp_alloc_laddr( wksp, fd_txncache_shmem_align(), fd_txncache_shmem_footprint( 1UL, max_txn_per_slot ), 1UL ); + ulong cache_footprint = FD_TXNCACHE_MAX_SLOT_DELTAS*((max_txn_per_slot+1UL)/2UL)*sizeof(blockhash_group_t)+alignof(blockhash_group_t)-1UL; + void * shmem = fd_wksp_alloc_laddr( wksp, fd_txncache_shmem_align(), fd_txncache_shmem_footprint( 1UL, max_txn_per_slot, cache_footprint ), 1UL ); FD_TEST( shmem ); - fd_txncache_shmem_t * txncache_shmem = fd_txncache_shmem_join( fd_txncache_shmem_new( shmem, 1UL, max_txn_per_slot, 0UL ) ); + fd_txncache_shmem_t * txncache_shmem = fd_txncache_shmem_join( fd_txncache_shmem_new( shmem, 1UL, max_txn_per_slot, cache_footprint, 0UL ) ); FD_TEST( txncache_shmem ); void * local = fd_wksp_alloc_laddr( wksp, fd_txncache_align(), fd_txncache_footprint( 1UL ), 1UL ); FD_TEST( local ); - fd_txncache_t * txncache = fd_txncache_join( fd_txncache_new( local, txncache_shmem ) ); + fd_txncache_t * txncache = fd_txncache_join( fd_txncache_new( local, txncache_shmem, -1 ) ); FD_TEST( txncache ); return txncache; } @@ -1802,6 +1804,31 @@ test_txncache_staging_groups_fit_txncache_scratch( fd_wksp_t * wksp ) { FD_TEST( txncache_staging_groups_join( (void *)66UL, ring_sz+2UL, TEST_MAX_STAGED_GROUPS ) ); } +/* The caller-provided cache budget must fit blockhash-group staging. + An exact page multiple also needs room for group-alignment padding. */ +static void +test_txncache_staging_bench_scratch( ulong max_txn_per_slot, + ulong cache_footprint ) { + ulong shmem_sz = fd_txncache_shmem_footprint( 4UL, 2UL*max_txn_per_slot, cache_footprint ); + ulong local_sz = fd_txncache_footprint( 4UL ); + ulong sz = shmem_sz+local_sz; + uchar * mem = mmap( NULL, sz, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0 ); + FD_TEST( mem!=MAP_FAILED ); + fd_txncache_shmem_t * shmem = fd_txncache_shmem_join( fd_txncache_shmem_new( mem, 4UL, 2UL*max_txn_per_slot, cache_footprint, 0UL ) ); + FD_TEST( shmem ); + fd_txncache_t * txncache = fd_txncache_join( fd_txncache_new( mem+shmem_sz, shmem, -1 ) ); + FD_TEST( txncache ); + + ulong scratch_sz; + void * scratch = fd_txncache_snapin_scratch( txncache, &scratch_sz ); + ulong groups_max = FD_TXNCACHE_MAX_SLOT_DELTAS*max_txn_per_slot; + blockhash_group_t * groups = txncache_staging_groups_join( scratch, scratch_sz, groups_max ); + FD_TEST( groups ); + groups[ groups_max-1UL ].txncache_entry_cnt = 2U; + FD_TEST( groups[ groups_max-1UL ].txncache_entry_cnt==2U ); + FD_TEST( !munmap( mem, sz ) ); +} + static void test_txncache_staging_evicts_oldest_slot( fd_wksp_t * wksp ) { fd_snapin_tile_t * ctx = test_ctx; @@ -3414,6 +3441,8 @@ main( int argc, fd_wksp_reset( wksp, 1UL ); test_streaming_stake_delegation( wksp ); fd_wksp_reset( wksp, 1UL ); test_snoop_outcomes( wksp ); fd_wksp_reset( wksp, 1UL ); test_snoop_incremental_fork( wksp ); + test_txncache_staging_bench_scratch( 529411UL, 3072UL<<20 ); /* benchmark TOML budget */ + test_txncache_staging_bench_scratch( 122881UL, FD_TXNCACHE_MAX_SLOT_DELTAS*122881UL*sizeof(blockhash_group_t)+alignof(blockhash_group_t)-1UL ); /* exact page multiple */ test_txncache_staging_entry_size(); test_txncache_staging_group_record_size(); fd_wksp_reset( wksp, 1UL ); test_txncache_staging_groups_fit_txncache_scratch( wksp ); diff --git a/src/flamenco/runtime/Local.mk b/src/flamenco/runtime/Local.mk index 05eff761c19..40a8c9701c2 100644 --- a/src/flamenco/runtime/Local.mk +++ b/src/flamenco/runtime/Local.mk @@ -30,6 +30,7 @@ endif $(call add-hdrs,fd_txncache_shmem.h fd_txncache.h) $(call add-objs,fd_txncache_shmem fd_txncache,fd_flamenco) + $(call add-hdrs,fd_cost_tracker.h) $(call add-objs,fd_cost_tracker,fd_flamenco) $(call add-hdrs,fd_cost_tracker_store.h) @@ -105,6 +106,5 @@ $(call make-unit-test,test_system_ids,test_system_ids,fd_flamenco fd_util fd_bal $(call run-unit-test,test_system_ids) ifdef FD_HAS_HOSTED -# TODO: Flakes -# $(call run-unit-test,test_txncache) +$(call run-unit-test,test_txncache) endif diff --git a/src/flamenco/runtime/fd_txncache.c b/src/flamenco/runtime/fd_txncache.c index ce8a24fe125..d4ba5cb57e7 100644 --- a/src/flamenco/runtime/fd_txncache.c +++ b/src/flamenco/runtime/fd_txncache.c @@ -1,6 +1,10 @@ #include "fd_txncache.h" #include "fd_txncache_private.h" #include "../../util/log/fd_log.h" +#include "../../util/racesan/fd_racesan_target.h" + +#include +#include FD_FN_CONST ulong fd_txncache_align( void ) { @@ -13,14 +17,15 @@ fd_txncache_footprint( ulong max_live_slots ) { ulong l; l = FD_LAYOUT_INIT; - l = FD_LAYOUT_APPEND( l, FD_TXNCACHE_SHMEM_ALIGN, sizeof(fd_txncache_t) ); - l = FD_LAYOUT_APPEND( l, alignof(blockcache_t), max_active_slots*sizeof(blockcache_t) ); + l = FD_LAYOUT_APPEND( l, FD_TXNCACHE_ALIGN, sizeof(fd_txncache_t) ); + l = FD_LAYOUT_APPEND( l, alignof(blockcache_t), max_active_slots*sizeof(blockcache_t) ); return FD_LAYOUT_FINI( l, FD_TXNCACHE_ALIGN ); } void * fd_txncache_new( void * ljoin, - fd_txncache_shmem_t * shmem ) { + fd_txncache_shmem_t * shmem, + int spill_fd ) { if( FD_UNLIKELY( !ljoin ) ) { FD_LOG_WARNING(( "NULL ljoin" )); return NULL; @@ -38,9 +43,7 @@ fd_txncache_new( void * ljoin, /* Page counts come from the shmem header rather than being re-derived, so the layout walk below cannot desync from the one in fd_txncache_shmem_new. */ - ulong _max_txnpages = shmem->max_txnpages; - ulong _max_txnpages_per_blockhash = shmem->txnpages_per_blockhash_max; - ulong _txnpage_idx_sz = shmem->txnpage_idx_sz; + ulong _txnpages_max = shmem->txnpages_max; ulong _descends_footprint = descends_set_footprint( max_active_slots ); if( FD_UNLIKELY( !_descends_footprint ) ) { @@ -49,17 +52,19 @@ fd_txncache_new( void * ljoin, } FD_SCRATCH_ALLOC_INIT( l, shmem ); - fd_txncache_shmem_t * tc = FD_SCRATCH_ALLOC_APPEND( l, FD_TXNCACHE_SHMEM_ALIGN, sizeof(fd_txncache_shmem_t) ); - void * _blockhash_map = FD_SCRATCH_ALLOC_APPEND( l, blockhash_map_align(), blockhash_map_footprint( blockhash_map_chains ) ); - void * _blockcache_pool = FD_SCRATCH_ALLOC_APPEND( l, blockcache_pool_align(), blockcache_pool_footprint( max_active_slots ) ); - void * _blockcache_pages = FD_SCRATCH_ALLOC_APPEND( l, _txnpage_idx_sz, max_active_slots*_max_txnpages_per_blockhash*_txnpage_idx_sz ); - void * _blockcache_heads = FD_SCRATCH_ALLOC_APPEND( l, alignof(uint), max_active_slots*bucket_cnt*sizeof(uint) ); - void * _blockcache_descends = FD_SCRATCH_ALLOC_APPEND( l, descends_set_align(), max_active_slots*_descends_footprint ); - void * _txnpages_free = FD_SCRATCH_ALLOC_APPEND( l, _txnpage_idx_sz, _max_txnpages*_txnpage_idx_sz ); - void * _txnpages = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_txncache_txnpage_t), _max_txnpages*sizeof(fd_txncache_txnpage_t) ); - void * _scratch_pages = FD_SCRATCH_ALLOC_APPEND( l, _txnpage_idx_sz, _max_txnpages_per_blockhash*_txnpage_idx_sz ); - void * _scratch_heads = FD_SCRATCH_ALLOC_APPEND( l, alignof(uint), bucket_cnt*sizeof(uint) ); - void * _scratch_txnpage = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_txncache_txnpage_t), sizeof(fd_txncache_txnpage_t) ); + fd_txncache_shmem_t * tc = FD_SCRATCH_ALLOC_APPEND( l, FD_TXNCACHE_SHMEM_ALIGN, sizeof(fd_txncache_shmem_t) ); + void * _blockhash_map = FD_SCRATCH_ALLOC_APPEND( l, blockhash_map_align(), blockhash_map_footprint( blockhash_map_chains ) ); + void * _blockcache_pool = FD_SCRATCH_ALLOC_APPEND( l, blockcache_pool_align(), blockcache_pool_footprint( max_active_slots ) ); + void * _blockcache_heads = FD_SCRATCH_ALLOC_APPEND( l, alignof(uint), max_active_slots*bucket_cnt*sizeof(uint) ); + void * _blockcache_descends = FD_SCRATCH_ALLOC_APPEND( l, descends_set_align(), max_active_slots*_descends_footprint ); + void * _txnpages_free = FD_SCRATCH_ALLOC_APPEND( l, alignof(uint), _txnpages_max*sizeof(uint) ); + void * _page_meta = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_txncache_page_meta_t), _txnpages_max*sizeof(fd_txncache_page_meta_t) ); + void * _frame_page = FD_SCRATCH_ALLOC_APPEND( l, alignof(uint), shmem->frames_max*sizeof(uint) ); + void * _frames_free = FD_SCRATCH_ALLOC_APPEND( l, alignof(uint), shmem->frames_max*sizeof(uint) ); + void * _txnpages = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_txncache_txnpage_t), shmem->frames_max*sizeof(fd_txncache_txnpage_t) ); + void * _scratch_heads = FD_SCRATCH_ALLOC_APPEND( l, alignof(uint), bucket_cnt*sizeof(uint) ); + void * _scratch_txnpage = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_txncache_txnpage_t), sizeof(fd_txncache_txnpage_t) ); + void * _scratch_input = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_txncache_txnpage_t), sizeof(fd_txncache_txnpage_t) ); FD_SCRATCH_ALLOC_INIT( l2, ljoin ); fd_txncache_t * ltc = FD_SCRATCH_ALLOC_APPEND( l2, FD_TXNCACHE_ALIGN, sizeof(fd_txncache_t) ); @@ -71,7 +76,6 @@ fd_txncache_new( void * ljoin, ltc->blockcache_shmem_pool = blockcache_pool_join( _blockcache_pool ); for( ulong i=0UL; iactive_slots_max; i++ ) { - ltc->blockcache_pool[ i ].pages = (uchar *)_blockcache_pages + i*_max_txnpages_per_blockhash*_txnpage_idx_sz; ltc->blockcache_pool[ i ].heads = (uint *)_blockcache_heads + i*bucket_cnt; ltc->blockcache_pool[ i ].descends = descends_set_join( (uchar *)_blockcache_descends + i*_descends_footprint ); ltc->blockcache_pool[ i ].shmem = ltc->blockcache_shmem_pool + i; @@ -83,16 +87,28 @@ fd_txncache_new( void * ljoin, ltc->blockhash_map = blockhash_map_join( _blockhash_map ); FD_TEST( ltc->blockhash_map ); - ltc->txnpages_free = _txnpages_free; - ltc->txnpages = (fd_txncache_txnpage_t *)_txnpages; - - ltc->scratch_pages = _scratch_pages; + ltc->txnpages_free = _txnpages_free; + ltc->page_meta = _page_meta; + ltc->frame_page = _frame_page; + ltc->frames_free = _frames_free; + ltc->txnpages = (fd_txncache_txnpage_t *)_txnpages; ltc->scratch_heads = _scratch_heads; ltc->scratch_txnpage = _scratch_txnpage; + ltc->scratch_input = _scratch_input; + ltc->spill_fd = spill_fd; + + memset( ltc->metrics, 0, sizeof(ltc->metrics) ); + fd_histf_join( fd_histf_new( ltc->metrics->query_ticks, 1UL, 1UL<<30 ) ); + fd_histf_join( fd_histf_new( ltc->metrics->insert_ticks, 1UL, 1UL<<30 ) ); return (void *)ltc; } +fd_txncache_join_metrics_t * +fd_txncache_join_metrics( fd_txncache_t * tc ) { + return tc->metrics; +} + fd_txncache_t * fd_txncache_join( void * ljoin ) { if( FD_UNLIKELY( !ljoin ) ) { @@ -110,30 +126,280 @@ fd_txncache_join( void * ljoin ) { return tc; } +/* lock_read and lock_write take the txncache lock, adding the time + spent waiting to the lock wait metrics. Uncontended acquisitions + are not timed. */ + +static inline void +lock_read( fd_txncache_t * tc ) FD_ACQUIRE_SHARED( tc->shmem->lock ) FD_NO_THREAD_SAFETY_ANALYSIS { + if( FD_LIKELY( fd_rwlock_tryread( tc->shmem->lock ) ) ) return; + long t0 = fd_tickcount(); + fd_rwlock_read( tc->shmem->lock ); + FD_ATOMIC_FETCH_AND_ADD( &tc->shmem->lock_read_wait_ticks, (ulong)(fd_tickcount()-t0) ); +} + +static inline void +lock_write( fd_txncache_t * tc ) FD_ACQUIRE( tc->shmem->lock ) FD_NO_THREAD_SAFETY_ANALYSIS { + if( FD_LIKELY( fd_rwlock_trywrite( tc->shmem->lock ) ) ) return; + long t0 = fd_tickcount(); + fd_rwlock_write( tc->shmem->lock ); + tc->shmem->lock_write_wait_ticks += (ulong)(fd_tickcount()-t0); +} + +static void +spill_io( fd_txncache_t * tc, + ulong page, + fd_txncache_txnpage_t * data, + int write ) { + FD_TEST( tc->spill_fd>=0 ); + long t0 = fd_tickcount(); + ulong off = page*sizeof(*data); + ulong rem = sizeof(*data); + uchar * buf = (uchar *)data; + while( rem ) { + long n = write ? pwrite( tc->spill_fd, buf, rem, (off_t)off ) + : pread ( tc->spill_fd, buf, rem, (off_t)off ); + if( FD_UNLIKELY( n<0L && errno==EINTR ) ) continue; + if( FD_UNLIKELY( n<0L ) ) FD_LOG_CRIT(( "txncache spill %s failed for page %lu at offset %lu (%ld, errno %d)", write ? "write" : "read", page, off, n, errno )); + if( FD_UNLIKELY( !n ) ) FD_LOG_CRIT(( "txncache spill %s for page %lu at offset %lu", write ? "write made no progress" : "read reached unexpected EOF", page, off )); + rem -= (ulong)n; + if( FD_UNLIKELY( rem && !fd_ulong_is_aligned( (ulong)n, FD_TXNCACHE_TXNPAGE_ALIGN ) ) ) { + FD_LOG_CRIT(( "txncache spill %s returned an unaligned short transfer for page %lu", write ? "write" : "read", page )); + } + buf += (ulong)n; + off += (ulong)n; + } + if( write ) FD_ATOMIC_FETCH_AND_ADD( &tc->shmem->page_spilled_cnt, 1UL ); + else FD_ATOMIC_FETCH_AND_ADD( &tc->shmem->page_spill_read_cnt, 1UL ); + FD_ATOMIC_FETCH_AND_ADD( &tc->shmem->spill_ticks, (ulong)(fd_tickcount()-t0) ); +} + +/* Waits until no fetch is moving the page. Fetches publish without the + lock, so this may be called with the lock held. */ + +static inline void +page_wait_idle( fd_txncache_page_meta_t const * meta ) { + while( FD_UNLIKELY( FD_VOLATILE_CONST( meta->io ) ) ) FD_SPIN_PAUSE(); + FD_COMPILER_MFENCE(); +} + +/* Takes a frame to load a page into, under the write lock. Uses a free + frame, or evicts by CLOCK: the sweep spares, once, each frame + accessed since it last passed, and skips frames a fetch is filling. + On eviction *victim is the evicted page, which is no longer resident + and whose dirty contents are still in the frame, otherwise UINT_MAX. + Returns UINT_MAX if every frame is being filled. */ + +static uint +frame_take( fd_txncache_t * tc, + uint * victim ) { + fd_txncache_shmem_t * shmem = tc->shmem; + *victim = UINT_MAX; + if( FD_LIKELY( shmem->frames_free_cnt ) ) return tc->frames_free[ --shmem->frames_free_cnt ]; + for( ulong i=0UL; i<2UL*shmem->frames_max; i++ ) { + uint frame = (uint)shmem->evict_next; + shmem->evict_next = (shmem->evict_next+1UL)%shmem->frames_max; + uint owner = tc->frame_page[ frame ]; + FD_TEST( owner!=UINT_MAX ); + fd_txncache_page_meta_t * old = &tc->page_meta[ owner ]; + if( FD_VOLATILE_CONST( old->io ) ) continue; + if( old->ref ) { + old->ref = 0U; + continue; + } + old->frame = UINT_MAX; + shmem->frame_evicted_cnt++; + *victim = owner; + return frame; + } + return UINT_MAX; +} + +static inline fd_txncache_txnpage_t * +page_io( fd_txncache_t * tc, + ulong page, + int write_lock ) { + /* Fast path: the page is resident. A fetch may publish the frame + concurrently. */ + fd_txncache_page_meta_t * meta = &tc->page_meta[ page ]; + uint resident = FD_VOLATILE_CONST( meta->frame ); + if( FD_LIKELY( resident!=UINT_MAX ) ) { + if( FD_UNLIKELY( !meta->ref ) ) meta->ref = 1U; + return &tc->txnpages[ resident ]; + } + + /* Loading changes the frame cache, which needs the write lock. */ + if( FD_LIKELY( !write_lock ) ) { + FD_ATOMIC_FETCH_AND_ADD( &tc->shmem->page_miss_cnt, 1UL ); + return NULL; + } + FD_TEST( pageshmem->txnpages_max ); + + /* A fetch may be loading or writing back the page. */ + page_wait_idle( meta ); + if( FD_UNLIKELY( meta->frame!=UINT_MAX ) ) { + if( FD_UNLIKELY( !meta->ref ) ) meta->ref = 1U; + return &tc->txnpages[ meta->frame ]; + } + + uint victim; + uint frame; + while( FD_UNLIKELY( (frame=frame_take( tc, &victim ))==UINT_MAX ) ) FD_SPIN_PAUSE(); + if( victim!=UINT_MAX ) { + fd_txncache_page_meta_t * old = &tc->page_meta[ victim ]; + if( old->dirty ) { + spill_io( tc, victim, &tc->txnpages[ frame ], 1 ); + old->dirty = 0U; + old->disk_valid = 1; + } + } + + /* Read the page back, or start it empty if it was never written. */ + fd_txncache_txnpage_t * data = &tc->txnpages[ frame ]; + if( meta->disk_valid ) { + spill_io( tc, page, data, 0 ); + } else { + data->free = FD_TXNCACHE_TXNS_PER_PAGE; + } + tc->frame_page[ frame ] = (uint)page; + meta->frame = frame; + meta->ref = 1U; + return data; +} + +/* page_fetch loads page into a frame for a caller that does not hold + the lock, so that its spill I/O does not block other threads. It + takes the write lock only to reserve the frame, marking the page and + any dirty evicted page busy, and publishes them without the lock. + seq is the release_seq the caller saw while it held the lock and + found page in use. If any page has been released since, page may + no longer be in use and the fetch is abandoned. The caller retakes + the lock and checks again, since another thread may evict page + before then. */ + +static void +page_fetch( fd_txncache_t * tc, + ulong page, + ulong seq ) { + fd_txncache_page_meta_t * meta = &tc->page_meta[ page ]; + lock_write( tc ); + if( FD_UNLIKELY( tc->shmem->release_seq!=seq || meta->frame!=UINT_MAX ) ) { + fd_rwlock_unwrite( tc->shmem->lock ); + return; + } + if( FD_UNLIKELY( meta->io ) ) { + fd_rwlock_unwrite( tc->shmem->lock ); + page_wait_idle( meta ); + return; + } + uint victim; + uint frame = frame_take( tc, &victim ); + if( FD_UNLIKELY( frame==UINT_MAX ) ) { + fd_rwlock_unwrite( tc->shmem->lock ); + FD_SPIN_PAUSE(); + return; + } + fd_txncache_page_meta_t * old = victim!=UINT_MAX ? &tc->page_meta[ victim ] : NULL; + int write = old && old->dirty; + if( write ) { + old->dirty = 0U; + old->io = 1U; + } + int read = meta->disk_valid; + meta->io = 1U; + tc->frame_page[ frame ] = (uint)page; + fd_rwlock_unwrite( tc->shmem->lock ); + + fd_racesan_hook( "txncache:fetch_io" ); + fd_txncache_txnpage_t * data = &tc->txnpages[ frame ]; + if( write ) spill_io( tc, victim, data, 1 ); + if( read ) spill_io( tc, page, data, 0 ); + else data->free = FD_TXNCACHE_TXNS_PER_PAGE; + + /* Publish without the lock, as write lock holders may be waiting for + these pages to become idle. */ + if( write ) { + old->disk_valid = 1; + FD_HW_MFENCE_ST(); + FD_VOLATILE( old->io ) = 0U; + } + meta->ref = 1U; + FD_HW_MFENCE_ST(); + FD_VOLATILE( meta->frame ) = frame; + FD_HW_MFENCE_ST(); + FD_VOLATILE( meta->io ) = 0U; +} + +/* A freed logical page must never reload contents from its former owner. */ +static void +release_page( fd_txncache_t * tc, + ulong page ) { + fd_txncache_page_meta_t * meta = &tc->page_meta[ page ]; + page_wait_idle( meta ); + tc->shmem->release_seq++; + if( meta->frame!=UINT_MAX ) { + tc->frame_page[ meta->frame ] = UINT_MAX; + tc->frames_free[ tc->shmem->frames_free_cnt++ ] = meta->frame; + meta->frame = UINT_MAX; + } + meta->dirty = 0U; + meta->disk_valid = 0; + meta->next = UINT_MAX; + tc->txnpages_free[ tc->shmem->txnpages_free_cnt++ ] = (uint)page; +} + void fd_txncache_reset( fd_txncache_t * tc ) { - fd_rwlock_write( tc->shmem->lock ); - fd_txncache_mutation_begin( tc->shmem ); - __atomic_store_n( &tc->shmem->root_gen, tc->shmem->root_gen+1UL, __ATOMIC_RELAXED ); + lock_write( tc ); tc->shmem->root_cnt = 0UL; root_slist_remove_all( tc->shmem->root_ll, tc->blockcache_shmem_pool ); - tc->shmem->txnpages_free_cnt = tc->shmem->max_txnpages; - for( ulong i=0UL; ishmem->max_txnpages; i++ ) fd_txncache_txnpage_idx_st( tc->shmem->txnpage_idx_sz, tc->txnpages_free, i, i ); + tc->shmem->txnpages_free_cnt = tc->shmem->txnpages_max; + tc->shmem->frames_free_cnt = tc->shmem->frames_max; + tc->shmem->evict_next = 0UL; + tc->shmem->release_seq++; + for( ulong i=0UL; ishmem->txnpages_max; i++ ) page_wait_idle( &tc->page_meta[ i ] ); + for( ulong i=0UL; ishmem->txnpages_max; i++ ) { + tc->txnpages_free[ i ] = (uint)i; + tc->page_meta[ i ].frame = UINT_MAX; + tc->page_meta[ i ].dirty = 0U; + tc->page_meta[ i ].disk_valid = 0; + tc->page_meta[ i ].next = UINT_MAX; + } + for( ulong i=0UL; ishmem->frames_max; i++ ) { + tc->frame_page[ i ] = UINT_MAX; + tc->frames_free[ i ] = (uint)i; + } blockcache_pool_reset( tc->blockcache_shmem_pool ); blockhash_map_reset( tc->blockhash_map ); - fd_txncache_mutation_end( tc->shmem ); fd_rwlock_unwrite( tc->shmem->lock ); } +void +fd_txncache_metrics_snap( fd_txncache_t const * tc, + fd_txncache_metrics_t * out ) { + fd_txncache_shmem_t const * shmem = tc->shmem; + out->page_miss_cnt = FD_VOLATILE_CONST( shmem->page_miss_cnt ); + out->page_spilled_cnt = FD_VOLATILE_CONST( shmem->page_spilled_cnt ); + out->page_spill_read_cnt = FD_VOLATILE_CONST( shmem->page_spill_read_cnt ); + out->frame_evicted_cnt = FD_VOLATILE_CONST( shmem->frame_evicted_cnt ); + out->spill_ticks = FD_VOLATILE_CONST( shmem->spill_ticks ); + out->lock_read_wait_ticks = FD_VOLATILE_CONST( shmem->lock_read_wait_ticks ); + out->lock_write_wait_ticks = FD_VOLATILE_CONST( shmem->lock_write_wait_ticks ); + out->page_capacity = shmem->txnpages_max; + out->page_used = shmem->txnpages_max-fd_ulong_min( FD_VOLATILE_CONST( shmem->txnpages_free_cnt ), shmem->txnpages_max ); + out->frame_capacity = shmem->frames_max; + out->frame_used = shmem->frames_max-fd_ulong_min( FD_VOLATILE_CONST( shmem->frames_free_cnt ), shmem->frames_max ); +} + void * fd_txncache_snapin_scratch( fd_txncache_t * tc, ulong * out_sz ) { - FD_TEST_ERR( tc->shmem->txnpages_free_cnt==tc->shmem->max_txnpages ); - *out_sz = (ulong)tc->shmem->max_txnpages*sizeof(fd_txncache_txnpage_t); + FD_TEST_ERR( tc->shmem->txnpages_free_cnt==tc->shmem->txnpages_max ); + *out_sz = tc->shmem->frames_max*sizeof(fd_txncache_txnpage_t); return tc->txnpages; } @@ -143,64 +409,75 @@ fd_txncache_bucket( fd_txncache_t const * tc, return fd_ulong_hash( FD_LOAD( ulong, txnhash )^tc->shmem->seed )%tc->shmem->bucket_cnt; } +/* Returns a resident page of blockcache with room, with its index in + *out_idx. Without the write lock, returns NULL with the index of a + page of blockcache to fetch in *out_idx, or ULONG_MAX if the caller + must retry under the write lock. */ + static fd_txncache_txnpage_t * fd_txncache_ensure_txnpage( fd_txncache_t * tc, - blockcache_t * blockcache ) { - ulong page_cnt = blockcache->shmem->pages_cnt; - if( FD_UNLIKELY( page_cnt>tc->shmem->txnpages_per_blockhash_max ) ) return NULL; - - ulong idx_sz = tc->shmem->txnpage_idx_sz; - if( FD_LIKELY( page_cnt ) ) { - ulong txnpage_idx = fd_txncache_txnpage_idx_ld( idx_sz, blockcache->pages, page_cnt-1UL ); - ushort txnpage_free = tc->txnpages[ txnpage_idx ].free; - if( FD_LIKELY( txnpage_free ) ) return &tc->txnpages[ txnpage_idx ]; - } - - if( FD_UNLIKELY( page_cnt==tc->shmem->txnpages_per_blockhash_max ) ) return NULL; - ulong idx_null = idx_sz==sizeof(uint) ? UINT_MAX : USHORT_MAX; - int claimed = idx_sz==sizeof(uint) ? FD_ATOMIC_CAS( (uint *)blockcache->pages+page_cnt, UINT_MAX, UINT_MAX-1U )==UINT_MAX - : FD_ATOMIC_CAS( (ushort *)blockcache->pages+page_cnt, (ushort)USHORT_MAX, (ushort)(USHORT_MAX-1U) )==(ushort)USHORT_MAX; - if( FD_LIKELY( claimed ) ) { - ulong txnpages_free_cnt = tc->shmem->txnpages_free_cnt; - for(;;) { - if( FD_UNLIKELY( !txnpages_free_cnt ) ) { - fd_txncache_txnpage_idx_st( idx_sz, blockcache->pages, page_cnt, idx_null ); - FD_COMPILER_MFENCE(); - return NULL; + blockcache_t * blockcache, + int write, + ulong * out_idx ) { + fd_txncache_shmem_t * shmem = tc->shmem; + for(;;) { + /* Use the newest page if it has room. */ + uint head = FD_VOLATILE_CONST( blockcache->shmem->page_head ); + if( FD_LIKELY( head!=UINT_MAX ) ) { + fd_txncache_txnpage_t * page = page_io( tc, head, write ); + *out_idx = head; + if( FD_UNLIKELY( !page ) ) return NULL; + if( FD_LIKELY( FD_VOLATILE_CONST( page->free ) ) ) return page; + } + + /* Need a new page. Try to make sure we can safely take the lock + and that there's enough capacity. */ + if( !write ) { + fd_rwlock_write( shmem->alloc_lock ); + if( FD_UNLIKELY( FD_VOLATILE_CONST( blockcache->shmem->page_head )!=head ) ) { + fd_rwlock_unwrite( shmem->alloc_lock ); + continue; } - ulong old_txnpages_free_cnt = FD_ATOMIC_CAS( &tc->shmem->txnpages_free_cnt, txnpages_free_cnt, txnpages_free_cnt-1UL ); - if( FD_LIKELY( old_txnpages_free_cnt==txnpages_free_cnt ) ) break; - txnpages_free_cnt = old_txnpages_free_cnt; - FD_SPIN_PAUSE(); } - ulong txnpage_idx = fd_txncache_txnpage_idx_ld( idx_sz, tc->txnpages_free, txnpages_free_cnt-1UL ); - fd_txncache_txnpage_t * txnpage = &tc->txnpages[ txnpage_idx ]; - txnpage->free = FD_TXNCACHE_TXNS_PER_PAGE; - FD_HW_MFENCE_ST(); - fd_txncache_txnpage_idx_st( idx_sz, blockcache->pages, page_cnt, txnpage_idx ); - FD_HW_MFENCE_ST(); - blockcache->shmem->pages_cnt = page_cnt+1UL; - return txnpage; - } else { - ulong txnpage_idx = fd_txncache_txnpage_idx_ld( idx_sz, blockcache->pages, page_cnt ); - while( FD_UNLIKELY( txnpage_idx==idx_null-1UL ) ) { - txnpage_idx = fd_txncache_txnpage_idx_ld( idx_sz, blockcache->pages, page_cnt ); - FD_SPIN_PAUSE(); + ulong page_cnt = blockcache->shmem->pages_cnt; + if( FD_UNLIKELY( page_cnt>=shmem->txnpages_per_blockhash_max || !shmem->txnpages_free_cnt ) ) { + if( !write ) fd_rwlock_unwrite( shmem->alloc_lock ); + *out_idx = ULONG_MAX; + return NULL; } - if( FD_UNLIKELY( txnpage_idx==idx_null ) ) return NULL; - return &tc->txnpages[ txnpage_idx ]; + + /* We have safe and exclusive access to allocating a new page. + Without the write lock and a free frame, the page is allocated + not resident, and fetching it starts it empty. */ + ulong idx = tc->txnpages_free[ --shmem->txnpages_free_cnt ]; + fd_txncache_txnpage_t * page = NULL; + if( write ) { + page = page_io( tc, idx, 1 ); + } else if( FD_LIKELY( shmem->frames_free_cnt ) ) { + uint frame = tc->frames_free[ --shmem->frames_free_cnt ]; + tc->frame_page[ frame ] = (uint)idx; + tc->page_meta[ idx ].frame = frame; + page = &tc->txnpages[ frame ]; + } + if( FD_LIKELY( page ) ) page->free = FD_TXNCACHE_TXNS_PER_PAGE; + tc->page_meta[ idx ].next = head; + blockcache->shmem->pages_cnt = page_cnt+1UL; + FD_HW_MFENCE_ST(); + FD_VOLATILE( blockcache->shmem->page_head ) = (uint)idx; + if( !write ) fd_rwlock_unwrite( shmem->alloc_lock ); + *out_idx = idx; + return page; } } -static int +static inline int fd_txncache_insert_txn( fd_txncache_t * tc, blockcache_t * blockcache, fd_txncache_txnpage_t * txnpage, + ulong txnpage_idx, fd_txncache_fork_id_t fork_id, uchar const * txnhash ) { - ulong txnpage_idx = (ulong)(txnpage - tc->txnpages); - for(;;) { ushort txnpage_free = txnpage->free; if( FD_UNLIKELY( !txnpage_free ) ) return 0; @@ -225,6 +502,8 @@ fd_txncache_insert_txn( fd_txncache_t * tc, FD_SPIN_PAUSE(); } + /* This races, but we expect x86 stores to be atomic. */ + if( FD_UNLIKELY( !tc->page_meta[ txnpage_idx ].dirty ) ) tc->page_meta[ txnpage_idx ].dirty = 1U; return 1; } } @@ -232,7 +511,7 @@ fd_txncache_insert_txn( fd_txncache_t * tc, fd_txncache_fork_id_t fd_txncache_attach_child( fd_txncache_t * tc, fd_txncache_fork_id_t parent_fork_id ) { - fd_rwlock_write( tc->shmem->lock ); + lock_write( tc ); FD_TEST( blockcache_pool_free( tc->blockcache_shmem_pool ) ); ulong idx = blockcache_pool_idx_acquire( tc->blockcache_shmem_pool ); @@ -268,7 +547,7 @@ fd_txncache_attach_child( fd_txncache_t * tc, fork->shmem->frozen = 0; memset( fork->heads, 0xFF, tc->shmem->bucket_cnt*sizeof(uint) ); fork->shmem->pages_cnt = 0UL; - memset( fork->pages, 0xFF, tc->shmem->txnpages_per_blockhash_max*tc->shmem->txnpage_idx_sz ); + fork->shmem->page_head = UINT_MAX; fd_rwlock_unwrite( tc->shmem->lock ); return fork_id; @@ -278,7 +557,7 @@ void fd_txncache_attach_blockhash( fd_txncache_t * tc, fd_txncache_fork_id_t fork_id, uchar const * blockhash ) { - fd_rwlock_write( tc->shmem->lock ); + lock_write( tc ); blockcache_t * fork = &tc->blockcache_pool[ fork_id.val ]; FD_TEST( !fork->shmem->frozen ); @@ -296,7 +575,7 @@ fd_txncache_finalize_fork( fd_txncache_t * tc, fd_txncache_fork_id_t fork_id, ulong txnhash_offset, uchar const * blockhash ) { - fd_rwlock_write( tc->shmem->lock ); + lock_write( tc ); blockcache_t * fork = &tc->blockcache_pool[ fork_id.val ]; FD_TEST( fork->shmem->frozen<=1 ); @@ -315,9 +594,13 @@ static inline void remove_blockcache( fd_txncache_t * tc, blockcache_t * blockcache ) { FD_TEST( blockcache->shmem->frozen>=0 ); - ulong idx_sz = tc->shmem->txnpage_idx_sz; - memcpy( (uchar *)tc->txnpages_free+tc->shmem->txnpages_free_cnt*idx_sz, blockcache->pages, blockcache->shmem->pages_cnt*idx_sz ); - tc->shmem->txnpages_free_cnt += blockcache->shmem->pages_cnt; + for( uint page=blockcache->shmem->page_head; page!=UINT_MAX; ) { + uint next = tc->page_meta[ page ].next; + release_page( tc, page ); + page = next; + } + blockcache->shmem->page_head = UINT_MAX; + blockcache->shmem->pages_cnt = 0UL; ulong idx = blockcache_pool_idx( tc->blockcache_shmem_pool, blockcache->shmem ); for( ulong i=0UL; ishmem->active_slots_max; i++ ) descends_set_remove( tc->blockcache_pool[ i ].descends, idx ); @@ -346,7 +629,7 @@ remove_children( fd_txncache_t * tc, void fd_txncache_cancel_fork( fd_txncache_t * tc, fd_txncache_fork_id_t fork_id ) { - fd_rwlock_write( tc->shmem->lock ); + lock_write( tc ); blockcache_t * fork = &tc->blockcache_pool[ fork_id.val ]; FD_TEST( fork->shmem->parent_id.val!=USHORT_MAX ); @@ -368,7 +651,7 @@ fd_txncache_cancel_fork( fd_txncache_t * tc, void fd_txncache_advance_root( fd_txncache_t * tc, fd_txncache_fork_id_t fork_id ) { - fd_rwlock_write( tc->shmem->lock ); + lock_write( tc ); blockcache_t * fork = &tc->blockcache_pool[ fork_id.val ]; FD_TEST( fork->shmem->parent_id.val!=USHORT_MAX ); @@ -390,9 +673,6 @@ fd_txncache_advance_root( fd_txncache_t * tc, parent_blockhash_b58, fork_blockhash_b58 )); - fd_txncache_mutation_begin( tc->shmem ); - __atomic_store_n( &tc->shmem->root_gen, tc->shmem->root_gen+1UL, __ATOMIC_RELAXED ); - /* When a fork is rooted, any competing forks can be immediately removed as they will not be needed again. This includes child forks of the pruned siblings as well. */ @@ -416,7 +696,6 @@ fd_txncache_advance_root( fd_txncache_t * tc, tc->shmem->root_cnt--; } - fd_txncache_mutation_end( tc->shmem ); fd_rwlock_unwrite( tc->shmem->lock ); } @@ -439,29 +718,33 @@ static void purge_stale_on_blockcache( fd_txncache_t * tc, blockcache_t * blockcache ) { FD_TEST( blockcache->shmem->frozen>=0 ); - ulong idx_sz = tc->shmem->txnpage_idx_sz; memset( tc->scratch_heads, 0xFF, tc->shmem->bucket_cnt*sizeof(tc->scratch_heads[ 0 ]) ); - memset( tc->scratch_pages, 0xFF, tc->shmem->txnpages_per_blockhash_max*idx_sz ); + uint scratch_page_head = UINT_MAX; ulong scratch_pages_cnt = 0UL; ulong scratch_txnpage_idx = ULONG_MAX; tc->scratch_txnpage->free = 0; - for( ulong i=0UL; ishmem->pages_cnt; i++ ) { - ulong curr_txnpage_idx = fd_txncache_txnpage_idx_ld( idx_sz, blockcache->pages, blockcache->shmem->pages_cnt-i-1UL ); - ulong curr_txn_cnt = FD_TXNCACHE_TXNS_PER_PAGE-tc->txnpages[ curr_txnpage_idx ].free; + uint next_txnpage_idx = blockcache->shmem->page_head; + while( next_txnpage_idx!=UINT_MAX ) { + ulong curr_txnpage_idx = next_txnpage_idx; + next_txnpage_idx = tc->page_meta[ curr_txnpage_idx ].next; + fd_txncache_txnpage_t * input = tc->scratch_input; + memcpy( input, page_io( tc, curr_txnpage_idx, 1 ), sizeof(*input) ); + ulong curr_txn_cnt = FD_TXNCACHE_TXNS_PER_PAGE-input->free; for( ulong j=0UL; jtxnpages[ curr_txnpage_idx ].txns[ curr_txn_cnt-j-1UL ]; + fd_txncache_single_txn_t const * curr_txn = input->txns[ curr_txn_cnt-j-1UL ]; blockcache_t const * txn_fork = &tc->blockcache_pool[ curr_txn->fork_id.val ]; if( FD_LIKELY( txn_fork->shmem->frozen>=0 && txn_fork->shmem->generation==curr_txn->generation ) ) { /* Valid transaction. Keep. */ if( FD_UNLIKELY( !tc->scratch_txnpage->free ) ) { FD_TEST( scratch_txnpage_idx!=curr_txnpage_idx ); if( FD_LIKELY( scratch_txnpage_idx!=ULONG_MAX ) ) { - fd_txncache_txnpage_t * txnpage = &tc->txnpages[ scratch_txnpage_idx ]; - memcpy( txnpage, tc->scratch_txnpage, sizeof(*txnpage) ); + memcpy( page_io( tc, scratch_txnpage_idx, 1 ), tc->scratch_txnpage, sizeof(fd_txncache_txnpage_t) ); + tc->page_meta[ scratch_txnpage_idx ].dirty = 1U; } scratch_txnpage_idx = curr_txnpage_idx; tc->scratch_txnpage->free = FD_TXNCACHE_TXNS_PER_PAGE; - fd_txncache_txnpage_idx_st( idx_sz, tc->scratch_pages, scratch_pages_cnt, scratch_txnpage_idx ); + tc->page_meta[ scratch_txnpage_idx ].next = scratch_page_head; + scratch_page_head = (uint)scratch_txnpage_idx; scratch_pages_cnt++; } ulong txn_idx = FD_TXNCACHE_TXNS_PER_PAGE-tc->scratch_txnpage->free; @@ -480,16 +763,15 @@ purge_stale_on_blockcache( fd_txncache_t * tc, } if( FD_UNLIKELY( curr_txnpage_idx!=scratch_txnpage_idx ) ) { /* The txnpage is not being used for compaction, free it up. */ - fd_txncache_txnpage_idx_st( idx_sz, tc->txnpages_free, tc->shmem->txnpages_free_cnt, curr_txnpage_idx ); - tc->shmem->txnpages_free_cnt++; + release_page( tc, curr_txnpage_idx ); } } if( FD_LIKELY( scratch_txnpage_idx!=ULONG_MAX ) ) { - fd_txncache_txnpage_t * txnpage = &tc->txnpages[ scratch_txnpage_idx ]; - memcpy( txnpage, tc->scratch_txnpage, sizeof(*txnpage) ); + memcpy( page_io( tc, scratch_txnpage_idx, 1 ), tc->scratch_txnpage, sizeof(fd_txncache_txnpage_t) ); + tc->page_meta[ scratch_txnpage_idx ].dirty = 1U; } + blockcache->shmem->page_head = scratch_page_head; blockcache->shmem->pages_cnt = scratch_pages_cnt; - memcpy( blockcache->pages, tc->scratch_pages, tc->shmem->txnpages_per_blockhash_max*idx_sz ); memcpy( blockcache->heads, tc->scratch_heads, tc->shmem->bucket_cnt*sizeof(blockcache->heads[0]) ); } @@ -518,9 +800,7 @@ purge_stale( fd_txncache_t * tc ) { transactions in any blockcache descending from that. Unfortunately, frontier eviction means that any blockcache in the fork tree can have stale transactions. */ - fd_txncache_mutation_begin( tc->shmem ); purge_stale_on_fork( tc, root ); - fd_txncache_mutation_end( tc->shmem ); FD_LOG_WARNING(( "purge_stale: txnpages_free %lu -> %lu", free_before, tc->shmem->txnpages_free_cnt )); } @@ -529,7 +809,8 @@ fd_txncache_insert( fd_txncache_t * tc, fd_txncache_fork_id_t fork_id, uchar const * blockhash, uchar const * txnhash ) { - fd_rwlock_read( tc->shmem->lock ); + long t0 = fd_tickcount(); + lock_read( tc ); blockcache_t const * fork = &tc->blockcache_pool[ fork_id.val ]; FD_TEST( fork->shmem->frozen<=1 ); @@ -537,8 +818,25 @@ fd_txncache_insert( fd_txncache_t * tc, blockcache_t * blockcache = blockhash_on_fork( tc, fork, blockhash ); FD_TEST( blockcache ); + ulong fetch_cnt = 0UL; + int miss = 0; + int write = 0; for(;;) { - fd_txncache_txnpage_t * txnpage = fd_txncache_ensure_txnpage( tc, blockcache ); + /* First look for an in-memory page (write==0) */ + ulong txnpage_idx; + fd_txncache_txnpage_t * txnpage = fd_txncache_ensure_txnpage( tc, blockcache, 0, &txnpage_idx ); + if( FD_UNLIKELY( !txnpage && txnpage_idx!=ULONG_MAX ) ) miss = 1; + if( FD_UNLIKELY( !txnpage && txnpage_idx!=ULONG_MAX && fetch_cntshmem->release_seq; + fd_rwlock_unread( tc->shmem->lock ); + page_fetch( tc, txnpage_idx, seq ); + fetch_cnt++; + lock_read( tc ); + blockcache = blockhash_on_fork( tc, fork, blockhash ); + FD_TEST( blockcache ); + continue; + } if( FD_UNLIKELY( !txnpage ) ) { /* Because of sizing invariants when creating the structure, it is not typically possible to fill it, unless there are stale @@ -548,27 +846,41 @@ fd_txncache_insert( fd_txncache_t * tc, page still unavailable after the purge means the cache is undersized for the caller's usage. */ fd_rwlock_unread( tc->shmem->lock ); - fd_rwlock_write( tc->shmem->lock ); - if( FD_LIKELY( !fd_txncache_ensure_txnpage( tc, blockcache ) ) ) { + lock_write( tc ); + blockcache = blockhash_on_fork( tc, fork, blockhash ); + FD_TEST( blockcache ); + /* Load or allocate a frame under exclusive access. */ + txnpage = fd_txncache_ensure_txnpage( tc, blockcache, 1, &txnpage_idx ); + if( FD_UNLIKELY( !txnpage ) ) { + /* Slow path: there is no available txnpage in-memory or on + disk. Try to purge any stale entries and try one more time. + If that fails, capacity has been reached. */ purge_stale( tc ); - if( FD_UNLIKELY( !fd_txncache_ensure_txnpage( tc, blockcache ) ) ) + txnpage = fd_txncache_ensure_txnpage( tc, blockcache, 1, &txnpage_idx ); + if( FD_UNLIKELY( !txnpage ) ) FD_LOG_ERR(( "txncache full after purging stale entries: blockcache holds %lu of %lu txnpages, pool has %lu of %lu free (active_slots_max=%lu txn_per_slot_max=%lu)", blockcache->shmem->pages_cnt, tc->shmem->txnpages_per_blockhash_max, - tc->shmem->txnpages_free_cnt, tc->shmem->max_txnpages, + tc->shmem->txnpages_free_cnt, tc->shmem->txnpages_max, tc->shmem->active_slots_max, tc->shmem->txn_per_slot_max )); } - fd_rwlock_unwrite( tc->shmem->lock ); - fd_rwlock_read( tc->shmem->lock ); - continue; + FD_TEST( fd_txncache_insert_txn( tc, blockcache, txnpage, txnpage_idx, fork_id, txnhash ) ); + write = 1; + break; } - int success = fd_txncache_insert_txn( tc, blockcache, txnpage, fork_id, txnhash ); + /* Fast path: insert the transaction into an in-memory page. */ + int success = fd_txncache_insert_txn( tc, blockcache, txnpage, txnpage_idx, fork_id, txnhash ); if( FD_LIKELY( success ) ) break; FD_SPIN_PAUSE(); } - fd_rwlock_unread( tc->shmem->lock ); + if( FD_UNLIKELY( write ) ) fd_rwlock_unwrite( tc->shmem->lock ); + else fd_rwlock_unread ( tc->shmem->lock ); + + tc->metrics->insert_miss_cnt += (ulong)miss; + tc->metrics->stalled_cnt += (ulong)write; + fd_histf_sample( tc->metrics->insert_ticks, (ulong)(fd_tickcount()-t0) ); } int @@ -576,29 +888,194 @@ fd_txncache_query( fd_txncache_t * tc, fd_txncache_fork_id_t fork_id, uchar const * blockhash, uchar const * txnhash ) { - fd_rwlock_read( tc->shmem->lock ); + long t0 = fd_tickcount(); + /* Prevent fork removal and pruning while allowing inserts. */ + lock_read( tc ); + /* Find the blockhash cache visible from the requested fork. */ blockcache_t const * fork = &tc->blockcache_pool[ fork_id.val ]; FD_TEST( fork->shmem->frozen>=0 ); blockcache_t const * blockcache = blockhash_on_fork( tc, fork, blockhash ); FD_TEST( blockcache ); FD_TEST( blockcache->shmem->frozen==2 ); - int found = 0; + int found = 0; + int write = 0; + int miss = 0; + ulong fetch_cnt = 0UL; + /* Select the bucket using the stored portion of the txn hash. */ ulong txnhash_offset = blockcache->shmem->txnhash_offset; ulong head_hash = fd_txncache_bucket( tc, txnhash+txnhash_offset ); - for( uint head=blockcache->heads[ head_hash ]; head!=UINT_MAX; head=tc->txnpages[ head/FD_TXNCACHE_TXNS_PER_PAGE ].txns[ head%FD_TXNCACHE_TXNS_PER_PAGE ]->blockcache_next ) { - fd_txncache_single_txn_t * txn = tc->txnpages[ head/FD_TXNCACHE_TXNS_PER_PAGE ].txns[ head%FD_TXNCACHE_TXNS_PER_PAGE ]; + /* Walk the bucket's list. UINT_MAX marks the end. */ + for( uint head=blockcache->heads[ head_hash ]; head!=UINT_MAX; ) { + /* If the page is not resident, load it without the lock and walk + again, or after too many loads walk under the write lock, which + loads pages in place. */ + fd_txncache_txnpage_t * page = page_io( tc, head/FD_TXNCACHE_TXNS_PER_PAGE, write ); + if( FD_UNLIKELY( !page ) ) { + miss = 1; + ulong seq = tc->shmem->release_seq; + fd_rwlock_unread( tc->shmem->lock ); + if( FD_LIKELY( fetch_cntheads[ head_hash ]; + continue; + } + fd_txncache_single_txn_t const * txn = page->txns[ head%FD_TXNCACHE_TXNS_PER_PAGE ]; + head = txn->blockcache_next; + /* Accept transactions from this fork or its ancestors. Reject + removed forks and old records whose fork ID has been reused. */ blockcache_t const * txn_fork = &tc->blockcache_pool[ txn->fork_id.val ]; int descends = (txn->fork_id.val==fork_id.val || descends_set_test( fork->descends, txn->fork_id.val )) && txn_fork->shmem->frozen>=0 && txn_fork->shmem->generation==txn->generation; + /* Different hashes can share a bucket, so compare the stored hash. */ if( FD_LIKELY( descends && !memcmp( txnhash+txnhash_offset, txn->txnhash, 20UL ) ) ) { found = 1; break; } } - fd_rwlock_unread( tc->shmem->lock ); + if( FD_UNLIKELY( write ) ) fd_rwlock_unwrite( tc->shmem->lock ); + else fd_rwlock_unread ( tc->shmem->lock ); + + tc->metrics->query_miss_cnt += (ulong)miss; + tc->metrics->stalled_cnt += (ulong)write; + fd_histf_sample( tc->metrics->query_ticks, (ulong)(fd_tickcount()-t0) ); return found; } + +ulong +fd_txncache_snapshot_roots( fd_txncache_t * tc, + fd_txncache_root_t * out ) { + lock_read( tc ); + ulong cnt = 0UL; + for( ulong it = root_slist_iter_init( tc->shmem->root_ll, tc->blockcache_shmem_pool ); + !root_slist_iter_done( it, tc->shmem->root_ll, tc->blockcache_shmem_pool ); + it = root_slist_iter_next( it, tc->shmem->root_ll, tc->blockcache_shmem_pool ) ) { + ulong blockcache_idx = root_slist_iter_idx( it, tc->shmem->root_ll, tc->blockcache_shmem_pool ); + fd_txncache_blockcache_shmem_t const * blockcache_shmem = &tc->blockcache_shmem_pool[ blockcache_idx ]; + FD_TEST( cntblockcache_idx = blockcache_idx; + root->txnhash_offset = blockcache_shmem->txnhash_offset; + root->generation = blockcache_shmem->generation; + memcpy( root->blockhash, blockcache_shmem->blockhash.uc, 32UL ); + } + fd_rwlock_unread( tc->shmem->lock ); + return cnt; +} + +static fd_txncache_single_txn_t const * +bucket_chain_txn( fd_txncache_t * tc, + uint txn_gidx, + int write_lock ) { + if( FD_UNLIKELY( txn_gidx==UINT_MAX ) ) return NULL; + fd_txncache_txnpage_t const * page = page_io( tc, txn_gidx/FD_TXNCACHE_TXNS_PER_PAGE, write_lock ); + FD_TEST( page ); + return page->txns[ txn_gidx%FD_TXNCACHE_TXNS_PER_PAGE ]; +} + +/* Locks tc and positions iter at the head of its current bucket. */ + +static void +blockcache_iter_bucket_open( fd_txncache_blockcache_iter_t * iter ) { + fd_txncache_t * tc = iter->tc; + fd_rwlock_t * lock = tc->shmem->lock; + uint const * heads = tc->blockcache_pool[ iter->blockcache_idx ].heads; + ulong bucket = iter->bucket; + + fd_racesan_hook( "txncache:bucket_started" ); + lock_read( tc ); + + uint head; + int write_lock = 0; + for( ulong fetch_cnt=0UL;; fetch_cnt++ ) { + /* Inserts may publish a new head concurrently under the read lock. + The chain below the head loaded here is immutable until unlock. */ + head = FD_VOLATILE_CONST( heads[ bucket ] ); + + /* Check that every page of the chain is resident. */ + ulong miss = ULONG_MAX; + for( uint idx=head; idx!=UINT_MAX; ) { + fd_txncache_txnpage_t const * page = page_io( tc, idx/FD_TXNCACHE_TXNS_PER_PAGE, 0 ); + if( FD_UNLIKELY( !page ) ) { + miss = idx/FD_TXNCACHE_TXNS_PER_PAGE; + break; + } + idx = page->txns[ idx%FD_TXNCACHE_TXNS_PER_PAGE ]->blockcache_next; + } + if( FD_LIKELY( miss==ULONG_MAX ) ) break; + + /* Load the missing page without the lock and check again, or after + too many loads hold the write lock, which loads pages in place. + The chain may be compacted while unlocked, so the head is + reloaded. */ + ulong seq = tc->shmem->release_seq; + fd_rwlock_unread( lock ); + if( !fetch_cnt ) { fd_racesan_hook( "txncache:bucket_iter_miss" ); } + if( FD_LIKELY( fetch_cntwrite_lock = write_lock; + iter->txn = bucket_chain_txn( tc, head, write_lock ); +} + +static void +blockcache_iter_bucket_close( fd_txncache_blockcache_iter_t * iter ) { + if( iter->write_lock ) fd_rwlock_unwrite( iter->tc->shmem->lock ); + else fd_rwlock_unread ( iter->tc->shmem->lock ); +} + +/* Moves past empty buckets, holding the lock of one bucket at a time. */ + +static void +blockcache_iter_skip_empty( fd_txncache_blockcache_iter_t * iter ) { + while( !iter->txn && iter->bucket+1ULtc->shmem->bucket_cnt ) { + blockcache_iter_bucket_close( iter ); + iter->bucket++; + blockcache_iter_bucket_open( iter ); + } +} + +fd_txncache_blockcache_iter_t * +fd_txncache_blockcache_iter_init( fd_txncache_blockcache_iter_t * iter, + fd_txncache_t * tc, + ulong blockcache_idx ) { + iter->tc = tc; + iter->blockcache_idx = blockcache_idx; + iter->bucket = 0UL; + blockcache_iter_bucket_open( iter ); + blockcache_iter_skip_empty( iter ); + return iter; +} + +void +fd_txncache_blockcache_iter_next( fd_txncache_blockcache_iter_t * iter ) { + /* Under the write lock, loading the next page may evict the current + transaction's frame, so read the link first. */ + uint next = iter->txn->blockcache_next; + iter->txn = bucket_chain_txn( iter->tc, next, iter->write_lock ); + blockcache_iter_skip_empty( iter ); +} + +void +fd_txncache_blockcache_iter_fini( fd_txncache_blockcache_iter_t * iter ) { + blockcache_iter_bucket_close( iter ); +} diff --git a/src/flamenco/runtime/fd_txncache.h b/src/flamenco/runtime/fd_txncache.h index d1d1fccfd45..4ed6eb2e3e4 100644 --- a/src/flamenco/runtime/fd_txncache.h +++ b/src/flamenco/runtime/fd_txncache.h @@ -17,11 +17,11 @@ anything twice. The replay stage queries to make sure that blocks do not contain duplicate transactions. - Both of these operations are concurrent and lockless, assuming there - are no other (non-insert/query) operations occurring on the txn - cache. Most other operations lock the entire structure and will - prevent both insertion and query from proceeding, but are rare - (once per slot) so it's OK. + Queries and inserts on resident pages run concurrently under the + shared lock. A cache miss takes the exclusive lock only to reserve a + frame and reads or writes the spill file without the lock, so it + does not stall other threads. Structural operations take the + exclusive lock. Dirty pages are written back on eviction. The txn cache is somewhat CPU and memory sensitive. To store message hashes requires 20 bytes (only the first 20 of the 32 bytes of the @@ -108,9 +108,16 @@ are nonce blockhashes (not in the recent blockhashes sysvar). */ #include "fd_txncache_shmem.h" +#include "../../util/hist/fd_histf.h" #define FD_TXNCACHE_ALIGN (128UL) +/* 123454 is vote stakes, 123455 is cost tracker, 123456 is epoch + credits, 123457 is stake delegations, 123458/123459 are store, + 123460/123461 are accdb, and 123462+ are reserved by XDP. */ + +#define FD_TXNCACHE_FD (123453) + struct fd_txncache_private; typedef struct fd_txncache_private fd_txncache_t; @@ -139,6 +146,10 @@ FD_PROTOTYPES_BEGIN with a matching max_live_slots. Returns ljoin on success and NULL on failure (logs details). The caller is not joined on return. + spill_fd is the read/write spill file shared by all joins, opened + with O_DIRECT and kept open by the caller. Pass -1 if the cache is + fully in memory. + fd_txncache_join joins the caller to a txn cache. Assumes ljoin points to the first byte of the local join region holding the state. Returns a local handle to the join on success (this is not @@ -153,7 +164,8 @@ fd_txncache_footprint( ulong max_live_slots ); void * fd_txncache_new( void * ljoin, - fd_txncache_shmem_t * shmem ); + fd_txncache_shmem_t * shmem, + int spill_fd ); fd_txncache_t * fd_txncache_join( void * ljoin ); @@ -161,6 +173,54 @@ fd_txncache_join( void * ljoin ); void fd_txncache_reset( fd_txncache_t * tc ); +/* fd_txncache_metrics_t holds the txncache's spill and lock metrics. + Counts only grow; *_ticks are in fd_tickcount() units. */ + +struct fd_txncache_metrics { + ulong page_miss_cnt; + ulong page_spilled_cnt; + ulong page_spill_read_cnt; + ulong frame_evicted_cnt; + ulong spill_ticks; + ulong lock_read_wait_ticks; + ulong lock_write_wait_ticks; + ulong page_used; + ulong page_capacity; + ulong frame_used; + ulong frame_capacity; +}; + +typedef struct fd_txncache_metrics fd_txncache_metrics_t; + +/* fd_txncache_metrics_snap copies the metrics of tc into out without + taking the lock, so values from different fields may be slightly + out of step. */ + +void +fd_txncache_metrics_snap( fd_txncache_t const * tc, + fd_txncache_metrics_t * out ); + +/* fd_txncache_join_metrics_t holds the metrics of the queries and + inserts made through one join, so each tile can report its own. + Counts only grow. The histograms sample fd_tickcount() durations; + fd_txncache_new formats them with placeholder buckets, which the + tile reformats with its metric's edges before its first query. */ + +struct fd_txncache_join_metrics { + ulong query_miss_cnt; /* Queries that found a page not resident. */ + ulong insert_miss_cnt; /* Inserts that found a page not resident. */ + ulong stalled_cnt; /* Queries and inserts that fell back to the write lock. */ + fd_histf_t query_ticks [1]; /* Duration of each query. */ + fd_histf_t insert_ticks[1]; /* Duration of each insert. */ +}; + +typedef struct fd_txncache_join_metrics fd_txncache_join_metrics_t; + +/* fd_txncache_join_metrics returns the metrics of the join tc. */ + +fd_txncache_join_metrics_t * +fd_txncache_join_metrics( fd_txncache_t * tc ); + /* fd_txncache_snapin_scratch returns the txncache's transaction page pool for use as scratch memory during snapshot loading, and sets *out_sz to its size in bytes. @@ -170,17 +230,27 @@ fd_txncache_reset( fd_txncache_t * tc ); contents. A page is fully reinitialized when an insert allocates it. A caller that has just reset the txncache may therefore use the region freely until it performs the first insert, at which point the - contents should be assumed clobbered. - - The pool is sized to hold at least - FD_TXNCACHE_MAX_SLOT_DELTAS*max_txn_per_slot transactions, so the - region is at least that many times sizeof(fd_txncache_single_txn_t) - bytes regardless of max_live_slots. */ + contents should be assumed clobbered. */ void * fd_txncache_snapin_scratch( fd_txncache_t * tc, ulong * out_sz ); +/* FD_TXNCACHE_SNAPIN_GROUP_SZ is the size of one status cache blockhash + group that snapshot loading stages in the scratch region. */ + +#define FD_TXNCACHE_SNAPIN_GROUP_SZ (40UL) + +/* fd_txncache_snapin_scratch_min_sz returns the scratch size snapshot + loading needs to stage the status cache, one group per transaction + for each slot delta. max_txn_per_slot is the value passed to + fd_txncache_shmem_new, which counts each transaction twice. */ + +FD_FN_CONST static inline ulong +fd_txncache_snapin_scratch_min_sz( ulong max_txn_per_slot ) { + return FD_TXNCACHE_MAX_SLOT_DELTAS*(max_txn_per_slot/2UL)*FD_TXNCACHE_SNAPIN_GROUP_SZ; +} + /* fd_txncache_attach_child notifies the txncache that a new child bank has been created, off some parent. This must be called before any transaction executed on this child fork is inserted. The parent fork diff --git a/src/flamenco/runtime/fd_txncache_private.h b/src/flamenco/runtime/fd_txncache_private.h index f01686d7237..68173ca0d2c 100644 --- a/src/flamenco/runtime/fd_txncache_private.h +++ b/src/flamenco/runtime/fd_txncache_private.h @@ -1,16 +1,17 @@ #ifndef HEADER_fd_src_flamenco_runtime_fd_txncache_private_h #define HEADER_fd_src_flamenco_runtime_fd_txncache_private_h -#include "fd_txncache_shmem.h" +#include "fd_txncache.h" #include "../fd_flamenco_base.h" #include "../fd_rwlock.h" /* The number of transactions in each page. This needs to be high - enough to amoritze the cost of caller code reserving pages from, + enough to amortize the cost of caller code reserving pages from, and returning pages to the pool, but not so high that the memory wasted from blockhashes with only one transaction is significant. */ #define FD_TXNCACHE_TXNS_PER_PAGE (8192UL) +#define FD_TXNCACHE_TXNPAGE_ALIGN (4096UL) /* The maximum distance a transaction blockhash reference can be (inclusive). For example, if no slots were skipped, and the value is @@ -18,13 +19,24 @@ [149, 300). */ #define FD_TXNCACHE_MAX_BLOCKHASH_DISTANCE (151UL) +/* The most roots the txncache holds. root_cnt does not count the + initial root. */ +#define FD_TXNCACHE_MAX_ROOTS (FD_TXNCACHE_MAX_BLOCKHASH_DISTANCE+1UL) + /* The global txn index (page*FD_TXNCACHE_TXNS_PER_PAGE+txn) is a uint with UINT_MAX as the null link, which bounds the txnpage pool. */ #define FD_TXNCACHE_MAX_TXNPAGES (UINT_MAX/FD_TXNCACHE_TXNS_PER_PAGE) +/* The most pages a read-locked operation loads without the lock before + it retries under the write lock. The fallback guarantees progress + when the pages an operation needs do not fit in the frames at + once. */ + +#define FD_TXNCACHE_FETCH_MAX (4UL) + struct __attribute__((packed)) fd_txncache_single_txn { - uint blockcache_next; /* Pointer to the next element in the blockcache hash chain containing this entry from the pool. */ + uint blockcache_next; /* Global txn index of the next element in this blockcache hash chain, or UINT_MAX. */ uint generation; /* The generation of the fork when this transaction was inserted. Used to determine if the transaction is still valid for a fork that might have advanced since insertion. */ @@ -35,18 +47,26 @@ struct __attribute__((packed)) fd_txncache_single_txn { bytes, but is 20 bytes starting at some arbitrary offset given by the txnhash_offset value of the containing blockcache entry. */ }; - typedef struct fd_txncache_single_txn fd_txncache_single_txn_t; FD_STATIC_ASSERT( sizeof(fd_txncache_single_txn_t)==30UL, fd_txncache_single_txn ); -struct fd_txncache_txnpage { +struct __attribute__((aligned(FD_TXNCACHE_TXNPAGE_ALIGN))) fd_txncache_txnpage { ushort free; /* The number of free txn entries in this page. */ fd_txncache_single_txn_t txns[ FD_TXNCACHE_TXNS_PER_PAGE][ 1 ]; /* The transactions in the page. */ }; - typedef struct fd_txncache_txnpage fd_txncache_txnpage_t; +struct fd_txncache_page_meta { + uint frame; /* UINT_MAX if not resident. */ + uint dirty; /* Resident contents changed since the last writeback. */ + int disk_valid; /* A complete page has been written to its disk slot. */ + uint next; /* Next older txnpage of the owning blockcache, or UINT_MAX. */ + uint ref; /* Resident and accessed since the eviction sweep last passed its frame. */ + uint io; /* A fetch is loading or writing back the page without the lock. */ +}; +typedef struct fd_txncache_page_meta fd_txncache_page_meta_t; + struct fd_txncache_blockcache_shmem { fd_txncache_fork_id_t parent_id; fd_txncache_fork_id_t child_id; @@ -70,10 +90,11 @@ struct fd_txncache_blockcache_shmem { which only communicate the offset and the respective 20 bytes. To make sure we are deduplicating incoming transactions correctly, we must replicate this system even though it would be easier to just always take the first 20 bytes. For transactions that we - insert into the cache ourselves, we do just always use a key_offset of zero, so this is - only nonzero when constructed form a peer snapshot. */ + insert into the cache ourselves, we do just always use a txnhash_offset of zero, so this is + only nonzero when constructed from a peer snapshot. */ ulong pages_cnt; /* The number of txnpages currently in use to store the transactions in this blockcache. */ + uint page_head; /* Newest logical page, or UINT_MAX. Older pages follow page_meta[ page ].next. */ struct { ulong next; @@ -122,65 +143,41 @@ typedef struct fd_txncache_blockcache_shmem fd_txncache_blockcache_shmem_t; #include "../../util/tmpl/fd_set_dynamic.c" struct __attribute__((aligned(FD_TXNCACHE_SHMEM_ALIGN))) fd_txncache_shmem_private { - /* The txncache is a concurrent structure and will be accessed by multiple threads - concurrently. Insertion and querying only take a read lock as they can be done - lockless but all other operations will take a write lock internally. - - The lock needs to be aligned to 128 bytes to avoid false sharing with other - data that might be on the same cache line. */ + /* The txncache is a concurrent structure and will be accessed by + multiple threads concurrently. Resident queries and inserts take + a read lock. Structural operations take a write lock. A cache + miss takes the write lock only to reserve a frame and does its + spill I/O without the lock, marking the pages it moves busy (see + page_meta io) until it publishes them. The lock prevents eviction + while a reader uses a frame. + + The lock needs to be aligned to 128 bytes to avoid false sharing + with other data that might be on the same cache line. */ fd_rwlock_t lock[ 1 ] __attribute__((aligned(128UL))); - ulong txn_per_slot_max; - ulong active_slots_max; - ulong bucket_cnt; /* Hash buckets per blockcache. Decoupled from txn_per_slot_max (load - factor 8) to reduce the heads arrays' memory footprint. */ - ulong txnpages_per_blockhash_max; - ulong max_txnpages; - ulong txnpage_idx_sz; /* Element size of every txnpage index array (blockcache->pages, - txnpages_free, scratch_pages), see fd_txncache_txnpage_idx_sz. */ + ulong txn_per_slot_max; + ulong active_slots_max; + ulong bucket_cnt; /* Hash buckets per blockcache, txn_per_slot_max/32 (load factor 16, as live forks hold at most txn_per_slot_max/2). */ + ulong txnpages_per_blockhash_max; + ulong txnpages_max; + ulong frames_max; uint blockcache_generation; /* Incremented for every blockcache. */ - ulong txnpages_free_cnt; /* The number of pages in the txnpages that are not currently in use. */ - - /* Helps the snapshot producer walk the txncache lock-free and detect - relevant changes to the txncache. mutation_gen covers transaction - storage, and root_gen covers the root history. mutation_gen is odd - when the storage is being modified in ways that could affect the - snapshot, such as during reset, root advancement, and compaction, - and bumped once the mutation is done. root_gen increments on root - advancement and reset. - - For the snapshot producer, fd_txncache_writer, mutation_gen changes - trigger retry in the writer, and root_gen changes are treated as - fatal invariant violation because the replay tile is expected to - pause root history changes while a snapshot is in progress. - - Note that attach/cancel of the unrooted frontier does not bump - mutation_gen. Suppose an unrooted child executed transaction T - referencing an old rooted blockhash. Before cancellation, T is - unrooted, so the writer skips it. After cancellation, T is stale, - so the writer still skips it. So the desired snapshot did not - change. - - A note on concurrency ... Accesses to the two generation numbers - are race free under C11. Every store after initialization is - atomic and pairs with an atomic load in the snapshot producer tile. - The loads in the write locked mutators need no atomicity because - every store is made under the write lock so said loads are already - ordered properly with respect to the writes. In contrast, the - transaction entries the snapshot walker loads and the txncache - stores are plain on both sides, and the blockcache bucket heads are - stored plainly by compaction while the walker loads them with an - acquire load. Both are data races under C11. The walker is - nevertheless correct, given that the compiler emits the plain entry - accesses as ordinary loads and stores. FD_HW_MFENCE_ST in - fd_txncache_mutation_begin orders the odd store before the entry - stores and FD_HW_MFENCE_LD in the walker orders the entry loads - before the generation re-check, so an entry load that observed a - store made inside a mutation section is followed by a re-check that - fails, and whatever was read, torn or otherwise, is discarded. */ - ulong mutation_gen; - ulong root_gen; + ulong txnpages_free_cnt; /* Number of unallocated logical pages. */ + ulong frames_free_cnt; /* Free in-memory resident frames. */ + ulong evict_next; /* CLOCK hand: next frame the eviction sweep checks. */ + ulong release_seq; /* Incremented whenever a logical page is released, under the write lock. */ + fd_rwlock_t alloc_lock[ 1 ]; /* Lock for allocating new logical page + frame. */ + + /* Metrics. page_miss_cnt, lock_read_wait_ticks and the spill + counters are updated atomically, the rest under the write lock. */ + ulong page_miss_cnt; /* Read-locked lookups that found their page not resident. */ + ulong page_spilled_cnt; /* Dirty pages written to the spill file. */ + ulong page_spill_read_cnt; /* Pages read back from the spill file. */ + ulong frame_evicted_cnt; /* Frames evicted to load another page. */ + ulong spill_ticks; /* Ticks spent in spill file I/O. */ + ulong lock_read_wait_ticks; /* Ticks spent waiting to take the read lock. */ + ulong lock_write_wait_ticks; /* Ticks spent waiting to take the write lock. */ ulong root_cnt; root_slist_t root_ll[1]; /* A singly linked list of the forks that are roots of fork chains. The tail is the @@ -194,11 +191,9 @@ struct __attribute__((aligned(FD_TXNCACHE_SHMEM_ALIGN))) fd_txncache_shmem_priva struct blockcache { fd_txncache_blockcache_shmem_t * shmem; - uint * heads; /* The hash table for the blockhash. Each entry is a pointer to the head of a linked list of + uint * heads; /* The hash table for the blockhash. Each entry is the global txn index of the head of a linked list of transactions that reference this blockhash. As we add transactions to the bucket, the head pointer is updated to the new item, and the new item is pointed to the previous head. */ - void * pages; /* A list of the txnpages containing the transactions for this blockcache, elements of - shmem->txnpage_idx_sz bytes (see fd_txncache_txnpage_idx_ld). */ descends_set_t * descends; /* Each fork can descend from other forks in the txncache, and this bit vector contains one value for each fork in the txncache. If this fork descends from some other fork F, then @@ -211,51 +206,28 @@ struct fd_txncache_private { fd_txncache_shmem_t * shmem; fd_txncache_blockcache_shmem_t * blockcache_shmem_pool; - blockcache_t * blockcache_pool; - blockhash_map_t * blockhash_map; + blockcache_t * blockcache_pool; + blockhash_map_t * blockhash_map; + + int spill_fd; - void * txnpages_free; /* The index in the txnpages array that is free, for each of the free pages. - Elements are shmem->txnpage_idx_sz bytes, as are scratch_pages below. */ + fd_txncache_page_meta_t * page_meta; /* Metadata for each logical page */ + uint * txnpages_free; /* Stack of free logical pages */ + uint * frame_page; /* Array mapping frame to logical page */ + uint * frames_free; /* Stack of unused frame indices. */ - fd_txncache_txnpage_t * txnpages; /* The actual storage for the transactions. The blockcache points to these - pages when storing transactions. Transaction are grouped into pages of - size 16384 to make certain allocation and deallocation operations faster - (just the pages are acquired/released, rather than each txn). */ + fd_txncache_txnpage_t * txnpages; /* Aligned resident frames, indexed through page_meta. */ - void * scratch_pages; - uint * scratch_heads; + /* Scratch space used for cleaning up stale entries for compaction. */ + uint * scratch_heads; fd_txncache_txnpage_t * scratch_txnpage; + fd_txncache_txnpage_t * scratch_input; + + fd_txncache_join_metrics_t metrics[1]; }; FD_PROTOTYPES_BEGIN -/* Use these to bracket a write locked change that is visible to the - lock-free snapshot chain walker (fd_txncache_writer). mutation_gen - is odd while a change is in progress. The walker uses an acquire - load of the generation before starting its chain reads and a LoadLoad - barrier before each re-check. */ - -static inline void -fd_txncache_mutation_begin( fd_txncache_shmem_t * shmem ) { - /* Plain load is fine because the write lock holder is the only writer. */ - ulong gen = shmem->mutation_gen; - FD_TEST_CRIT( !(gen&1UL) ); - __atomic_store_n( &shmem->mutation_gen, gen+1UL, __ATOMIC_RELAXED ); - - /* The walker does not write/publish any shared state visible to the - txncache. So no need for a StoreLoad barrier here. Just - StoreStore. */ - FD_HW_MFENCE_ST(); -} - -static inline void -fd_txncache_mutation_end( fd_txncache_shmem_t * shmem ) { - /* Plain load is fine because the write lock holder is the only writer. */ - ulong gen = shmem->mutation_gen; - FD_TEST_CRIT( gen&1UL ); - __atomic_store_n( &shmem->mutation_gen, gen+1UL, __ATOMIC_RELEASE ); -} - /* fd_txncache_max_txnpages{,_per_blockhash} return the txnpage pool size and the per blockcache page cap for the given parameters. The result is not bounded; callers compare against @@ -271,36 +243,91 @@ fd_txncache_max_txnpages( ulong max_active_slots, FD_FN_CONST static inline ulong fd_txncache_bucket_cnt( ulong max_txn_per_slot ) { - return fd_ulong_max( 1UL, (max_txn_per_slot+7UL)/8UL ); + return fd_ulong_max( 1UL, (max_txn_per_slot+31UL)/32UL ); } -/* Txnpage indices are stored as ushort, which covers production - parameters, and widen to uint only when the pool exceeds the ushort - range (development.bench sizing), keeping the production footprint - unchanged. The all-ones value of the width is the null index and - all-ones minus one is the allocation-in-progress flag, so memset 0xFF - nulls an array of either width. ld/st access an index array of the - given element size. */ +/* Snapshot support. These let the snapshot writer read the rooted part + of the txncache without knowing its locking or paging. */ -FD_FN_CONST static inline ulong -fd_txncache_txnpage_idx_sz( ulong max_txnpages ) { - return max_txnpages>USHORT_MAX-2UL ? sizeof(uint) : sizeof(ushort); -} +struct fd_txncache_root { + ulong blockcache_idx; + ulong txnhash_offset; + uint generation; + uchar blockhash[ 32UL ]; +}; + +typedef struct fd_txncache_root fd_txncache_root_t; + +/* fd_txncache_snapshot_roots copies every root, oldest first, to out + and returns the count. out must have room for + FD_TXNCACHE_MAX_ROOTS roots. The caller must keep the + root from advancing, so the newest root is the snapshot root. */ + +ulong +fd_txncache_snapshot_roots( fd_txncache_t * tc, + fd_txncache_root_t * out ); + +/* fd_txncache_blockcache_iter_t iterates the transactions of a + blockcache. It visits one hash chain at a time and holds the + txncache lock for that chain, so the chain cannot be compacted, + freed, or evicted underneath it. Concurrent inserts only prepend to + a chain, so the entries reachable from the head loaded when the chain + is opened do not change while it is held. The lock is released + between chains. + + For each chain, the iterator holds the read lock if every page of the + chain is resident. Otherwise it holds the write lock, under which it + loads pages on demand. + + fd_txncache_blockcache_iter_init locks tc and positions iter at the + first transaction of blockcache blockcache_idx. Returns iter. init + must be paired with fd_txncache_blockcache_iter_fini. The caller + must not call into the txncache between init and fini. + + fd_txncache_blockcache_iter_ele returns the current transaction, + valid until the next call to fd_txncache_blockcache_iter_next or + fd_txncache_blockcache_iter_fini. + + Typical usage: + + fd_txncache_blockcache_iter_t iter[1]; + for( fd_txncache_blockcache_iter_init( iter, tc, blockcache_idx ); + !fd_txncache_blockcache_iter_done( iter ); + fd_txncache_blockcache_iter_next( iter ) ) { + fd_txncache_single_txn_t const * txn = fd_txncache_blockcache_iter_ele( iter ); + ... + } + fd_txncache_blockcache_iter_fini( iter ); */ + +struct fd_txncache_blockcache_iter { + fd_txncache_t * tc; + fd_txncache_single_txn_t const * txn; /* NULL once done */ + ulong blockcache_idx; + ulong bucket; + int write_lock; +}; + +typedef struct fd_txncache_blockcache_iter fd_txncache_blockcache_iter_t; + +fd_txncache_blockcache_iter_t * +fd_txncache_blockcache_iter_init( fd_txncache_blockcache_iter_t * iter, + fd_txncache_t * tc, + ulong blockcache_idx ); + +void +fd_txncache_blockcache_iter_next( fd_txncache_blockcache_iter_t * iter ); + +void +fd_txncache_blockcache_iter_fini( fd_txncache_blockcache_iter_t * iter ); -static inline ulong -fd_txncache_txnpage_idx_ld( ulong idx_sz, - void const * idx, - ulong i ) { - return idx_sz==sizeof(uint) ? (ulong)((uint const *)idx)[ i ] : (ulong)((ushort const *)idx)[ i ]; +FD_FN_PURE static inline int +fd_txncache_blockcache_iter_done( fd_txncache_blockcache_iter_t const * iter ) { + return !iter->txn; } -static inline void -fd_txncache_txnpage_idx_st( ulong idx_sz, - void * idx, - ulong i, - ulong val ) { - if( idx_sz==sizeof(uint) ) ((uint *)idx)[ i ] = (uint)val; - else ((ushort *)idx)[ i ] = (ushort)val; +FD_FN_PURE static inline fd_txncache_single_txn_t const * +fd_txncache_blockcache_iter_ele( fd_txncache_blockcache_iter_t const * iter ) { + return iter->txn; } FD_PROTOTYPES_END diff --git a/src/flamenco/runtime/fd_txncache_shmem.c b/src/flamenco/runtime/fd_txncache_shmem.c index ab6f0a36826..9a255e78110 100644 --- a/src/flamenco/runtime/fd_txncache_shmem.c +++ b/src/flamenco/runtime/fd_txncache_shmem.c @@ -82,7 +82,9 @@ fd_txncache_shmem_align( void ) { ulong fd_txncache_shmem_footprint( ulong max_live_slots, - ulong max_txn_per_slot ) { + ulong max_txn_per_slot, + ulong cache_footprint ) { + if( FD_UNLIKELY( !cache_footprint ) ) return 0UL; if( FD_UNLIKELY( max_live_slots<1UL ) ) return 0UL; if( FD_UNLIKELY( max_txn_per_slot<1UL ) ) return 0UL; @@ -90,29 +92,30 @@ fd_txncache_shmem_footprint( ulong max_live_slots, ulong blockhash_map_chains = fd_ulong_pow2_up( 2UL*max_active_slots ); ulong bucket_cnt = fd_txncache_bucket_cnt( max_txn_per_slot ); - ulong _max_txnpages = fd_txncache_max_txnpages ( max_active_slots, max_txn_per_slot ); - ulong _max_txnpages_per_blockhash = fd_txncache_max_txnpages_per_blockhash( max_active_slots, max_txn_per_slot ); - if( FD_UNLIKELY( _max_txnpages>FD_TXNCACHE_MAX_TXNPAGES ) ) + ulong _txnpages_max = fd_txncache_max_txnpages( max_active_slots, max_txn_per_slot ); + if( FD_UNLIKELY( _txnpages_max>FD_TXNCACHE_MAX_TXNPAGES ) ) FD_LOG_ERR(( "txncache needs %lu txnpages but at most %lu are addressable (max_live_slots=%lu max_txn_per_slot=%lu)", - _max_txnpages, FD_TXNCACHE_MAX_TXNPAGES, max_live_slots, max_txn_per_slot )); - ulong _txnpage_idx_sz = fd_txncache_txnpage_idx_sz( _max_txnpages ); + _txnpages_max, FD_TXNCACHE_MAX_TXNPAGES, max_live_slots, max_txn_per_slot )); + ulong frames_max = fd_ulong_min( 1UL+(cache_footprint-1UL)/sizeof(fd_txncache_txnpage_t), _txnpages_max ); ulong _descends_footprint = descends_set_footprint( max_active_slots ); if( FD_UNLIKELY( !_descends_footprint ) ) return 0UL; ulong l; l = FD_LAYOUT_INIT; - l = FD_LAYOUT_APPEND( l, FD_TXNCACHE_SHMEM_ALIGN, sizeof(fd_txncache_shmem_t) ); - l = FD_LAYOUT_APPEND( l, blockhash_map_align(), blockhash_map_footprint( blockhash_map_chains ) ); - l = FD_LAYOUT_APPEND( l, blockcache_pool_align(), blockcache_pool_footprint( max_active_slots ) ); - l = FD_LAYOUT_APPEND( l, _txnpage_idx_sz, max_active_slots*_max_txnpages_per_blockhash*_txnpage_idx_sz ); /* blockcache->pages */ - l = FD_LAYOUT_APPEND( l, alignof(uint), max_active_slots*bucket_cnt*sizeof(uint) ); /* blockcache->heads */ - l = FD_LAYOUT_APPEND( l, descends_set_align(), max_active_slots*_descends_footprint ); /* blockcache->descends */ - l = FD_LAYOUT_APPEND( l, _txnpage_idx_sz, _max_txnpages*_txnpage_idx_sz ); /* txnpages_free */ - l = FD_LAYOUT_APPEND( l, alignof(fd_txncache_txnpage_t), _max_txnpages*sizeof(fd_txncache_txnpage_t) ); /* txnpages */ - l = FD_LAYOUT_APPEND( l, _txnpage_idx_sz, _max_txnpages_per_blockhash*_txnpage_idx_sz ); /* scratchpad txnpage pointer array for purge stale */ - l = FD_LAYOUT_APPEND( l, alignof(uint), bucket_cnt*sizeof(uint) ); /* scratchpad heads for purge stale */ - l = FD_LAYOUT_APPEND( l, alignof(fd_txncache_txnpage_t), sizeof(fd_txncache_txnpage_t) ); /* scratchpad txnpage for purge stale */ + l = FD_LAYOUT_APPEND( l, FD_TXNCACHE_SHMEM_ALIGN, sizeof(fd_txncache_shmem_t) ); + l = FD_LAYOUT_APPEND( l, blockhash_map_align(), blockhash_map_footprint( blockhash_map_chains ) ); + l = FD_LAYOUT_APPEND( l, blockcache_pool_align(), blockcache_pool_footprint( max_active_slots ) ); + l = FD_LAYOUT_APPEND( l, alignof(uint), max_active_slots*bucket_cnt*sizeof(uint) ); /* blockcache->heads */ + l = FD_LAYOUT_APPEND( l, descends_set_align(), max_active_slots*_descends_footprint ); /* blockcache->descends */ + l = FD_LAYOUT_APPEND( l, alignof(uint), _txnpages_max*sizeof(uint) ); /* txnpages_free */ + l = FD_LAYOUT_APPEND( l, alignof(fd_txncache_page_meta_t), _txnpages_max*sizeof(fd_txncache_page_meta_t) ); /* page_meta */ + l = FD_LAYOUT_APPEND( l, alignof(uint), frames_max*sizeof(uint) ); /* frame_page */ + l = FD_LAYOUT_APPEND( l, alignof(uint), frames_max*sizeof(uint) ); /* frames_free */ + l = FD_LAYOUT_APPEND( l, alignof(fd_txncache_txnpage_t), frames_max*sizeof(fd_txncache_txnpage_t) ); /* txnpages */ + l = FD_LAYOUT_APPEND( l, alignof(uint), bucket_cnt*sizeof(uint) ); /* scratchpad heads for purge stale */ + l = FD_LAYOUT_APPEND( l, alignof(fd_txncache_txnpage_t), sizeof(fd_txncache_txnpage_t) ); /* scratchpad txnpage for purge stale */ + l = FD_LAYOUT_APPEND( l, alignof(fd_txncache_txnpage_t), sizeof(fd_txncache_txnpage_t) ); /* compaction input */ return FD_LAYOUT_FINI( l, FD_TXNCACHE_SHMEM_ALIGN ); } @@ -120,6 +123,7 @@ void * fd_txncache_shmem_new( void * shmem, ulong max_live_slots, ulong max_txn_per_slot, + ulong cache_footprint, ulong seed ) { if( FD_UNLIKELY( !shmem ) ) { FD_LOG_WARNING(( "NULL shmem" )); @@ -131,6 +135,7 @@ fd_txncache_shmem_new( void * shmem, return NULL; } + if( FD_UNLIKELY( !cache_footprint ) ) return NULL; if( FD_UNLIKELY( !max_live_slots ) ) return NULL; if( FD_UNLIKELY( !max_txn_per_slot ) ) return NULL; @@ -138,28 +143,32 @@ fd_txncache_shmem_new( void * shmem, ulong blockhash_map_chains = fd_ulong_pow2_up( 2UL*max_active_slots ); ulong bucket_cnt = fd_txncache_bucket_cnt( max_txn_per_slot ); - ulong _max_txnpages = fd_txncache_max_txnpages ( max_active_slots, max_txn_per_slot ); - ulong _max_txnpages_per_blockhash = fd_txncache_max_txnpages_per_blockhash( max_active_slots, max_txn_per_slot ); - if( FD_UNLIKELY( _max_txnpages>FD_TXNCACHE_MAX_TXNPAGES ) ) + ulong _txnpages_max = fd_txncache_max_txnpages ( max_active_slots, max_txn_per_slot ); + ulong _txnpages_per_blockhash_max = fd_txncache_max_txnpages_per_blockhash( max_active_slots, max_txn_per_slot ); + if( FD_UNLIKELY( _txnpages_max>FD_TXNCACHE_MAX_TXNPAGES ) ) FD_LOG_ERR(( "txncache needs %lu txnpages but at most %lu are addressable (max_live_slots=%lu max_txn_per_slot=%lu)", - _max_txnpages, FD_TXNCACHE_MAX_TXNPAGES, max_live_slots, max_txn_per_slot )); - ulong _txnpage_idx_sz = fd_txncache_txnpage_idx_sz( _max_txnpages ); + _txnpages_max, FD_TXNCACHE_MAX_TXNPAGES, max_live_slots, max_txn_per_slot )); + ulong frames_max = fd_ulong_min( 1UL+(cache_footprint-1UL)/sizeof(fd_txncache_txnpage_t), _txnpages_max ); ulong _descends_footprint = descends_set_footprint( max_active_slots ); if( FD_UNLIKELY( !_descends_footprint ) ) return NULL; FD_SCRATCH_ALLOC_INIT( l, shmem ); - fd_txncache_shmem_t * tc = FD_SCRATCH_ALLOC_APPEND( l, FD_TXNCACHE_SHMEM_ALIGN, sizeof(fd_txncache_shmem_t) ); - void * _blockhash_map = FD_SCRATCH_ALLOC_APPEND( l, blockhash_map_align(), blockhash_map_footprint( blockhash_map_chains ) ); - void * _blockcache_pool = FD_SCRATCH_ALLOC_APPEND( l, blockcache_pool_align(), blockcache_pool_footprint( max_active_slots ) ); - FD_SCRATCH_ALLOC_APPEND( l, _txnpage_idx_sz, max_active_slots*_max_txnpages_per_blockhash*_txnpage_idx_sz ); - FD_SCRATCH_ALLOC_APPEND( l, alignof(uint), max_active_slots*bucket_cnt*sizeof(uint) ); - void * _blockcache_descends = FD_SCRATCH_ALLOC_APPEND( l, descends_set_align(), max_active_slots*_descends_footprint ); - void * _txnpages_free = FD_SCRATCH_ALLOC_APPEND( l, _txnpage_idx_sz, _max_txnpages*_txnpage_idx_sz ); - FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_txncache_txnpage_t), _max_txnpages*sizeof(fd_txncache_txnpage_t) ); - FD_SCRATCH_ALLOC_APPEND( l, _txnpage_idx_sz, _max_txnpages_per_blockhash*_txnpage_idx_sz ); - FD_SCRATCH_ALLOC_APPEND( l, alignof(uint), bucket_cnt*sizeof(uint) ); - FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_txncache_txnpage_t), sizeof(fd_txncache_txnpage_t) ); + fd_txncache_shmem_t * tc = FD_SCRATCH_ALLOC_APPEND( l, FD_TXNCACHE_SHMEM_ALIGN, sizeof(fd_txncache_shmem_t) ); + void * _blockhash_map = FD_SCRATCH_ALLOC_APPEND( l, blockhash_map_align(), blockhash_map_footprint( blockhash_map_chains ) ); + void * _blockcache_pool = FD_SCRATCH_ALLOC_APPEND( l, blockcache_pool_align(), blockcache_pool_footprint( max_active_slots ) ); + FD_SCRATCH_ALLOC_APPEND( l, alignof(uint), max_active_slots*bucket_cnt*sizeof(uint) ); + void * _blockcache_descends = FD_SCRATCH_ALLOC_APPEND( l, descends_set_align(), max_active_slots*_descends_footprint ); + uint * txnpages_free = FD_SCRATCH_ALLOC_APPEND( l, alignof(uint), _txnpages_max*sizeof(uint) ); + void * _page_meta = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_txncache_page_meta_t), _txnpages_max*sizeof(fd_txncache_page_meta_t) ); + uint * frame_page = FD_SCRATCH_ALLOC_APPEND( l, alignof(uint), frames_max*sizeof(uint) ); + uint * frames_free = FD_SCRATCH_ALLOC_APPEND( l, alignof(uint), frames_max*sizeof(uint) ); + FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_txncache_txnpage_t), frames_max*sizeof(fd_txncache_txnpage_t) ); + FD_SCRATCH_ALLOC_APPEND( l, alignof(uint), bucket_cnt*sizeof(uint) ); + FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_txncache_txnpage_t), sizeof(fd_txncache_txnpage_t) ); + FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_txncache_txnpage_t), sizeof(fd_txncache_txnpage_t) ); + tc->frames_max = frames_max; + fd_txncache_page_meta_t * page_meta = _page_meta; fd_txncache_blockcache_shmem_t * blockcache_pool = blockcache_pool_join( blockcache_pool_new( _blockcache_pool, max_active_slots ) ); FD_TEST( blockcache_pool ); @@ -173,23 +182,44 @@ fd_txncache_shmem_new( void * shmem, FD_TEST( descends_set ); } - tc->root_cnt = 0UL; - tc->mutation_gen = 0UL; - tc->root_gen = 0UL; + tc->root_cnt = 0UL; FD_TEST( root_slist_join( root_slist_new( tc->root_ll ) ) ); tc->lock->value = 0; + fd_rwlock_new( tc->alloc_lock ); tc->txn_per_slot_max = max_txn_per_slot; tc->active_slots_max = max_active_slots; tc->bucket_cnt = bucket_cnt; - tc->txnpages_per_blockhash_max = _max_txnpages_per_blockhash; - tc->max_txnpages = _max_txnpages; - tc->txnpage_idx_sz = _txnpage_idx_sz; + tc->txnpages_per_blockhash_max = _txnpages_per_blockhash_max; + tc->txnpages_max = _txnpages_max; tc->blockcache_generation = 0U; - tc->txnpages_free_cnt = _max_txnpages; - for( ulong i=0UL; i<_max_txnpages; i++ ) fd_txncache_txnpage_idx_st( _txnpage_idx_sz, _txnpages_free, i, i ); + tc->txnpages_free_cnt = _txnpages_max; + tc->frames_free_cnt = frames_max; + tc->evict_next = 0UL; + tc->release_seq = 0UL; + + tc->page_miss_cnt = 0UL; + tc->page_spilled_cnt = 0UL; + tc->page_spill_read_cnt = 0UL; + tc->frame_evicted_cnt = 0UL; + tc->spill_ticks = 0UL; + tc->lock_read_wait_ticks = 0UL; + tc->lock_write_wait_ticks = 0UL; + for( ulong i=0UL; i<_txnpages_max; i++ ) { + txnpages_free[ i ] = (uint)i; + page_meta[ i ].frame = UINT_MAX; + page_meta[ i ].dirty = 0U; + page_meta[ i ].disk_valid = 0; + page_meta[ i ].next = UINT_MAX; + page_meta[ i ].ref = 0U; + page_meta[ i ].io = 0U; + } + for( ulong i=0UL; iseed = seed; diff --git a/src/flamenco/runtime/fd_txncache_shmem.h b/src/flamenco/runtime/fd_txncache_shmem.h index e71e3df3594..b7590f5b4ba 100644 --- a/src/flamenco/runtime/fd_txncache_shmem.h +++ b/src/flamenco/runtime/fd_txncache_shmem.h @@ -3,7 +3,7 @@ #include "../../util/fd_util_base.h" -#define FD_TXNCACHE_SHMEM_ALIGN (128UL) +#define FD_TXNCACHE_SHMEM_ALIGN (4096UL) #define FD_TXNCACHE_SHMEM_MAGIC (0xF17EDA2CE58CC4E1UL) /* FIREDANCE SMCCHE V1 */ @@ -30,18 +30,24 @@ fd_txncache_shmem_align( void ); 2*config->limits.max_txn_per_slot, so a raised [development.bench] block cost limit sizes the pool up through that value. - footprint and new return 0 / NULL for zero max_live_slots or - max_txn_per_slot, and log an error and exit if the parameters need a - txnpage pool larger than the structure can address. */ + cache_footprint is the in-memory transaction page budget in bytes + (ULONG_MAX for a fully in-memory cache). Pages that do not fit spill + to disk. + + footprint and new return 0 / NULL for zero max_live_slots, + max_txn_per_slot, or cache_footprint, and log an error and exit if the + parameters need more pages than can be addressed. */ ulong fd_txncache_shmem_footprint( ulong max_live_slots, - ulong max_txn_per_slot ); + ulong max_txn_per_slot, + ulong cache_footprint ); void * fd_txncache_shmem_new( void * shmem, ulong max_live_slots, ulong max_txn_per_slot, + ulong cache_footprint, ulong seed ); fd_txncache_shmem_t * diff --git a/src/flamenco/runtime/fuzz_txncache_fork_graph.c b/src/flamenco/runtime/fuzz_txncache_fork_graph.c index 9fe24dd1ed7..f1416b59045 100644 --- a/src/flamenco/runtime/fuzz_txncache_fork_graph.c +++ b/src/flamenco/runtime/fuzz_txncache_fork_graph.c @@ -2,8 +2,11 @@ #error "This target requires FD_HAS_HOSTED" #endif +#define _GNU_SOURCE +#include #include #include +#include #include "../../util/fd_util.h" #include "fd_txncache.h" @@ -105,12 +108,14 @@ fuzz_bounded( fuzz_cursor_t * cur, return x % bound; } +static int fuzz_spill_fd = -1; + static fd_txncache_t * setup( ulong max_live_slots, ulong max_txn_per_slot ) { - fd_txncache_shmem_t * shtc = fd_txncache_shmem_join( fd_txncache_shmem_new( fuzz_shmem, max_live_slots, max_txn_per_slot, 0UL ) ); + fd_txncache_shmem_t * shtc = fd_txncache_shmem_join( fd_txncache_shmem_new( fuzz_shmem, max_live_slots, max_txn_per_slot, 2UL*sizeof(fd_txncache_txnpage_t), 0UL ) ); FD_TEST( shtc ); - fd_txncache_t * tc = fd_txncache_join( fd_txncache_new( fuzz_ljoin, shtc ) ); + fd_txncache_t * tc = fd_txncache_join( fd_txncache_new( fuzz_ljoin, shtc, fuzz_spill_fd ) ); FD_TEST( tc ); return tc; } @@ -354,21 +359,41 @@ model_record_txn( model_t * m, txn->txnhash = *txnhash; } +/* Counts the transactions of blockcache block_fork through the + blockcache iterator. If stale is non-NULL, sets it if any of them + belongs to a removed fork or an old generation. */ + static ulong -blockcache_txn_cnt( model_t const * m, - ushort block_fork ) { +blockcache_txn_scan( model_t const * m, + ushort block_fork, + int * stale ) { fd_txncache_t * tc = m->tc; - blockcache_t const * bc = &tc->blockcache_pool[ block_fork ]; ulong cnt = 0UL; - for( ulong i=0UL; ishmem->pages_cnt; i++ ) { - ulong page = fd_txncache_txnpage_idx_ld( tc->shmem->txnpage_idx_sz, bc->pages, i ); - FD_TEST( pageshmem->max_txnpages ); - cnt += FD_TXNCACHE_TXNS_PER_PAGE - tc->txnpages[ page ].free; + fd_txncache_blockcache_iter_t iter[1]; + for( fd_txncache_blockcache_iter_init( iter, tc, block_fork ); + !fd_txncache_blockcache_iter_done( iter ); + fd_txncache_blockcache_iter_next( iter ) ) { + fd_txncache_single_txn_t const * txn = fd_txncache_blockcache_iter_ele( iter ); + ushort txn_fork = txn->fork_id.val; + FD_TEST( txn_forkshmem->active_slots_max ); + blockcache_t const * fork = &tc->blockcache_pool[ txn_fork ]; + if( stale && ( fork->shmem->frozen<0 || fork->shmem->generation!=txn->generation ) ) *stale = 1; + cnt++; } + fd_txncache_blockcache_iter_fini( iter ); return cnt; } +static ulong +blockcache_txn_cnt( model_t const * m, + ushort block_fork ) { + return blockcache_txn_scan( m, block_fork, NULL ); +} + +/* Only the newest txnpage of a blockcache can be partially filled, so + the tail is full when every page is. */ + static int blockcache_needs_purge_for_insert( model_t const * m, ushort block_fork ) { @@ -377,34 +402,15 @@ blockcache_needs_purge_for_insert( model_t const * m, if( FD_UNLIKELY( !bc->shmem->pages_cnt ) ) return 0; if( FD_UNLIKELY( bc->shmem->pages_cnt!=tc->shmem->txnpages_per_blockhash_max ) ) return 0; - - ulong tail_page = fd_txncache_txnpage_idx_ld( tc->shmem->txnpage_idx_sz, bc->pages, bc->shmem->pages_cnt-1UL ); - FD_TEST( tail_pageshmem->max_txnpages ); - return !tc->txnpages[ tail_page ].free; + return blockcache_txn_cnt( m, block_fork )==bc->shmem->pages_cnt*FD_TXNCACHE_TXNS_PER_PAGE; } static int blockcache_has_stale_txn( model_t const * m, ushort block_fork ) { - fd_txncache_t * tc = m->tc; - blockcache_t const * bc = &tc->blockcache_pool[ block_fork ]; - - for( ulong i=0UL; ishmem->pages_cnt; i++ ) { - ulong page = fd_txncache_txnpage_idx_ld( tc->shmem->txnpage_idx_sz, bc->pages, i ); - FD_TEST( pageshmem->max_txnpages ); - - ulong txn_cnt = FD_TXNCACHE_TXNS_PER_PAGE - tc->txnpages[ page ].free; - for( ulong j=0UL; jtxnpages[ page ].txns[ j ]; - ushort txn_fork = txn->fork_id.val; - FD_TEST( txn_forkshmem->active_slots_max ); - - blockcache_t const * fork = &tc->blockcache_pool[ txn_fork ]; - if( FD_UNLIKELY( fork->shmem->frozen<0 || fork->shmem->generation!=txn->generation ) ) return 1; - } - } - - return 0; + int stale = 0; + blockcache_txn_scan( m, block_fork, &stale ); + return stale; } static void @@ -556,8 +562,12 @@ model_pick_stale_txn_query( model_t const * m, static void check_invariants( model_t const * m ) { fd_txncache_t * tc = m->tc; - FD_TEST( tc->shmem->txnpages_free_cnt<=tc->shmem->max_txnpages ); - FD_TEST( tc->shmem->max_txnpages<=512U ); + fd_rwlock_write( tc->shmem->lock ); + FD_TEST( tc->shmem->txnpages_free_cnt<=tc->shmem->txnpages_max ); + FD_TEST( tc->shmem->frames_maxshmem->txnpages_max ); + FD_TEST( tc->shmem->frames_free_cnt<=tc->shmem->frames_max ); + FD_TEST( tc->shmem->evict_nextshmem->frames_max ); + FD_TEST( tc->shmem->txnpages_max<=512U ); ulong pool_free = blockcache_pool_free( tc->blockcache_shmem_pool ); FD_TEST( pool_free + m->live_cnt == blockcache_pool_max( tc->blockcache_shmem_pool ) ); @@ -567,8 +577,6 @@ check_invariants( model_t const * m ) { uchar page_seen[ 512 ]; memset( page_seen, 0, sizeof(page_seen) ); ulong used_pages = 0UL; - ulong idx_sz = tc->shmem->txnpage_idx_sz; - ulong null_idx = idx_sz==sizeof(uint) ? UINT_MAX : USHORT_MAX; for( ulong i=0UL; ifork[ i ].alive ) ) continue; @@ -578,26 +586,58 @@ check_invariants( model_t const * m ) { FD_TEST( bc->shmem->pages_cnt<=tc->shmem->txnpages_per_blockhash_max ); FD_TEST( bc->shmem->frozen==m->fork[ i ].frozen ); - for( ulong j=0UL; jshmem->pages_cnt; j++ ) { - ulong page = fd_txncache_txnpage_idx_ld( idx_sz, bc->pages, j ); - FD_TEST( pageshmem->max_txnpages ); + ulong list_cnt = 0UL; + for( uint page=bc->shmem->page_head; page!=UINT_MAX; page=tc->page_meta[ page ].next ) { + FD_TEST( pageshmem->txnpages_max ); FD_TEST( !page_seen[ page ] ); + FD_TEST( tc->page_meta[ page ].frame!=UINT_MAX || tc->page_meta[ page ].disk_valid ); page_seen[ page ] = 1U; used_pages++; + list_cnt++; } - for( ulong j=bc->shmem->pages_cnt; jshmem->txnpages_per_blockhash_max; j++ ) { - FD_TEST( fd_txncache_txnpage_idx_ld( idx_sz, bc->pages, j )==null_idx ); - } + FD_TEST( list_cnt==bc->shmem->pages_cnt ); } for( ulong i=0UL; ishmem->txnpages_free_cnt; i++ ) { - ulong page = fd_txncache_txnpage_idx_ld( idx_sz, tc->txnpages_free, i ); - FD_TEST( pageshmem->max_txnpages ); + ulong page = tc->txnpages_free[ i ]; + FD_TEST( pageshmem->txnpages_max ); FD_TEST( !page_seen[ page ] ); page_seen[ page ] = 1U; + FD_TEST( tc->page_meta[ page ].frame==UINT_MAX ); + FD_TEST( !tc->page_meta[ page ].dirty ); + FD_TEST( !tc->page_meta[ page ].disk_valid ); + FD_TEST( tc->page_meta[ page ].next==UINT_MAX ); } - FD_TEST( used_pages + tc->shmem->txnpages_free_cnt == tc->shmem->max_txnpages ); + FD_TEST( used_pages + tc->shmem->txnpages_free_cnt == tc->shmem->txnpages_max ); + + /* Resident frames are mapped to one logical page or on the free + stack. Evicted pages must be clean and retain their disk copy. */ + uchar frame_seen[ 512 ]; + memset( frame_seen, 0, sizeof(frame_seen) ); + ulong resident_pages = 0UL; + for( ulong page=0UL; pageshmem->txnpages_max; page++ ) { + FD_TEST( page_seen[ page ] ); + fd_txncache_page_meta_t const * meta = &tc->page_meta[ page ]; + if( meta->frame==UINT_MAX ) { + FD_TEST( !meta->dirty ); + continue; + } + FD_TEST( meta->frameshmem->frames_max ); + FD_TEST( !frame_seen[ meta->frame ] ); + FD_TEST( tc->frame_page[ meta->frame ]==page ); + frame_seen[ meta->frame ] = 1U; + resident_pages++; + } + for( ulong i=0UL; ishmem->frames_free_cnt; i++ ) { + uint frame = tc->frames_free[ i ]; + FD_TEST( frameshmem->frames_max ); + FD_TEST( !frame_seen[ frame ] ); + FD_TEST( tc->frame_page[ frame ]==UINT_MAX ); + frame_seen[ frame ] = 1U; + } + FD_TEST( resident_pages + tc->shmem->frames_free_cnt == tc->shmem->frames_max ); + fd_rwlock_unwrite( tc->shmem->lock ); /* Root history: the real root slist must match the model's live root window exactly. The list length is root_cnt+1: the original root @@ -1109,6 +1149,7 @@ op_extend_root( model_t * m, static void fuzz_cleanup( void ) { + if( fuzz_spill_fd>=0 ) FD_TEST( !close( fuzz_spill_fd ) ); free( fuzz_shmem ); free( fuzz_ljoin ); } @@ -1124,7 +1165,11 @@ LLVMFuzzerInitialize( int * argc, fd_log_level_logfile_set( 4 ); atexit( fd_halt ); - fuzz_shmem_fp = fd_txncache_shmem_footprint( FUZZ_MAX_LIVE_SLOTS, FUZZ_MAX_TXN_PER_SLOT ); + char path[] = "/tmp/fd-txncache-fuzz-XXXXXX"; + fuzz_spill_fd = mkstemp( path ); + FD_TEST( fuzz_spill_fd>=0 ); FD_TEST( !unlink( path ) ); + FD_TEST( !fcntl( fuzz_spill_fd, F_SETFL, O_DIRECT ) ); + fuzz_shmem_fp = fd_txncache_shmem_footprint( FUZZ_MAX_LIVE_SLOTS, FUZZ_MAX_TXN_PER_SLOT, 2UL*sizeof(fd_txncache_txnpage_t) ); fuzz_ljoin_fp = fd_txncache_footprint( FUZZ_MAX_LIVE_SLOTS ); FD_TEST( fuzz_shmem_fp ); FD_TEST( fuzz_ljoin_fp ); diff --git a/src/flamenco/runtime/test_txncache.c b/src/flamenco/runtime/test_txncache.c index 1056a0cf7cb..b0ef4e6638c 100644 --- a/src/flamenco/runtime/test_txncache.c +++ b/src/flamenco/runtime/test_txncache.c @@ -8,6 +8,13 @@ #include "fd_txncache_private.h" #include +#include +#include +#include +#include +#if FD_HAS_RACESAN +#include "../../util/racesan/fd_racesan.h" +#endif FD_STATIC_ASSERT( FD_TXNCACHE_ALIGN==128UL, unit_test ); @@ -17,13 +24,294 @@ FD_STATIC_ASSERT( FD_TXNCACHE_ALIGN==128UL, unit_test ); #define NULL_FORK ((fd_txncache_fork_id_t){ .val = USHORT_MAX }) +static void +test_spill( void ) { + ulong live = 4UL, txns = 256UL; + FD_TEST( !fd_txncache_shmem_footprint( live, txns, 0UL ) ); + ulong shsz = fd_txncache_shmem_footprint( live, txns, 1UL*sizeof(fd_txncache_txnpage_t) ); + FD_TEST( fd_txncache_shmem_footprint( live, txns, 1UL )==shsz ); + FD_TEST( fd_txncache_shmem_footprint( live, txns, sizeof(fd_txncache_txnpage_t)+1UL )>shsz ); + ulong sz = shsz+fd_txncache_footprint( live ); + uchar * mem = mmap( NULL, sz, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0 ); + FD_TEST( mem!=MAP_FAILED ); + FD_TEST( !fd_txncache_shmem_new( mem, live, txns, 0UL, 0UL ) ); + fd_txncache_shmem_t * sh = fd_txncache_shmem_join( fd_txncache_shmem_new( mem, live, txns, 1UL*sizeof(fd_txncache_txnpage_t), 0UL ) ); + FD_TEST( sh->frames_max==1UL ); + char path[] = "/tmp/fd-txncache-test-XXXXXX"; + int fd = mkstemp( path ); + FD_TEST( fd>=0 ); + FD_TEST( !fcntl( fd, F_SETFL, O_DIRECT ) ); + FD_TEST( !unlink( path ) ); + fd_txncache_t * tc = fd_txncache_join( fd_txncache_new( mem+shsz, sh, fd ) ); + + for( ulong round=0UL; round<2UL; round++ ) { + fd_txncache_fork_id_t root = fd_txncache_attach_child( tc, NULL_FORK ); + fd_txncache_finalize_fork( tc, root, 0UL, BLOCKHASH(1UL) ); + fd_txncache_fork_id_t a = fd_txncache_attach_child( tc, root ); + fd_txncache_fork_id_t b = fd_txncache_attach_child( tc, root ); + for( ulong i=0UL; i0 ); + FD_TEST( !close( fd ) ); + FD_TEST( !munmap( mem, sz ) ); + ulong cache_footprint = 640UL<<20; /* default runtime.transaction_cache_size_mib */ + ulong production_sz = fd_txncache_shmem_footprint( 2048UL, 196078UL, cache_footprint ); + FD_LOG_NOTICE(( "production txncache footprint %lu bytes", production_sz )); + FD_TEST( production_sz<(1UL<<30) ); +} + +static void +test_spill_page_reuse( void ) { + ulong shsz = fd_txncache_shmem_footprint( 4UL, 256UL, 2UL*sizeof(fd_txncache_txnpage_t) ); + ulong sz = shsz+fd_txncache_footprint( 4UL ); + uchar * mem = mmap( NULL, sz, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0 ); + FD_TEST( mem!=MAP_FAILED ); + fd_txncache_shmem_t * sh = fd_txncache_shmem_join( fd_txncache_shmem_new( mem, 4UL, 256UL, 2UL*sizeof(fd_txncache_txnpage_t), 0UL ) ); + char path[] = "/tmp/fd-txncache-free-XXXXXX"; + int fd = mkstemp( path ); + FD_TEST( fd>=0 ); + FD_TEST( !fcntl( fd, F_SETFL, O_DIRECT ) ); + FD_TEST( !unlink( path ) ); + fd_txncache_t * tc = fd_txncache_join( fd_txncache_new( mem+shsz, sh, fd ) ); + + fd_txncache_fork_id_t root = fd_txncache_attach_child( tc, NULL_FORK ); + fd_txncache_finalize_fork( tc, root, 0UL, BLOCKHASH(1UL) ); + for( ulong i=0UL; i<8UL; i++ ) { + fd_txncache_fork_id_t a = fd_txncache_attach_child( tc, root ); + fd_txncache_finalize_fork( tc, a, 0UL, BLOCKHASH(2UL) ); + fd_txncache_fork_id_t child = fd_txncache_attach_child( tc, a ); + fd_txncache_insert( tc, child, BLOCKHASH(1UL), TXNHASH(1UL+i) ); + fd_txncache_insert( tc, child, BLOCKHASH(2UL), TXNHASH(10UL+i) ); + FD_TEST( fd_txncache_query( tc, child, BLOCKHASH(2UL), TXNHASH(10UL+i) ) ); + FD_TEST( !fd_txncache_query( tc, child, BLOCKHASH(2UL), TXNHASH(9UL+i) ) ); + /* A returned frame must be reused before evicting a live dirty page. */ + FD_TEST( lseek( fd, 0, SEEK_END )==0L ); + fd_txncache_cancel_fork( tc, a ); + FD_TEST( sh->frames_free_cnt==1UL ); + } + fd_txncache_reset( tc ); + FD_TEST( sh->txnpages_free_cnt==sh->txnpages_max ); + FD_TEST( sh->frames_free_cnt==2UL ); + root = fd_txncache_attach_child( tc, NULL_FORK ); + fd_txncache_finalize_fork( tc, root, 0UL, BLOCKHASH(1UL) ); + fd_txncache_fork_id_t child = fd_txncache_attach_child( tc, root ); + fd_txncache_insert( tc, child, BLOCKHASH(1UL), TXNHASH(99UL) ); + FD_TEST( !fd_txncache_query( tc, child, BLOCKHASH(1UL), TXNHASH(1UL) ) ); + FD_TEST( lseek( fd, 0, SEEK_END )==0L ); + FD_TEST( !close( fd ) ); + FD_TEST( !munmap( mem, sz ) ); +} + +/* Cached pages need no file access; eviction writes and later reloads whole pages. */ +static void +test_spill_page_cache( void ) { + ulong shsz = fd_txncache_shmem_footprint( 4UL, 4UL, sizeof(fd_txncache_txnpage_t) ); + ulong lsz = fd_txncache_footprint( 4UL ); + ulong sz = shsz+2UL*lsz; + uchar * mem = mmap( NULL, sz, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0 ); + FD_TEST( mem!=MAP_FAILED ); + fd_txncache_shmem_t * sh = fd_txncache_shmem_join( fd_txncache_shmem_new( mem, 4UL, 4UL, sizeof(fd_txncache_txnpage_t), 0UL ) ); + char path[] = "/tmp/fd-txncache-query-XXXXXX"; + int fd = mkstemp( path ); + FD_TEST( fd>=0 ); + FD_TEST( !fcntl( fd, F_SETFL, O_DIRECT ) ); + FD_TEST( !unlink( path ) ); + fd_txncache_t * tc = fd_txncache_join( fd_txncache_new( mem+shsz, sh, fd ) ); + fd_txncache_t * cached = fd_txncache_join( fd_txncache_new( mem+shsz+lsz, sh, -1 ) ); + + fd_txncache_fork_id_t root = fd_txncache_attach_child( tc, NULL_FORK ); + fd_txncache_finalize_fork( tc, root, 0UL, BLOCKHASH(1UL) ); + fd_txncache_fork_id_t a = fd_txncache_attach_child( tc, root ); + fd_txncache_finalize_fork( tc, a, 0UL, BLOCKHASH(2UL) ); + fd_txncache_fork_id_t b = fd_txncache_attach_child( tc, a ); + fd_txncache_insert( tc, b, BLOCKHASH(1UL), TXNHASH(1UL) ); + FD_TEST( lseek( fd, 0, SEEK_END )==0L ); + FD_TEST( fd_txncache_query( cached, b, BLOCKHASH(1UL), TXNHASH(1UL) ) ); + fd_txncache_insert( cached, b, BLOCKHASH(1UL), TXNHASH(2UL) ); + + /* One frame forces blockhash 1 out when blockhash 2 first gets a page. */ + fd_txncache_insert( tc, b, BLOCKHASH(2UL), TXNHASH(3UL) ); + FD_TEST( lseek( fd, 0, SEEK_END )>0L ); + FD_TEST( fd_txncache_query( cached, b, BLOCKHASH(2UL), TXNHASH(3UL) ) ); + fd_txncache_insert( cached, b, BLOCKHASH(2UL), TXNHASH(4UL) ); + for( ulong i=5UL; i<100UL; i++ ) fd_txncache_insert( cached, b, BLOCKHASH(2UL), TXNHASH(i) ); + + /* Reload after eviction and check the complete chain through another join. */ + FD_TEST( fd_txncache_query( tc, b, BLOCKHASH(1UL), TXNHASH(1UL) ) ); + FD_TEST( fd_txncache_query( cached, b, BLOCKHASH(1UL), TXNHASH(2UL) ) ); + FD_TEST( fd_txncache_query( tc, b, BLOCKHASH(2UL), TXNHASH(3UL) ) ); + for( ulong i=4UL; i<100UL; i++ ) FD_TEST( fd_txncache_query( cached, b, BLOCKHASH(2UL), TXNHASH(i) ) ); + FD_TEST( !fd_txncache_query( cached, b, BLOCKHASH(2UL), TXNHASH(100UL) ) ); + FD_TEST( !close( fd ) ); + FD_TEST( !munmap( mem, sz ) ); +} + +static void +test_spill_io_failure( int read_failure ) { + pid_t pid = fork(); + FD_TEST( pid>=0 ); + if( !pid ) { + ulong shsz = fd_txncache_shmem_footprint( 4UL, 256UL, 1UL*sizeof(fd_txncache_txnpage_t) ); + ulong sz = shsz+fd_txncache_footprint( 4UL ); + uchar * mem = mmap( NULL, sz, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0 ); + FD_TEST( mem!=MAP_FAILED ); + fd_txncache_shmem_t * sh = fd_txncache_shmem_join( fd_txncache_shmem_new( mem, 4UL, 256UL, 1UL*sizeof(fd_txncache_txnpage_t), 0UL ) ); + char path[] = "/tmp/fd-txncache-error-XXXXXX"; + int fd = mkstemp( path ); + FD_TEST( fd>=0 ); + FD_TEST( !fcntl( fd, F_SETFL, O_DIRECT ) ); + FD_TEST( !unlink( path ) ); + fd_txncache_t * tc = fd_txncache_join( fd_txncache_new( mem+shsz, sh, fd ) ); + + fd_txncache_fork_id_t root = fd_txncache_attach_child( tc, NULL_FORK ); + fd_txncache_finalize_fork( tc, root, 0UL, BLOCKHASH(1UL) ); + fd_txncache_fork_id_t a = fd_txncache_attach_child( tc, root ); + fd_txncache_finalize_fork( tc, a, 0UL, BLOCKHASH(2UL) ); + fd_txncache_fork_id_t b = fd_txncache_attach_child( tc, a ); + fd_txncache_insert( tc, b, BLOCKHASH(1UL), TXNHASH(1UL) ); + if( !read_failure ) FD_TEST( !close( fd ) ); + fd_txncache_insert( tc, b, BLOCKHASH(2UL), TXNHASH(2UL) ); + if( read_failure ) { + FD_TEST( !ftruncate( fd, 0 ) ); + (void)fd_txncache_query( tc, b, BLOCKHASH(1UL), TXNHASH(1UL) ); + } + _exit( 0 ); /* A disk failure must never return to its caller. */ + } + int status; + FD_TEST( waitpid( pid, &status, 0 )==pid ); + FD_TEST( WIFSIGNALED(status) && WTERMSIG(status)==6 ); +} + +#if FD_HAS_RACESAN + +struct fetch_io_inject { + fd_txncache_t * tc; + ulong cnt; +}; + +static void +inject_fetch_io( void * ctx, + ulong name_hash ) { + (void)name_hash; + struct fetch_io_inject * inject = ctx; + FD_TEST( fd_rwlock_trywrite( inject->tc->shmem->lock ) ); + fd_rwlock_unwrite( inject->tc->shmem->lock ); + inject->cnt++; +} + +/* A query that misses does its spill I/O without the lock. */ + +static void +test_fetch_unlocked( void ) { + ulong shsz = fd_txncache_shmem_footprint( 4UL, 256UL, sizeof(fd_txncache_txnpage_t) ); + ulong sz = shsz+fd_txncache_footprint( 4UL ); + uchar * mem = mmap( NULL, sz, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0 ); + FD_TEST( mem!=MAP_FAILED ); + fd_txncache_shmem_t * sh = fd_txncache_shmem_join( fd_txncache_shmem_new( mem, 4UL, 256UL, sizeof(fd_txncache_txnpage_t), 0UL ) ); + char path[] = "/tmp/fd-txncache-fetch-XXXXXX"; + int fd = mkstemp( path ); + FD_TEST( fd>=0 ); + FD_TEST( !fcntl( fd, F_SETFL, O_DIRECT ) ); + FD_TEST( !unlink( path ) ); + fd_txncache_t * tc = fd_txncache_join( fd_txncache_new( mem+shsz, sh, fd ) ); + fd_txncache_fork_id_t a = fd_txncache_attach_child( tc, NULL_FORK ); + fd_txncache_finalize_fork( tc, a, 0UL, BLOCKHASH(1UL) ); + fd_txncache_fork_id_t b = fd_txncache_attach_child( tc, a ); + fd_txncache_finalize_fork( tc, b, 0UL, BLOCKHASH(2UL) ); + fd_txncache_fork_id_t c = fd_txncache_attach_child( tc, b ); + fd_txncache_insert( tc, c, BLOCKHASH(1UL), TXNHASH(1UL) ); + fd_txncache_insert( tc, c, BLOCKHASH(2UL), TXNHASH(2UL) ); /* evicts the page of blockhash 1 */ + + struct fetch_io_inject inject = { .tc = tc }; + fd_racesan_t racesan[ 1 ]; + FD_TEST( fd_racesan_new( racesan, &inject ) ); + fd_racesan_inject( racesan, "txncache:fetch_io", inject_fetch_io ); + FD_RACESAN_INJECT_BEGIN( racesan ) { + FD_TEST( fd_txncache_query( tc, c, BLOCKHASH(1UL), TXNHASH(1UL) ) ); + } FD_RACESAN_INJECT_END; + FD_TEST( inject.cnt==1UL ); + fd_racesan_delete( racesan ); + FD_TEST( !close( fd ) ); + FD_TEST( !munmap( mem, sz ) ); +} + +#endif + +static ulong +histf_cnt( fd_histf_t const * hist ) { + ulong cnt = 0UL; + for( ulong i=0UL; i=0 ); + FD_TEST( !fcntl( fd, F_SETFL, O_DIRECT ) ); + FD_TEST( !unlink( path ) ); + fd_txncache_t * tc = fd_txncache_join( fd_txncache_new( mem+shsz, sh, fd ) ); + fd_txncache_join_metrics_t const * m = fd_txncache_join_metrics( tc ); + fd_txncache_fork_id_t a = fd_txncache_attach_child( tc, NULL_FORK ); + fd_txncache_finalize_fork( tc, a, 0UL, BLOCKHASH(1UL) ); + fd_txncache_fork_id_t b = fd_txncache_attach_child( tc, a ); + fd_txncache_finalize_fork( tc, b, 0UL, BLOCKHASH(2UL) ); + fd_txncache_fork_id_t c = fd_txncache_attach_child( tc, b ); + + fd_txncache_insert( tc, c, BLOCKHASH(1UL), TXNHASH(1UL) ); + FD_TEST( !m->insert_miss_cnt && !m->query_miss_cnt && !m->stalled_cnt ); + fd_txncache_insert( tc, c, BLOCKHASH(2UL), TXNHASH(2UL) ); /* new page, no free frame */ + FD_TEST( m->insert_miss_cnt==1UL ); + FD_TEST( fd_txncache_query( tc, c, BLOCKHASH(1UL), TXNHASH(1UL) ) ); + FD_TEST( m->query_miss_cnt==1UL && !m->stalled_cnt ); + + ulong const fill_cnt = FD_TXNCACHE_TXNS_PER_PAGE+FD_TXNCACHE_TXNS_PER_PAGE/2UL; + for( ulong i=0UL; iblockcache_pool[ a.val ].shmem->pages_cnt==2UL ); + ulong query_miss_cnt = m->query_miss_cnt; + FD_TEST( !fd_txncache_query( tc, c, BLOCKHASH(1UL), TXNHASH(999999UL) ) ); + FD_TEST( m->query_miss_cnt==query_miss_cnt+1UL && m->stalled_cnt==1UL ); + + FD_TEST( histf_cnt( m->query_ticks )==2UL ); + FD_TEST( histf_cnt( m->insert_ticks )==2UL+fill_cnt ); + FD_TEST( !close( fd ) ); + FD_TEST( !munmap( mem, sz ) ); +} + static void test_bucket_cnt( void ) { FD_TEST( fd_txncache_bucket_cnt( 0UL )==1UL ); FD_TEST( fd_txncache_bucket_cnt( 1UL )==1UL ); - FD_TEST( fd_txncache_bucket_cnt( 8UL )==1UL ); - FD_TEST( fd_txncache_bucket_cnt( 9UL )==2UL ); - FD_TEST( fd_txncache_bucket_cnt( FD_PACK_MAX_TXNCACHE_TXN_PER_SLOT )==12255UL ); + FD_TEST( fd_txncache_bucket_cnt( 32UL )==1UL ); + FD_TEST( fd_txncache_bucket_cnt( 33UL )==2UL ); + FD_TEST( fd_txncache_bucket_cnt( FD_PACK_MAX_TXNCACHE_TXN_PER_SLOT )==3064UL ); } static void @@ -33,34 +321,26 @@ test_page_sizing( void ) { FD_TEST( fd_txncache_max_txnpages ( max_active_slots, max_txn_per_slot )==17158UL ); FD_TEST( fd_txncache_max_txnpages_per_blockhash( max_active_slots, max_txn_per_slot )==17158UL ); - FD_TEST( fd_txncache_txnpage_idx_sz( 17158UL )==sizeof(ushort) ); - FD_TEST( fd_txncache_shmem_footprint( 2048UL, max_txn_per_slot )==4401349504UL ); /* production, ushort page indices */ + FD_TEST( fd_txncache_shmem_footprint( 2048UL, max_txn_per_slot, ULONG_MAX )==4316078080UL ); /* production */ /* development.bench.max_cost_per_block = 540M: 2*529,411 txns per - slot. The pool exceeds the ushort range, so page indices widen to - uint. */ + slot. */ ulong const bench_txn_per_slot = 1058822UL; FD_TEST( fd_txncache_max_txnpages ( max_active_slots, bench_txn_per_slot )==163762UL ); FD_TEST( fd_txncache_max_txnpages_per_blockhash( max_active_slots, bench_txn_per_slot )==163762UL ); - FD_TEST( fd_txncache_txnpage_idx_sz( 163762UL )==sizeof(uint) ); FD_TEST( 163762UL<=FD_TXNCACHE_MAX_TXNPAGES ); - FD_TEST( fd_txncache_shmem_footprint( 2048UL, bench_txn_per_slot )==42854135296UL ); - - FD_TEST( fd_txncache_txnpage_idx_sz( USHORT_MAX-2UL )==sizeof(ushort) ); - FD_TEST( fd_txncache_txnpage_idx_sz( USHORT_MAX-1UL )==sizeof(uint) ); + FD_TEST( fd_txncache_shmem_footprint( 2048UL, bench_txn_per_slot, ULONG_MAX )==41215467520UL ); } static void -test_bench_sizing( ulong max_live_slots, - ulong expected_idx_sz ) { +test_bench_sizing( ulong max_live_slots ) { FD_LOG_NOTICE(( "TEST BENCH SIZING (max_live_slots=%lu)", max_live_slots )); - /* The pool is tens of GiB at bench sizes but only the pages touched - by the inserts below are ever faulted in, so back it with a lazily - committed anonymous mapping rather than locked shmem. */ + /* Only the pages touched by the inserts below are faulted in, so + use a lazily committed anonymous mapping rather than locked shmem. */ ulong const max_txn_per_slot = 1058822UL; - ulong footprint_shmem = fd_txncache_shmem_footprint( max_live_slots, max_txn_per_slot ); + ulong footprint_shmem = fd_txncache_shmem_footprint( max_live_slots, max_txn_per_slot, ULONG_MAX ); ulong footprint_local = fd_txncache_footprint( max_live_slots ); FD_TEST( footprint_shmem ); @@ -69,17 +349,14 @@ test_bench_sizing( ulong max_live_slots, FD_TEST( mem!=MAP_FAILED ); FD_TEST( fd_ulong_is_aligned( (ulong)mem, FD_TXNCACHE_SHMEM_ALIGN ) ); - fd_txncache_shmem_t * shtc = fd_txncache_shmem_join( fd_txncache_shmem_new( mem, max_live_slots, max_txn_per_slot, 0UL ) ); + fd_txncache_shmem_t * shtc = fd_txncache_shmem_join( fd_txncache_shmem_new( mem, max_live_slots, max_txn_per_slot, ULONG_MAX, 0UL ) ); FD_TEST( shtc ); - fd_txncache_t * tc = fd_txncache_join( fd_txncache_new( mem+fd_ulong_align_up( footprint_shmem, FD_TXNCACHE_ALIGN ), shtc ) ); + fd_txncache_t * tc = fd_txncache_join( fd_txncache_new( mem+fd_ulong_align_up( footprint_shmem, FD_TXNCACHE_ALIGN ), shtc, -1 ) ); FD_TEST( tc ); - ulong mutation_gen = shtc->mutation_gen; - ulong root_gen = shtc->root_gen; ulong const max_active_slots = FD_TXNCACHE_MAX_BLOCKHASH_DISTANCE+max_live_slots; - FD_TEST( shtc->max_txnpages==fd_txncache_max_txnpages( max_active_slots, max_txn_per_slot ) ); - FD_TEST( shtc->txnpage_idx_sz==expected_idx_sz ); - FD_TEST( shtc->txnpages_free_cnt==shtc->max_txnpages ); + FD_TEST( shtc->txnpages_max==fd_txncache_max_txnpages( max_active_slots, max_txn_per_slot ) ); + FD_TEST( shtc->txnpages_free_cnt==shtc->txnpages_max ); /* Root -> a -> b. Inserts on b under a's blockhash land in a's blockcache and span three pages; cancelling a returns them. */ @@ -92,13 +369,12 @@ test_bench_sizing( ulong max_live_slots, ulong const txn_cnt = 2UL*FD_TXNCACHE_TXNS_PER_PAGE+1UL; for( ulong i=0UL; itxnpages_free_cnt==shtc->max_txnpages-3UL ); + FD_TEST( shtc->txnpages_free_cnt==shtc->txnpages_max-3UL ); for( ulong i=0UL; itxnpages_free_cnt==shtc->max_txnpages ); - FD_TEST( shtc->mutation_gen==mutation_gen && shtc->root_gen==root_gen ); /* unrooted attach, finalize, insert and cancel bump neither */ + FD_TEST( shtc->txnpages_free_cnt==shtc->txnpages_max ); fd_txncache_fork_id_t c = fd_txncache_attach_child( tc, root ); fd_txncache_insert( tc, c, BLOCKHASH(1UL), TXNHASH(7UL) ); @@ -106,19 +382,39 @@ test_bench_sizing( ulong max_live_slots, FD_TEST( !fd_txncache_query( tc, c, BLOCKHASH(1UL), TXNHASH(8UL) ) ); fd_txncache_reset( tc ); - FD_TEST( shtc->txnpages_free_cnt==shtc->max_txnpages ); + FD_TEST( shtc->txnpages_free_cnt==shtc->txnpages_max ); FD_TEST( !munmap( mem, sz ) ); } +/* Run the existing pruning/compaction regressions with one RAM page + as well as the fully resident configuration. */ +static int test_spill_fd = -1; +static fd_txncache_shmem_t * +test_shmem_new( void * mem, + ulong live, + ulong txns, + ulong seed ) { + ulong cache_footprint = test_spill_fd>=0 ? sizeof(fd_txncache_txnpage_t) : ULONG_MAX; + fd_txncache_shmem_t * sh = fd_txncache_shmem_join( fd_txncache_shmem_new( mem, live, txns, cache_footprint, seed ) ); + return sh; +} + +static fd_txncache_t * +test_join( void * mem, + fd_txncache_shmem_t * sh ) { + fd_txncache_t * tc = fd_txncache_join( fd_txncache_new( mem, sh, test_spill_fd ) ); + return tc; +} + void test0( uchar * scratch0, uchar * scratch1 ) { FD_LOG_NOTICE(( "TEST 0" )); - fd_txncache_shmem_t * shtc = fd_txncache_shmem_join( fd_txncache_shmem_new( scratch0, 4UL, 4UL, 0UL ) ); + fd_txncache_shmem_t * shtc = test_shmem_new( scratch0, 4UL, 4UL, 0UL ); FD_TEST( shtc ); - fd_txncache_t * tc = fd_txncache_join( fd_txncache_new( scratch1, shtc ) ); + fd_txncache_t * tc = test_join( scratch1, shtc ); FD_TEST( tc ); fd_txncache_fork_id_t root = fd_txncache_attach_child( tc, NULL_FORK ); @@ -143,18 +439,18 @@ void test_new_join( uchar * scratch0 ) { FD_LOG_NOTICE(( "TEST NEW" )); - FD_TEST( fd_txncache_shmem_new( NULL, 1UL, 1UL, 0UL )==NULL ); /* null shmem */ - FD_TEST( fd_txncache_shmem_new( (void *)0x1UL, 1UL, 1UL, 0UL )==NULL ); /* misaligned shmem */ - FD_TEST( fd_txncache_shmem_new( scratch0, 0UL, 1UL, 0UL )==NULL ); /* 0 max_live_slots */ - FD_TEST( fd_txncache_shmem_new( scratch0, 2UL, 0UL, 0UL )==NULL ); /* 0 max_txn_per_slot */ + FD_TEST( fd_txncache_shmem_new( NULL, 1UL, 1UL, ULONG_MAX, 0UL )==NULL ); /* null shmem */ + FD_TEST( fd_txncache_shmem_new( (void *)0x1UL, 1UL, 1UL, ULONG_MAX, 0UL )==NULL ); /* misaligned shmem */ + FD_TEST( fd_txncache_shmem_new( scratch0, 0UL, 1UL, ULONG_MAX, 0UL )==NULL ); /* 0 max_live_slots */ + FD_TEST( fd_txncache_shmem_new( scratch0, 2UL, 0UL, ULONG_MAX, 0UL )==NULL ); /* 0 max_txn_per_slot */ - FD_TEST( fd_txncache_shmem_new( scratch0, 1UL, 1UL, 0UL ) ); - FD_TEST( fd_txncache_shmem_new( scratch0, 2UL, 2UL, 0UL ) ); - FD_TEST( fd_txncache_shmem_new( scratch0, 2UL, 2UL, 0UL ) ); - FD_TEST( fd_txncache_shmem_new( scratch0, 4096UL, fd_ulong_pow2_up( FD_MAX_TXN_PER_SLOT ), 0UL ) ); - FD_TEST( fd_txncache_shmem_new( scratch0, 512UL, fd_ulong_pow2_up( FD_MAX_TXN_PER_SLOT ), 0UL ) ); - FD_TEST( fd_txncache_shmem_new( scratch0, 512UL, 1UL, 0UL ) ); - FD_TEST( fd_txncache_shmem_new( scratch0, 1UL, 1UL, 0UL ) ); + FD_TEST( fd_txncache_shmem_new( scratch0, 1UL, 1UL, ULONG_MAX, 0UL ) ); + FD_TEST( fd_txncache_shmem_new( scratch0, 2UL, 2UL, ULONG_MAX, 0UL ) ); + FD_TEST( fd_txncache_shmem_new( scratch0, 2UL, 2UL, ULONG_MAX, 0UL ) ); + FD_TEST( fd_txncache_shmem_new( scratch0, 4096UL, fd_ulong_pow2_up( FD_MAX_TXN_PER_SLOT ), ULONG_MAX, 0UL ) ); + FD_TEST( fd_txncache_shmem_new( scratch0, 512UL, fd_ulong_pow2_up( FD_MAX_TXN_PER_SLOT ), ULONG_MAX, 0UL ) ); + FD_TEST( fd_txncache_shmem_new( scratch0, 512UL, 1UL, ULONG_MAX, 0UL ) ); + FD_TEST( fd_txncache_shmem_new( scratch0, 1UL, 1UL, ULONG_MAX, 0UL ) ); FD_LOG_NOTICE(( "TEST JOIN" )); @@ -177,9 +473,9 @@ test_scratch( uchar * scratch0, ulong const max_live_slots = 4UL; ulong const max_txn_per_slot = FD_PACK_MAX_TXNCACHE_TXN_PER_SLOT; - fd_txncache_shmem_t * shtc = fd_txncache_shmem_join( fd_txncache_shmem_new( scratch0, max_live_slots, max_txn_per_slot, 0UL ) ); + fd_txncache_shmem_t * shtc = test_shmem_new( scratch0, max_live_slots, max_txn_per_slot, 0UL ); FD_TEST( shtc ); - fd_txncache_t * tc = fd_txncache_join( fd_txncache_new( scratch1, shtc ) ); + fd_txncache_t * tc = test_join( scratch1, shtc ); FD_TEST( tc ); ulong sz = 0UL; @@ -187,16 +483,13 @@ test_scratch( uchar * scratch0, FD_TEST( scratch ); FD_TEST( sz ); - ulong footprint = fd_txncache_shmem_footprint( max_live_slots, max_txn_per_slot ); + ulong footprint = fd_txncache_shmem_footprint( max_live_slots, max_txn_per_slot, ULONG_MAX ); FD_TEST( scratch>=scratch0 ); FD_TEST( scratch+sz<=scratch0+footprint ); FD_TEST( sz>=FD_TXNCACHE_MAX_SLOT_DELTAS*max_txn_per_slot*sizeof(fd_txncache_single_txn_t) ); - ulong mutation_gen = shtc->mutation_gen; - ulong root_gen = shtc->root_gen; fd_txncache_reset( tc ); - FD_TEST( shtc->mutation_gen==mutation_gen+2UL && shtc->root_gen==root_gen+1UL ); fd_memset( scratch, 0xFF, sz ); fd_txncache_fork_id_t root = fd_txncache_attach_child( tc, NULL_FORK ); @@ -211,15 +504,9 @@ test_scratch( uchar * scratch0, FD_TEST( !fd_txncache_query( tc, slot1, BLOCKHASH(1UL), TXNHASH(2UL) ) ); FD_TEST( fd_txncache_query( tc, slot1, BLOCKHASH(1UL), TXNHASH(5UL) ) ); - mutation_gen = shtc->mutation_gen; - root_gen = shtc->root_gen; fd_txncache_reset( tc ); - FD_TEST( shtc->mutation_gen==mutation_gen+2UL && shtc->root_gen==root_gen+1UL ); fd_memset( scratch, 0xA5, 4096UL ); - mutation_gen = shtc->mutation_gen; - root_gen = shtc->root_gen; fd_txncache_reset( tc ); - FD_TEST( shtc->mutation_gen==mutation_gen+2UL && shtc->root_gen==root_gen+1UL ); for( ulong i=0UL; i<4096UL; i++ ) FD_TEST( scratch[ i ]==0xA5 ); ulong sz2 = 0UL; @@ -232,24 +519,19 @@ test_advance_root( uchar * scratch0, uchar * scratch1 ) { FD_LOG_NOTICE(( "TEST ADVANCE ROOT" )); - fd_txncache_shmem_t * shtc = fd_txncache_shmem_join( fd_txncache_shmem_new( scratch0, 4UL, 4UL, 0UL ) ); + fd_txncache_shmem_t * shtc = test_shmem_new( scratch0, 4UL, 4UL, 0UL ); FD_TEST( shtc ); - fd_txncache_t * tc = fd_txncache_join( fd_txncache_new( scratch1, shtc ) ); + fd_txncache_t * tc = test_join( scratch1, shtc ); FD_TEST( tc ); fd_txncache_fork_id_t slot = fd_txncache_attach_child( tc, NULL_FORK ); fd_txncache_finalize_fork( tc, slot, 0UL, BLOCKHASH(0UL) ); - ulong mutation_gen = shtc->mutation_gen; - ulong root_gen = shtc->root_gen; for( ulong i=0UL; i<8192UL; i++ ) { slot = fd_txncache_attach_child( tc, slot ); fd_txncache_insert( tc, slot, BLOCKHASH(i), TXNHASH(i) ); fd_txncache_finalize_fork( tc, slot, 0UL, BLOCKHASH(i+1UL) ); fd_txncache_advance_root( tc, slot ); - mutation_gen += 2UL; - root_gen++; - FD_TEST( shtc->mutation_gen==mutation_gen && shtc->root_gen==root_gen ); } FD_TEST( !fd_txncache_query( tc, slot, BLOCKHASH(8191UL), TXNHASH(8190UL) ) ); @@ -263,7 +545,6 @@ test_advance_root( uchar * scratch0, slot = fd_txncache_attach_child( tc, slot ); fd_txncache_finalize_fork( tc, slot, 0UL, BLOCKHASH(8193UL) ); fd_txncache_advance_root( tc, slot ); - FD_TEST( shtc->mutation_gen==mutation_gen+2UL && shtc->root_gen==root_gen+1UL ); } void @@ -282,9 +563,9 @@ test_purge_stale( uchar * scratch0, returns the rest to the pool: only the valid_pre_purge txns survive the purge, so ceil(valid_pre_purge/txns_per_page)==1. */ - fd_txncache_shmem_t * shtc = fd_txncache_shmem_join( fd_txncache_shmem_new( scratch0, 4UL, max_txn_per_slot, 0UL ) ); + fd_txncache_shmem_t * shtc = test_shmem_new( scratch0, 4UL, max_txn_per_slot, 0UL ); FD_TEST( shtc ); - fd_txncache_t * tc = fd_txncache_join( fd_txncache_new( scratch1, shtc ) ); + fd_txncache_t * tc = test_join( scratch1, shtc ); FD_TEST( tc ); ulong const max_active_slots = FD_TXNCACHE_MAX_BLOCKHASH_DISTANCE+4UL; @@ -366,10 +647,7 @@ test_purge_stale( uchar * scratch0, retries and succeeds. */ FD_LOG_NOTICE(( "inserting trigger txn (should trigger purge_stale)" )); - ulong mutation_gen = shtc->mutation_gen; - ulong root_gen = shtc->root_gen; fd_txncache_insert( tc, query_fork, BLOCKHASH(0UL), TXNHASH(valid_pre_purge) ); - FD_TEST( shtc->mutation_gen==mutation_gen+2UL && shtc->root_gen==root_gen ); FD_TEST( (ulong)shtc->txnpages_free_cnt==max_txnpages-1UL ); @@ -395,6 +673,17 @@ test_purge_stale( uchar * scratch0, for( ulong i=0UL; i<10UL; i++ ) { FD_TEST( !fd_txncache_query( tc, query_fork, BLOCKHASH(0UL), TXNHASH(stale_id_base + i) ) ); } + + if( test_spill_fd>=0 ) { + /* Compaction with one frame must preserve both old and new blockcaches. */ + fd_txncache_finalize_fork( tc, query_fork, 0UL, BLOCKHASH(20000UL) ); + fd_txncache_fork_id_t child = fd_txncache_attach_child( tc, query_fork ); + fd_txncache_insert( tc, child, BLOCKHASH(20000UL), TXNHASH(20000UL) ); + uint page = tc->blockcache_pool[ query_fork.val ].shmem->page_head; + while( tc->page_meta[ page ].next!=UINT_MAX ) page = tc->page_meta[ page ].next; + FD_TEST( tc->page_meta[ page ].frame!=UINT_MAX ); + FD_TEST( fd_txncache_query( tc, child, BLOCKHASH(20000UL), TXNHASH(20000UL) ) ); + } } void @@ -422,9 +711,9 @@ test_purge_stale_global( uchar * scratch0, ulong const max_txnpages = fd_txncache_max_txnpages ( max_active_slots, max_txn_per_slot ); ulong const max_txnpages_per_blockhash = fd_txncache_max_txnpages_per_blockhash( max_active_slots, max_txn_per_slot ); - fd_txncache_shmem_t * shtc = fd_txncache_shmem_join( fd_txncache_shmem_new( scratch0, max_live_slots, max_txn_per_slot, 0UL ) ); + fd_txncache_shmem_t * shtc = test_shmem_new( scratch0, max_live_slots, max_txn_per_slot, 0UL ); FD_TEST( shtc ); - fd_txncache_t * tc = fd_txncache_join( fd_txncache_new( scratch1, shtc ) ); + fd_txncache_t * tc = test_join( scratch1, shtc ); FD_TEST( tc ); /* Step 1: Create root R0 with blockhash 0. */ @@ -556,9 +845,9 @@ test_advance_past_minority_then_purge( uchar * scratch0, repointed from loser_b to winner) and winner->sibling_id (must have been reset from loser_a to none). */ - fd_txncache_shmem_t * shtc = fd_txncache_shmem_join( fd_txncache_shmem_new( scratch0, 4UL, 4UL, 0UL ) ); + fd_txncache_shmem_t * shtc = test_shmem_new( scratch0, 4UL, 4UL, 0UL ); FD_TEST( shtc ); - fd_txncache_t * tc = fd_txncache_join( fd_txncache_new( scratch1, shtc ) ); + fd_txncache_t * tc = test_join( scratch1, shtc ); FD_TEST( tc ); fd_txncache_fork_id_t root = fd_txncache_attach_child( tc, NULL_FORK ); @@ -662,15 +951,26 @@ main( int argc, char ** argv ) { fd_boot( &argc, &argv ); + test_spill_page_cache(); + test_spill_page_reuse(); + test_spill_io_failure( 0 ); + test_spill_io_failure( 1 ); + test_spill(); +# if FD_HAS_RACESAN + test_fetch_unlocked(); +# endif + test_join_metrics(); test_bucket_cnt(); test_page_sizing(); - ulong max_footprint_shmem = fd_txncache_shmem_footprint( 4096UL, FD_MAX_TXN_PER_SLOT ); + ulong max_footprint_shmem = fd_txncache_shmem_footprint( 4096UL, fd_ulong_pow2_up( FD_MAX_TXN_PER_SLOT ), ULONG_MAX ); ulong max_footprint_local = fd_txncache_footprint( FD_MAX_TXN_PER_SLOT ); ulong max_footprint = fd_ulong_align_up( max_footprint_shmem, 4096UL ) + max_footprint_local; - uchar * scratch0 = fd_shmem_acquire( 4096UL, 1UL+(max_footprint/4096UL), 0UL ); - FD_TEST( scratch0 ); + /* Fully in-memory sizing tests reserve a large pool but touch only + a small fraction of its pages. */ + uchar * scratch0 = mmap( NULL, max_footprint, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_NORESERVE, -1, 0 ); + FD_TEST( scratch0!=MAP_FAILED ); uchar * scratch1 = scratch0+fd_ulong_align_up( max_footprint_shmem, 4096UL ); @@ -682,8 +982,20 @@ main( int argc, test_purge_stale( scratch0, scratch1, 256UL, 2000000UL ); /* Several pages per blockhash. */ test_purge_stale_global( scratch0, scratch1 ); test_advance_past_minority_then_purge( scratch0, scratch1 ); - test_bench_sizing( 4UL, sizeof(ushort) ); /* 29,624 txnpages */ - test_bench_sizing( 560UL, sizeof(uint) ); /* 66,111 txnpages, just past the ushort range */ + char path[] = "/tmp/fd-txncache-purge-XXXXXX"; + test_spill_fd = mkstemp( path ); + FD_TEST( test_spill_fd>=0 ); + FD_TEST( !fcntl( test_spill_fd, F_SETFL, O_DIRECT ) ); + FD_TEST( !unlink( path ) ); + test0( scratch0, scratch1 ); + test_advance_root( scratch0, scratch1 ); + test_purge_stale( scratch0, scratch1, 256UL, 2000000UL ); + test_advance_past_minority_then_purge( scratch0, scratch1 ); + FD_TEST( !close( test_spill_fd ) ); + test_spill_fd = -1; + test_bench_sizing( 4UL ); /* 29,624 txnpages */ + test_bench_sizing( 560UL ); /* 66,111 txnpages */ + FD_TEST( !munmap( scratch0, max_footprint ) ); FD_LOG_NOTICE(( "pass" )); fd_halt(); diff --git a/src/flamenco/runtime/tests/fd_svm_mini.c b/src/flamenco/runtime/tests/fd_svm_mini.c index bbabb2f8bda..d4dc1bb2a3b 100644 --- a/src/flamenco/runtime/tests/fd_svm_mini.c +++ b/src/flamenco/runtime/tests/fd_svm_mini.c @@ -137,7 +137,7 @@ fd_svm_mini_wksp_data_max( fd_svm_mini_limits_t const * limits ) { ulong joiner_cnt = fd_ulong_max( limits->accdb_joiner_cnt, 1UL ); ulong pcache_sz = fd_progcache_shmem_footprint( txn_max, fd_progcache_shmem_min_sz( txn_max ) ); - ulong txncache_shmem_sz = fd_txncache_shmem_footprint( txn_max, limits->max_txn_per_slot ); + ulong txncache_shmem_sz = fd_txncache_shmem_footprint( txn_max, limits->max_txn_per_slot, ULONG_MAX ); ulong txncache_sz = fd_txncache_footprint( txn_max ); ulong banks_sz = fd_banks_footprint( txn_max, 8UL, limits->max_stake_accounts, limits->max_vote_accounts ); ulong runtime_stack_sz = fd_runtime_stack_footprint( limits->max_vote_accounts, limits->max_vote_accounts, limits->max_stake_accounts ); @@ -176,10 +176,10 @@ fd_svm_mini_create( fd_wksp_t * wksp, ulong progcache_sz = fd_progcache_shmem_min_sz( txn_max ); ulong pcache_sz = fd_progcache_shmem_footprint( txn_max, progcache_sz ); - ulong txncache_shmem_sz = fd_txncache_shmem_footprint( txn_max, limits->max_txn_per_slot ); + ulong txncache_shmem_sz = fd_txncache_shmem_footprint( txn_max, limits->max_txn_per_slot, ULONG_MAX ); ulong txncache_sz = fd_txncache_footprint( txn_max ); - ulong banks_sz = fd_banks_footprint( txn_max, 8UL, limits->max_stake_accounts, limits->max_vote_accounts ); - ulong runtime_stack_sz = fd_runtime_stack_footprint( limits->max_vote_accounts, limits->max_vote_accounts, limits->max_stake_accounts ); + ulong banks_sz = fd_banks_footprint( txn_max, 8UL, limits->max_stake_accounts, limits->max_vote_accounts ); + ulong runtime_stack_sz = fd_runtime_stack_footprint( limits->max_vote_accounts, limits->max_vote_accounts, limits->max_stake_accounts ); ulong accdb_shmem_sz = fd_accdb_shmem_footprint( limits->max_accounts, limits->max_live_slots, TEST_WRITES_PER_SLOT, TEST_PARTITION_CNT, @@ -205,7 +205,7 @@ fd_svm_mini_create( fd_wksp_t * wksp, fd_memset( mini, 0, sizeof(fd_svm_mini_t) ); mini->wksp = wksp; - fd_txncache_shmem_t * shtxncache = fd_txncache_shmem_join( fd_txncache_shmem_new( txncache_shmem, txn_max, limits->max_txn_per_slot, 0UL ) ); + fd_txncache_shmem_t * shtxncache = fd_txncache_shmem_join( fd_txncache_shmem_new( txncache_shmem, txn_max, limits->max_txn_per_slot, ULONG_MAX, 0UL ) ); if( FD_UNLIKELY( !shtxncache ) ) FD_LOG_ERR(( "fd_txncache_shmem_new failed" )); /* Create accdb backed by memfd */ @@ -233,7 +233,7 @@ fd_svm_mini_create( fd_wksp_t * wksp, FD_TEST( fd_progcache_join( mini->progcache, pcache_mem, scratch, FD_PROGCACHE_SCRATCH_FOOTPRINT ) ); mini->txncache_shmem = shtxncache; - FD_TEST( (mini->txncache = fd_txncache_join( fd_txncache_new( txncache_mem, shtxncache ) )) ); + FD_TEST( (mini->txncache = fd_txncache_join( fd_txncache_new( txncache_mem, shtxncache, -1 ) )) ); mini->banks = fd_banks_join( fd_banks_new( banks_mem, FD_STAKE_DELEGATIONS_FD, txn_max, 8UL, From e104cdba3d0a7226440104c0d93c2645f9c4cbf5 Mon Sep 17 00:00:00 2001 From: Ishan Bhatt Date: Thu, 29 Oct 2026 20:50:56 +0000 Subject: [PATCH 2/2] snapin: stage status cache groups in the txncache spill file --- src/app/firedancer/config/default.toml | 5 +- src/app/firedancer/topology.c | 7 -- src/discof/restore/fd_snapin_tile.c | 57 +++++---- src/discof/restore/test_snapin_tile.c | 165 +++++++++++++------------ src/flamenco/runtime/fd_txncache.c | 67 ++++++---- src/flamenco/runtime/fd_txncache.h | 41 +++--- src/flamenco/runtime/test_txncache.c | 53 ++++---- 7 files changed, 210 insertions(+), 185 deletions(-) diff --git a/src/app/firedancer/config/default.toml b/src/app/firedancer/config/default.toml index 5bde1a775a5..8a4105fdd1a 100644 --- a/src/app/firedancer/config/default.toml +++ b/src/app/firedancer/config/default.toml @@ -829,10 +829,7 @@ telemetry = true # Memory budget for cached transactions in MiB, excluding metadata. # When full, the cache spills to paths.txncache. This value has - # been chosen and tuned around mainnet network behavior. Lowering - # this value risks introducing performance regressions. To support - # worst-case snapshot loading for mainnet, the cache size must be at - # least 565 on mainnet. + # been chosen and tuned around mainnet network behavior. # # It is not recommended to change this value, except in development # or test environments to decrease memory usage to test edge cases. diff --git a/src/app/firedancer/topology.c b/src/app/firedancer/topology.c index ada5258a934..0122a5c21a0 100644 --- a/src/app/firedancer/topology.c +++ b/src/app/firedancer/topology.c @@ -36,7 +36,6 @@ #include "../../flamenco/capture/fd_solcap_writer.h" #include "../../flamenco/progcache/fd_progcache.h" #include "../../flamenco/runtime/fd_cost_tracker.h" -#include "../../flamenco/runtime/fd_txncache.h" #include #include @@ -209,12 +208,6 @@ setup_topo_txncache( fd_topo_t * topo, ulong max_live_slots, ulong max_txn_per_slot, ulong cache_footprint ) { - ulong scratch_min_sz = fd_txncache_snapin_scratch_min_sz( max_txn_per_slot ); - if( FD_UNLIKELY( cache_footprint>20, (scratch_min_sz+(1UL<<20)-1UL)>>20 )); - } - ulong seed; FD_TEST( fd_rng_secure( &seed, sizeof( ulong ) ) ); diff --git a/src/discof/restore/fd_snapin_tile.c b/src/discof/restore/fd_snapin_tile.c index 092d5ae496c..3a8897fc81b 100644 --- a/src/discof/restore/fd_snapin_tile.c +++ b/src/discof/restore/fd_snapin_tile.c @@ -134,7 +134,15 @@ struct blockhash_group { typedef struct blockhash_group blockhash_group_t; -FD_STATIC_ASSERT( sizeof(blockhash_group_t)==FD_TXNCACHE_SNAPIN_GROUP_SZ, blockhash_group ); +FD_STATIC_ASSERT( sizeof(blockhash_group_t)==40UL, blockhash_group ); + +/* Each slot delta's groups are staged in the txncache at a fixed + stride of txncache_staging_row_sz bytes. */ + +FD_FN_CONST static inline ulong +txncache_staging_row_sz( ulong max_groups_per_slot ) { + return fd_ulong_align_up( max_groups_per_slot*sizeof(blockhash_group_t), FD_TXNCACHE_SNAPIN_STAGE_ALIGN ); +} /* After filtering with the recent blockhash queue, there is at most one group per retained (slot, recent blockhash) pair. Filtering can only @@ -224,7 +232,7 @@ struct fd_snapin_lead { fd_sysvar_cache_t sysvar_cache[1]; /* verify_sysvars scratch */ - blockhash_group_t * blockhash_groups; + blockhash_group_t * blockhash_groups; /* groups of one staged slot delta */ ulong blockhash_groups_cnt; /* every group parsed, including those from dropped slots */ recent_blockhash_group_t * recent_groups; ulong recent_groups_len; @@ -408,6 +416,7 @@ scratch_footprint( fd_topo_tile_t const * tile ) { l = FD_LAYOUT_APPEND( l, fd_slot_delta_parser_align(), fd_slot_delta_parser_footprint() ); l = FD_LAYOUT_APPEND( l, alignof(recent_blockhash_group_t), sizeof(recent_blockhash_group_t)*FD_SNAPIN_MAX_RECENT_GROUPS ); l = FD_LAYOUT_APPEND( l, alignof(fd_sstxncache_hash_t), sizeof(fd_sstxncache_hash_t)*FD_TXNCACHE_MAX_SLOT_DELTAS*2UL*tile->snapin.max_txn_per_slot ); + l = FD_LAYOUT_APPEND( l, FD_TXNCACHE_SNAPIN_STAGE_ALIGN, txncache_staging_row_sz( tile->snapin.max_txn_per_slot ) ); } return FD_LAYOUT_FINI( l, scratch_align() ); } @@ -801,25 +810,17 @@ transition_malformed( fd_snapin_tile_t * ctx, fd_stem_publish( stem, ctx->ct_out.idx, FD_SNAPSHOT_MSG_CTRL_ERROR, 0UL, 0UL, 0UL, 0UL, 0UL ); } -static blockhash_group_t * -txncache_staging_groups_join( void * scratch, - ulong scratch_sz, - ulong groups_max ) { - ulong start = fd_ulong_align_up( (ulong)scratch, alignof(blockhash_group_t) ); - ulong pad = start-(ulong)scratch; - if( FD_UNLIKELY( pad>scratch_sz ) ) return NULL; - if( FD_UNLIKELY( scratch_sz-padlead.txncache, &scratch_sz ); - ulong groups_max = FD_TXNCACHE_MAX_SLOT_DELTAS*ctx->lead.txncache_max_groups_per_slot; - blockhash_group_t * groups = txncache_staging_groups_join( scratch, scratch_sz, groups_max ); - if( FD_UNLIKELY( !groups ) ) FD_LOG_ERR(( "txncache scratch (%lu bytes) too small to stage %lu blockhash groups (%lu bytes); increase runtime.transaction_cache_size_mib", scratch_sz, groups_max, groups_max*sizeof(blockhash_group_t) )); - return groups; +static void +txncache_staging_row_io( fd_snapin_tile_t * ctx, + ulong slot_idx, + int write ) { + if( FD_UNLIKELY( slot_idx==ULONG_MAX ) ) return; + ulong sz = fd_ulong_align_up( ctx->lead.txncache_slots[ slot_idx ].group_cnt*sizeof(blockhash_group_t), FD_TXNCACHE_SNAPIN_STAGE_ALIGN ); + if( FD_LIKELY( sz ) ) fd_txncache_snapin_stage( ctx->lead.txncache, slot_idx*txncache_staging_row_sz( ctx->lead.txncache_max_groups_per_slot ), ctx->lead.blockhash_groups, sz, write ); } static void @@ -835,6 +836,8 @@ txncache_staging_reset( fd_snapin_tile_t * ctx ) { static ulong txncache_staging_slot_begin( fd_snapin_tile_t * ctx, ulong slot ) { + txncache_staging_row_io( ctx, ctx->lead.txncache_current_slot_idx, 1 ); + ulong candidate_idx; if( FD_LIKELY( ctx->lead.txncache_slots_lenlead.txncache_slots_len++; @@ -871,7 +874,7 @@ txncache_staging_group_begin( fd_snapin_tile_t * ctx, FD_TEST( slot_idxlead.txncache_slots_len ); txncache_staging_slot_t * staging_slot = &ctx->lead.txncache_slots[ slot_idx ]; FD_TEST( staging_slot->group_cntlead.txncache_max_groups_per_slot ); - blockhash_group_t * group = &ctx->lead.blockhash_groups[ slot_idx*ctx->lead.txncache_max_groups_per_slot+staging_slot->group_cnt ]; + blockhash_group_t * group = &ctx->lead.blockhash_groups[ staging_slot->group_cnt ]; memcpy( group->blockhash, blockhash, 32UL ); group->txnhash_offset = (uint)txnhash_offset; group->txncache_entry_cnt = 0U; @@ -894,7 +897,7 @@ txncache_staging_entry_add( fd_snapin_tile_t * ctx, FD_TEST( staging_slot->slot==slot ); FD_TEST( staging_slot->group_cnt ); FD_TEST( staging_slot->entry_cntlead.txncache_max_entries_per_slot ); - blockhash_group_t * group = &ctx->lead.blockhash_groups[ slot_idx*ctx->lead.txncache_max_groups_per_slot+staging_slot->group_cnt-1UL ]; + blockhash_group_t * group = &ctx->lead.blockhash_groups[ staging_slot->group_cnt-1UL ]; memcpy( ctx->lead.txncache_entries[ slot_idx*ctx->lead.txncache_max_entries_per_slot+staging_slot->entry_cnt ].txnhash, txnhash, sizeof(fd_sstxncache_hash_t) ); staging_slot->entry_cnt++; group->txncache_entry_cnt++; @@ -932,10 +935,13 @@ txncache_staging_filter_groups( fd_snapin_tile_t * ctx, return 1; } + txncache_staging_row_io( ctx, ctx->lead.txncache_current_slot_idx, 1 ); + ctx->lead.recent_groups_len = 0UL; for( ulong staging_idx=0UL; staging_idxlead.txncache_slots_len; staging_idx++ ) { txncache_staging_slot_t const * staging_slot = &ctx->lead.txncache_slots[ staging_idx ]; - blockhash_group_t const * groups = &ctx->lead.blockhash_groups[ staging_idx*ctx->lead.txncache_max_groups_per_slot ]; + txncache_staging_row_io( ctx, staging_idx, 0 ); + blockhash_group_t const * groups = ctx->lead.blockhash_groups; ulong entry_idx = staging_idx*ctx->lead.txncache_max_entries_per_slot; for( ulong i=0UL; igroup_cnt; i++ ) { @@ -1130,7 +1136,7 @@ populate_txncache( fd_snapin_tile_t * ctx, blockhash_map_ele_insert( blockhash_map, &blockhash_pool[ i ], blockhash_pool ); } - /* blockhash_groups aliases txncache memory, so filtering must finish + /* The staged groups live in the txncache, so filtering must finish before the first txncache insert. */ if( FD_UNLIKELY( txncache_staging_filter_groups( ctx, blockhash_map, blockhash_pool, snapshot_slot ) ) ) return 1; for( ulong i=0UL; ilead.recent_groups_len; i++ ) { @@ -1940,7 +1946,6 @@ handle_control_frag( fd_snapin_tile_t * ctx, fd_txncache_reset( ctx->lead.txncache ); txncache_staging_reset( ctx ); - ctx->lead.blockhash_groups = txncache_staging_scratch( ctx ); fd_ssmanifest_parser_init( ctx->lead.manifest_parser, fd_chunk_to_laddr( ctx->lead.manifest_out.mem, ctx->lead.manifest_out.chunk ) ); fd_slot_delta_parser_init( ctx->lead.slot_delta_parser ); fd_memset( &ctx->lead.flags, 0, sizeof(ctx->lead.flags) ); @@ -2413,6 +2418,7 @@ unprivileged_init( fd_topo_t const * topo, void * _sd_parser = FD_SCRATCH_ALLOC_APPEND( l, fd_slot_delta_parser_align(), fd_slot_delta_parser_footprint() ); ctx->lead.recent_groups = FD_SCRATCH_ALLOC_APPEND( l, alignof(recent_blockhash_group_t), sizeof(recent_blockhash_group_t)*FD_SNAPIN_MAX_RECENT_GROUPS ); ctx->lead.txncache_entries = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_sstxncache_hash_t), sizeof(fd_sstxncache_hash_t)*FD_TXNCACHE_MAX_SLOT_DELTAS*2UL*tile->snapin.max_txn_per_slot ); + ctx->lead.blockhash_groups = FD_SCRATCH_ALLOC_APPEND( l, FD_TXNCACHE_SNAPIN_STAGE_ALIGN, txncache_staging_row_sz( tile->snapin.max_txn_per_slot ) ); void * _txncache_shmem = fd_topo_obj_laddr( topo, tile->snapin.txncache_obj_id ); fd_txncache_shmem_t * txncache_shmem = fd_txncache_shmem_join( _txncache_shmem ); @@ -2420,7 +2426,6 @@ unprivileged_init( fd_topo_t const * topo, ctx->lead.txncache = fd_txncache_join( fd_txncache_new( _txncache, txncache_shmem, FD_TXNCACHE_FD ) ); FD_TEST( ctx->lead.txncache ); - ctx->lead.blockhash_groups = NULL; txncache_staging_reset( ctx ); ctx->lead.bank = fd_banks_init_bank( banks ); diff --git a/src/discof/restore/test_snapin_tile.c b/src/discof/restore/test_snapin_tile.c index 161d73887ba..deb43392ac7 100644 --- a/src/discof/restore/test_snapin_tile.c +++ b/src/discof/restore/test_snapin_tile.c @@ -123,9 +123,12 @@ test_ssparse_appendvec_parse( fd_ssparse_t * parser ); void mock_txncache_reset( fd_txncache_t * txncache ); -void * -mock_txncache_snapin_scratch( fd_txncache_t * txncache, - ulong * out_sz ); +void +mock_txncache_snapin_stage( fd_txncache_t * txncache, + ulong off, + void * buf, + ulong sz, + int write ); void mock_slot_delta_parser_init( fd_slot_delta_parser_t * parser ); @@ -170,7 +173,7 @@ test_stem_publish( fd_stem_context_t * stem, #define fd_accdb_snapshot_write_batch mock_accdb_snapshot_write_batch #define fd_accdb_read_one_nocache mock_accdb_read_one_nocache #define fd_txncache_reset mock_txncache_reset -#define fd_txncache_snapin_scratch mock_txncache_snapin_scratch +#define fd_txncache_snapin_stage mock_txncache_snapin_stage #define fd_ssmanifest_parser_init mock_ssmanifest_parser_init #define fd_slot_delta_parser_init mock_slot_delta_parser_init #define fd_stake_delegations_reset mock_stake_delegations_reset @@ -205,7 +208,7 @@ test_padded_sz( ulong used ) { #undef fd_ssload_manifest_validate #undef fd_slot_delta_parser_init #undef fd_ssmanifest_parser_init -#undef fd_txncache_snapin_scratch +#undef fd_txncache_snapin_stage #undef fd_txncache_reset #undef fd_accdb_read_one_nocache #undef fd_accdb_snapshot_write_batch @@ -250,7 +253,6 @@ void fd_stake_delegations_evict_fork( fd_stake_delegations_t * stake_delegatio derived staging bounds). */ #define TEST_MAX_GROUPS_PER_SLOT (FD_MAX_TXN_PER_SLOT) #define TEST_MAX_ENTRIES_PER_SLOT (2UL*FD_MAX_TXN_PER_SLOT) -#define TEST_MAX_STAGED_GROUPS (FD_TXNCACHE_MAX_SLOT_DELTAS*TEST_MAX_GROUPS_PER_SLOT) #define TEST_MAX_ENTRIES (FD_TXNCACHE_MAX_SLOT_DELTAS*TEST_MAX_ENTRIES_PER_SLOT) static fd_txncache_fork_id_t test_attached_fork_id[ FD_BLOCKHASHES_MAX ]; @@ -397,14 +399,22 @@ mock_accdb_snapshot_write_batch( fd_accdb_t * accdb, void mock_txncache_reset( fd_txncache_t * tc ) { (void)tc; } -void * -mock_txncache_snapin_scratch( fd_txncache_t * txncache, - ulong * out_sz ) { - if( FD_UNLIKELY( !txncache ) ) { - *out_sz = TEST_MAX_STAGED_GROUPS*sizeof(blockhash_group_t); - return (void *)4096UL; +/* Staging area for tests that stage before attaching a txncache. NULL + uses the txncache's staging area. */ +static uchar * test_stage_mem; + +void +mock_txncache_snapin_stage( fd_txncache_t * txncache, + ulong off, + void * buf, + ulong sz, + int write ) { + if( FD_UNLIKELY( test_stage_mem ) ) { + if( write ) fd_memcpy( test_stage_mem+off, buf, sz ); + else fd_memcpy( buf, test_stage_mem+off, sz ); + return; } - return fd_txncache_snapin_scratch( txncache, out_sz ); + fd_txncache_snapin_stage( txncache, off, buf, sz, write ); } void @@ -570,7 +580,8 @@ sync_ctx_init( fd_snapin_tile_t * ctx, ctx->tile_idx = 0UL; ctx->lane_cnt = lane_cnt; ctx->ct_out.idx = 0UL; - test_file_off = 0UL; + test_file_off = 0UL; + test_stage_mem = NULL; reset_attempt_state( ctx ); clear_control_barrier( ctx ); @@ -1761,8 +1772,11 @@ test_txncache_staging_ctx_init( fd_snapin_tile_t * ctx, ctx->lead.txncache_max_groups_per_slot = TEST_MAX_GROUPS_PER_SLOT; ctx->lead.txncache_max_entries_per_slot = TEST_MAX_ENTRIES_PER_SLOT; txncache_staging_reset( ctx ); - ctx->lead.blockhash_groups = fd_wksp_alloc_laddr( wksp, alignof(blockhash_group_t), TEST_MAX_STAGED_GROUPS*sizeof(blockhash_group_t), 1UL ); + ulong row_sz = txncache_staging_row_sz( TEST_MAX_GROUPS_PER_SLOT ); + ctx->lead.blockhash_groups = fd_wksp_alloc_laddr( wksp, FD_TXNCACHE_SNAPIN_STAGE_ALIGN, row_sz, 1UL ); FD_TEST( ctx->lead.blockhash_groups ); + test_stage_mem = fd_wksp_alloc_laddr( wksp, FD_TXNCACHE_SNAPIN_STAGE_ALIGN, FD_TXNCACHE_MAX_SLOT_DELTAS*row_sz, 1UL ); + FD_TEST( test_stage_mem ); test_txncache_staging_side_arrays_alloc( ctx, wksp ); } @@ -1770,7 +1784,7 @@ test_txncache_staging_ctx_init( fd_snapin_tile_t * ctx, static fd_txncache_t * new_txncache( fd_wksp_t * wksp, ulong max_txn_per_slot ) { - ulong cache_footprint = FD_TXNCACHE_MAX_SLOT_DELTAS*((max_txn_per_slot+1UL)/2UL)*sizeof(blockhash_group_t)+alignof(blockhash_group_t)-1UL; + ulong cache_footprint = FD_TXNCACHE_MAX_SLOT_DELTAS*txncache_staging_row_sz( (max_txn_per_slot+1UL)/2UL ); void * shmem = fd_wksp_alloc_laddr( wksp, fd_txncache_shmem_align(), fd_txncache_shmem_footprint( 1UL, max_txn_per_slot, cache_footprint ), 1UL ); FD_TEST( shmem ); fd_txncache_shmem_t * txncache_shmem = fd_txncache_shmem_join( fd_txncache_shmem_new( shmem, 1UL, max_txn_per_slot, cache_footprint, 0UL ) ); @@ -1783,50 +1797,27 @@ new_txncache( fd_wksp_t * wksp, return txncache; } -static void -test_txncache_staging_groups_fit_txncache_scratch( fd_wksp_t * wksp ) { - fd_txncache_t * txncache = new_txncache( wksp, FD_PACK_MAX_TXNCACHE_TXN_PER_SLOT ); - - ulong scratch_sz = 0UL; - void * scratch = fd_txncache_snapin_scratch( txncache, &scratch_sz ); - FD_TEST( scratch ); - - blockhash_group_t * groups = txncache_staging_groups_join( scratch, scratch_sz, TEST_MAX_STAGED_GROUPS ); - FD_TEST( groups ); - FD_TEST( fd_ulong_is_aligned( (ulong)groups, alignof(blockhash_group_t) ) ); - FD_TEST( (uchar *)groups>=(uchar *)scratch ); - FD_TEST( (uchar *)(groups+TEST_MAX_STAGED_GROUPS)<=(uchar *)scratch+scratch_sz ); +/* Builds a txncache with one frame, so snapshot loading stages most + groups in a spill file. */ +static fd_txncache_t * +new_spill_txncache( fd_wksp_t * wksp, + ulong max_txn_per_slot ) { + ulong cache_footprint = 1UL; + void * shmem = fd_wksp_alloc_laddr( wksp, fd_txncache_shmem_align(), fd_txncache_shmem_footprint( 1UL, max_txn_per_slot, cache_footprint ), 1UL ); + FD_TEST( shmem ); + fd_txncache_shmem_t * txncache_shmem = fd_txncache_shmem_join( fd_txncache_shmem_new( shmem, 1UL, max_txn_per_slot, cache_footprint, 0UL ) ); + FD_TEST( txncache_shmem ); - ulong ring_sz = TEST_MAX_STAGED_GROUPS*sizeof(blockhash_group_t); - FD_TEST( txncache_staging_groups_join( (void *)64UL, ring_sz, TEST_MAX_STAGED_GROUPS ) ); - FD_TEST( !txncache_staging_groups_join( (void *)64UL, ring_sz-1UL, TEST_MAX_STAGED_GROUPS ) ); - FD_TEST( !txncache_staging_groups_join( (void *)66UL, ring_sz, TEST_MAX_STAGED_GROUPS ) ); - FD_TEST( txncache_staging_groups_join( (void *)66UL, ring_sz+2UL, TEST_MAX_STAGED_GROUPS ) ); -} + char path[] = "/tmp/fd-snapin-spill-XXXXXX"; + int spill_fd = mkstemp( path ); + FD_TEST( spill_fd>=0 ); + FD_TEST( !unlink( path ) ); -/* The caller-provided cache budget must fit blockhash-group staging. - An exact page multiple also needs room for group-alignment padding. */ -static void -test_txncache_staging_bench_scratch( ulong max_txn_per_slot, - ulong cache_footprint ) { - ulong shmem_sz = fd_txncache_shmem_footprint( 4UL, 2UL*max_txn_per_slot, cache_footprint ); - ulong local_sz = fd_txncache_footprint( 4UL ); - ulong sz = shmem_sz+local_sz; - uchar * mem = mmap( NULL, sz, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0 ); - FD_TEST( mem!=MAP_FAILED ); - fd_txncache_shmem_t * shmem = fd_txncache_shmem_join( fd_txncache_shmem_new( mem, 4UL, 2UL*max_txn_per_slot, cache_footprint, 0UL ) ); - FD_TEST( shmem ); - fd_txncache_t * txncache = fd_txncache_join( fd_txncache_new( mem+shmem_sz, shmem, -1 ) ); + void * local = fd_wksp_alloc_laddr( wksp, fd_txncache_align(), fd_txncache_footprint( 1UL ), 1UL ); + FD_TEST( local ); + fd_txncache_t * txncache = fd_txncache_join( fd_txncache_new( local, txncache_shmem, spill_fd ) ); FD_TEST( txncache ); - - ulong scratch_sz; - void * scratch = fd_txncache_snapin_scratch( txncache, &scratch_sz ); - ulong groups_max = FD_TXNCACHE_MAX_SLOT_DELTAS*max_txn_per_slot; - blockhash_group_t * groups = txncache_staging_groups_join( scratch, scratch_sz, groups_max ); - FD_TEST( groups ); - groups[ groups_max-1UL ].txncache_entry_cnt = 2U; - FD_TEST( groups[ groups_max-1UL ].txncache_entry_cnt==2U ); - FD_TEST( !munmap( mem, sz ) ); + return txncache; } static void @@ -1872,7 +1863,7 @@ test_txncache_staging_evicted_slot_drops_groups( fd_wksp_t * wksp ) { FD_TEST( ctx->lead.txncache_slots[ oldest_idx ].group_cnt==1UL ); FD_TEST( ctx->lead.txncache_slots[ oldest_idx ].entry_cnt==2UL ); - blockhash_group_t const * group = &ctx->lead.blockhash_groups[ oldest_idx*TEST_MAX_GROUPS_PER_SLOT ]; + blockhash_group_t const * group = &ctx->lead.blockhash_groups[ 0 ]; FD_TEST( !memcmp( group->blockhash, blockhash_x, 32UL ) ); FD_TEST( group->txnhash_offset==3UL ); FD_TEST( group->txncache_entry_cnt==2UL ); @@ -2053,13 +2044,14 @@ test_txncache_staging_fits_one_gigantic_page( void ) { ulong footprint = scratch_footprint( &tile ); FD_TEST( footprint<(1UL<<30) ); - /* The staged entries scale with the per-slot limit. Both footprints - are rounded up to the scratch alignment, so the difference can land - on either side of the exact entries term by less than one - alignment. */ + /* The staged entries and the group buffer scale with the per-slot + limit. Both footprints are rounded up to the scratch alignment, so + the difference can land on either side of the exact term by less + than one alignment. */ tile.snapin.max_txn_per_slot = 2UL*FD_MAX_TXN_PER_SLOT; ulong delta = scratch_footprint( &tile )-footprint; - ulong entries = TEST_MAX_ENTRIES*sizeof(fd_sstxncache_hash_t); + ulong entries = TEST_MAX_ENTRIES*sizeof(fd_sstxncache_hash_t) + + txncache_staging_row_sz( 2UL*FD_MAX_TXN_PER_SLOT )-txncache_staging_row_sz( FD_MAX_TXN_PER_SLOT ); FD_TEST( delta+scratch_align()>entries && deltalead.blockhash_groups[ 3UL ].blockhash, blockhash, 32UL ) ); + FD_TEST( !memcmp( ctx->lead.blockhash_groups[ 0UL ].blockhash, blockhash, 32UL ) ); FD_TEST( ctx->lead.txncache_entries[ 6UL ].txnhash[ 0 ]==0x33 ); uchar __attribute__((aligned(alignof(blockhash_map_t)))) _map[ blockhash_map_footprint( 1024UL ) ]; @@ -2146,12 +2140,14 @@ test_txncache_staging_ignores_evicted_group_offsets( fd_wksp_t * wksp ) { } static void -test_populate_txncache_ctx_init( fd_snapin_tile_t * ctx, - fd_wksp_t * wksp ) { +test_populate_txncache_ctx_init_with( fd_snapin_tile_t * ctx, + fd_wksp_t * wksp, + fd_txncache_t * txncache ) { sync_ctx_init( ctx, 1UL, FD_SNAPSHOT_STATE_PROCESSING ); - ctx->lead.txncache = new_txncache( wksp, FD_PACK_MAX_TXNCACHE_TXN_PER_SLOT ); + ctx->lead.txncache = txncache; fd_txncache_reset( ctx->lead.txncache ); - ctx->lead.blockhash_groups = txncache_staging_scratch( ctx ); + ctx->lead.blockhash_groups = fd_wksp_alloc_laddr( wksp, FD_TXNCACHE_SNAPIN_STAGE_ALIGN, txncache_staging_row_sz( ctx->lead.txncache_max_groups_per_slot ), 1UL ); + FD_TEST( ctx->lead.blockhash_groups ); test_txncache_staging_side_arrays_alloc( ctx, wksp ); txncache_staging_reset( ctx ); @@ -2164,6 +2160,12 @@ test_populate_txncache_ctx_init( fd_snapin_tile_t * ctx, test_attached_fork_cnt = 0UL; } +static void +test_populate_txncache_ctx_init( fd_snapin_tile_t * ctx, + fd_wksp_t * wksp ) { + test_populate_txncache_ctx_init_with( ctx, wksp, new_txncache( wksp, FD_PACK_MAX_TXNCACHE_TXN_PER_SLOT ) ); +} + static void test_txncache_staging_rejects_oversized_slot_delta( fd_wksp_t * wksp ) { fd_snapin_tile_t * ctx = test_ctx; @@ -2283,10 +2285,7 @@ test_txnhash_init( uchar out[ static 32UL ], } static void -test_populate_txncache_slot_attribution( fd_wksp_t * wksp ) { - fd_snapin_tile_t * ctx = test_ctx; - test_populate_txncache_ctx_init( ctx, wksp ); - +test_populate_txncache_slot_attribution_run( fd_snapin_tile_t * ctx ) { fd_snapshot_manifest_blockhash_t blockhashes[ FD_BLOCKHASHES_MAX ] = {0}; test_populate_blockhashes_init( blockhashes ); uchar txn_1001_bh_1000[ 32UL ]; @@ -2324,6 +2323,22 @@ test_populate_txncache_slot_attribution( fd_wksp_t * wksp ) { FD_TEST( fd_txncache_query( ctx->lead.txncache, child_of_root, test_blockhash_1001, txn_1002_bh_1001 ) ); } +static void +test_populate_txncache_slot_attribution( fd_wksp_t * wksp ) { + fd_snapin_tile_t * ctx = test_ctx; + test_populate_txncache_ctx_init( ctx, wksp ); + test_populate_txncache_slot_attribution_run( ctx ); +} + +/* Same as above, but the frames hold one slot delta's groups, so the + others are staged in the spill file. */ +static void +test_populate_txncache_slot_attribution_spill( fd_wksp_t * wksp ) { + fd_snapin_tile_t * ctx = test_ctx; + test_populate_txncache_ctx_init_with( ctx, wksp, new_spill_txncache( wksp, FD_PACK_MAX_TXNCACHE_TXN_PER_SLOT ) ); + test_populate_txncache_slot_attribution_run( ctx ); +} + static void test_populate_txncache_rejects_invalid_blockhash_age( fd_wksp_t * wksp ) { fd_snapin_tile_t * ctx = test_ctx; @@ -3441,11 +3456,8 @@ main( int argc, fd_wksp_reset( wksp, 1UL ); test_streaming_stake_delegation( wksp ); fd_wksp_reset( wksp, 1UL ); test_snoop_outcomes( wksp ); fd_wksp_reset( wksp, 1UL ); test_snoop_incremental_fork( wksp ); - test_txncache_staging_bench_scratch( 529411UL, 3072UL<<20 ); /* benchmark TOML budget */ - test_txncache_staging_bench_scratch( 122881UL, FD_TXNCACHE_MAX_SLOT_DELTAS*122881UL*sizeof(blockhash_group_t)+alignof(blockhash_group_t)-1UL ); /* exact page multiple */ test_txncache_staging_entry_size(); test_txncache_staging_group_record_size(); - fd_wksp_reset( wksp, 1UL ); test_txncache_staging_groups_fit_txncache_scratch( wksp ); fd_wksp_reset( wksp, 1UL ); test_txncache_staging_evicts_oldest_slot( wksp ); fd_wksp_reset( wksp, 1UL ); test_txncache_staging_evicted_slot_drops_groups( wksp ); fd_wksp_reset( wksp, 1UL ); test_txncache_staging_rejects_group_overflow( wksp ); @@ -3462,6 +3474,7 @@ main( int argc, fd_wksp_reset( wksp, 1UL ); test_populate_txncache_slot_attribution_with_holes( wksp ); fd_wksp_reset( wksp, 1UL ); test_txncache_staging_populate_expired_by_index( wksp ); fd_wksp_reset( wksp, 1UL ); test_populate_txncache_slot_attribution( wksp ); + fd_wksp_reset( wksp, 1UL ); test_populate_txncache_slot_attribution_spill( wksp ); fd_wksp_reset( wksp, 1UL ); test_populate_txncache_rejects_invalid_blockhash_age( wksp ); fd_wksp_reset( wksp, 1UL ); test_populate_txncache_requires_snapshot_slot_delta( wksp ); fd_wksp_reset( wksp, 1UL ); test_populate_txncache_accepts_empty_rooted_status_cache( wksp ); diff --git a/src/flamenco/runtime/fd_txncache.c b/src/flamenco/runtime/fd_txncache.c index d4ba5cb57e7..888528bf7d2 100644 --- a/src/flamenco/runtime/fd_txncache.c +++ b/src/flamenco/runtime/fd_txncache.c @@ -147,28 +147,34 @@ lock_write( fd_txncache_t * tc ) FD_ACQUIRE( tc->shmem->lock ) FD_NO_THREAD_SAFE } static void -spill_io( fd_txncache_t * tc, - ulong page, - fd_txncache_txnpage_t * data, - int write ) { +spill_rw( fd_txncache_t * tc, + ulong off, + uchar * buf, + ulong sz, + int write ) { FD_TEST( tc->spill_fd>=0 ); - long t0 = fd_tickcount(); - ulong off = page*sizeof(*data); - ulong rem = sizeof(*data); - uchar * buf = (uchar *)data; - while( rem ) { - long n = write ? pwrite( tc->spill_fd, buf, rem, (off_t)off ) - : pread ( tc->spill_fd, buf, rem, (off_t)off ); + while( sz ) { + long n = write ? pwrite( tc->spill_fd, buf, sz, (off_t)off ) + : pread ( tc->spill_fd, buf, sz, (off_t)off ); if( FD_UNLIKELY( n<0L && errno==EINTR ) ) continue; - if( FD_UNLIKELY( n<0L ) ) FD_LOG_CRIT(( "txncache spill %s failed for page %lu at offset %lu (%ld, errno %d)", write ? "write" : "read", page, off, n, errno )); - if( FD_UNLIKELY( !n ) ) FD_LOG_CRIT(( "txncache spill %s for page %lu at offset %lu", write ? "write made no progress" : "read reached unexpected EOF", page, off )); - rem -= (ulong)n; - if( FD_UNLIKELY( rem && !fd_ulong_is_aligned( (ulong)n, FD_TXNCACHE_TXNPAGE_ALIGN ) ) ) { - FD_LOG_CRIT(( "txncache spill %s returned an unaligned short transfer for page %lu", write ? "write" : "read", page )); + if( FD_UNLIKELY( n<0L ) ) FD_LOG_CRIT(( "txncache spill %s failed at offset %lu (%ld, errno %d)", write ? "write" : "read", off, n, errno )); + if( FD_UNLIKELY( !n ) ) FD_LOG_CRIT(( "txncache spill %s at offset %lu", write ? "write made no progress" : "read reached unexpected EOF", off )); + sz -= (ulong)n; + if( FD_UNLIKELY( sz && !fd_ulong_is_aligned( (ulong)n, FD_TXNCACHE_TXNPAGE_ALIGN ) ) ) { + FD_LOG_CRIT(( "txncache spill %s returned an unaligned short transfer at offset %lu", write ? "write" : "read", off )); } buf += (ulong)n; off += (ulong)n; } +} + +static void +spill_io( fd_txncache_t * tc, + ulong page, + fd_txncache_txnpage_t * data, + int write ) { + long t0 = fd_tickcount(); + spill_rw( tc, page*sizeof(*data), (uchar *)data, sizeof(*data), write ); if( write ) FD_ATOMIC_FETCH_AND_ADD( &tc->shmem->page_spilled_cnt, 1UL ); else FD_ATOMIC_FETCH_AND_ADD( &tc->shmem->page_spill_read_cnt, 1UL ); FD_ATOMIC_FETCH_AND_ADD( &tc->shmem->spill_ticks, (ulong)(fd_tickcount()-t0) ); @@ -395,12 +401,31 @@ fd_txncache_metrics_snap( fd_txncache_t const * tc, out->frame_used = shmem->frames_max-fd_ulong_min( FD_VOLATILE_CONST( shmem->frames_free_cnt ), shmem->frames_max ); } -void * -fd_txncache_snapin_scratch( fd_txncache_t * tc, - ulong * out_sz ) { +FD_STATIC_ASSERT( !(sizeof(fd_txncache_txnpage_t)%FD_TXNCACHE_SNAPIN_STAGE_ALIGN), snapin_stage_align ); + +void +fd_txncache_snapin_stage( fd_txncache_t * tc, + ulong off, + void * buf, + ulong sz, + int write ) { FD_TEST_ERR( tc->shmem->txnpages_free_cnt==tc->shmem->txnpages_max ); - *out_sz = tc->shmem->frames_max*sizeof(fd_txncache_txnpage_t); - return tc->txnpages; + FD_TEST( fd_ulong_is_aligned( off, FD_TXNCACHE_SNAPIN_STAGE_ALIGN ) ); + FD_TEST( fd_ulong_is_aligned( sz, FD_TXNCACHE_SNAPIN_STAGE_ALIGN ) ); + FD_TEST( fd_ulong_is_aligned( (ulong)buf, FD_TXNCACHE_SNAPIN_STAGE_ALIGN ) ); + + ulong mem_sz = tc->shmem->frames_max*sizeof(fd_txncache_txnpage_t); + uchar * p = (uchar *)buf; + if( offtxnpages+off; + if( write ) fd_memcpy( mem, p, n ); + else fd_memcpy( p, mem, n ); + off += n; + p += n; + sz -= n; + } + if( sz ) spill_rw( tc, off-mem_sz, p, sz, write ); } FD_FN_PURE static inline ulong diff --git a/src/flamenco/runtime/fd_txncache.h b/src/flamenco/runtime/fd_txncache.h index 4ed6eb2e3e4..955e8c5ebd1 100644 --- a/src/flamenco/runtime/fd_txncache.h +++ b/src/flamenco/runtime/fd_txncache.h @@ -221,35 +221,24 @@ typedef struct fd_txncache_join_metrics fd_txncache_join_metrics_t; fd_txncache_join_metrics_t * fd_txncache_join_metrics( fd_txncache_t * tc ); -/* fd_txncache_snapin_scratch returns the txncache's transaction page - pool for use as scratch memory during snapshot loading, and sets - *out_sz to its size in bytes. +/* FD_TXNCACHE_SNAPIN_STAGE_ALIGN is the alignment of the offsets, + sizes and buffers of fd_txncache_snapin_stage. */ - Between a call to fd_txncache_reset and the first subsequent - fd_txncache_insert, the txncache neither reads nor writes page - contents. A page is fully reinitialized when an insert allocates it. - A caller that has just reset the txncache may therefore use the - region freely until it performs the first insert, at which point the - contents should be assumed clobbered. */ +#define FD_TXNCACHE_SNAPIN_STAGE_ALIGN (4096UL) -void * -fd_txncache_snapin_scratch( fd_txncache_t * tc, - ulong * out_sz ); - -/* FD_TXNCACHE_SNAPIN_GROUP_SZ is the size of one status cache blockhash - group that snapshot loading stages in the scratch region. */ - -#define FD_TXNCACHE_SNAPIN_GROUP_SZ (40UL) +/* fd_txncache_snapin_stage writes (write!=0) or reads sz bytes of buf + at byte offset off of a staging area for snapshot loading. The area + is the txncache's frames, then its spill file, which are unused + between fd_txncache_reset and the first fd_txncache_insert. The + first insert clobbers it. off, sz and buf must be multiples of + FD_TXNCACHE_SNAPIN_STAGE_ALIGN. */ -/* fd_txncache_snapin_scratch_min_sz returns the scratch size snapshot - loading needs to stage the status cache, one group per transaction - for each slot delta. max_txn_per_slot is the value passed to - fd_txncache_shmem_new, which counts each transaction twice. */ - -FD_FN_CONST static inline ulong -fd_txncache_snapin_scratch_min_sz( ulong max_txn_per_slot ) { - return FD_TXNCACHE_MAX_SLOT_DELTAS*(max_txn_per_slot/2UL)*FD_TXNCACHE_SNAPIN_GROUP_SZ; -} +void +fd_txncache_snapin_stage( fd_txncache_t * tc, + ulong off, + void * buf, + ulong sz, + int write ); /* fd_txncache_attach_child notifies the txncache that a new child bank has been created, off some parent. This must be called before any diff --git a/src/flamenco/runtime/test_txncache.c b/src/flamenco/runtime/test_txncache.c index b0ef4e6638c..3fc1c3db638 100644 --- a/src/flamenco/runtime/test_txncache.c +++ b/src/flamenco/runtime/test_txncache.c @@ -64,10 +64,15 @@ test_spill( void ) { for( ulong i=0UL; i<4UL*FD_TXNCACHE_TXNS_PER_PAGE; i+=127UL ) FD_TEST( fd_txncache_query( tc, a, BLOCKHASH(1UL), TXNHASH(100000UL+i) ) ); fd_txncache_reset( tc ); - ulong scratch_sz; - void * scratch = fd_txncache_snapin_scratch( tc, &scratch_sz ); - FD_TEST( scratch_sz==sizeof(fd_txncache_txnpage_t) ); - memset( scratch, 0xA5, scratch_sz ); + + /* One frame: the first page stages in memory, the second in the + spill file. */ + static uchar stage_in [ 2UL*sizeof(fd_txncache_txnpage_t) ] __attribute__((aligned(FD_TXNCACHE_SNAPIN_STAGE_ALIGN))); + static uchar stage_out[ 2UL*sizeof(fd_txncache_txnpage_t) ] __attribute__((aligned(FD_TXNCACHE_SNAPIN_STAGE_ALIGN))); + for( ulong i=0UL; i0 ); FD_TEST( !close( fd ) ); @@ -466,9 +471,9 @@ test_new_join( uchar * scratch0 ) { } static void -test_scratch( uchar * scratch0, - uchar * scratch1 ) { - FD_LOG_NOTICE(( "TEST SCRATCH" )); +test_snapin_stage( uchar * scratch0, + uchar * scratch1 ) { + FD_LOG_NOTICE(( "TEST SNAPIN STAGE" )); ulong const max_live_slots = 4UL; ulong const max_txn_per_slot = FD_PACK_MAX_TXNCACHE_TXN_PER_SLOT; @@ -478,20 +483,18 @@ test_scratch( uchar * scratch0, fd_txncache_t * tc = test_join( scratch1, shtc ); FD_TEST( tc ); - ulong sz = 0UL; - uchar * scratch = fd_txncache_snapin_scratch( tc, &sz ); - FD_TEST( scratch ); - FD_TEST( sz ); - - ulong footprint = fd_txncache_shmem_footprint( max_live_slots, max_txn_per_slot, ULONG_MAX ); - FD_TEST( scratch>=scratch0 ); - FD_TEST( scratch+sz<=scratch0+footprint ); - - FD_TEST( sz>=FD_TXNCACHE_MAX_SLOT_DELTAS*max_txn_per_slot*sizeof(fd_txncache_single_txn_t) ); + static uchar buf[ FD_TXNCACHE_SNAPIN_STAGE_ALIGN ] __attribute__((aligned(FD_TXNCACHE_SNAPIN_STAGE_ALIGN))); + /* A fully in-memory cache stages snapshot loading's worst case, one + 40 byte group per transaction for every slot delta, without a spill + file. */ + ulong row_sz = fd_ulong_align_up( (max_txn_per_slot/2UL)*40UL, FD_TXNCACHE_SNAPIN_STAGE_ALIGN ); + ulong last_off = FD_TXNCACHE_MAX_SLOT_DELTAS*row_sz-sizeof(buf); fd_txncache_reset( tc ); - fd_memset( scratch, 0xFF, sz ); + fd_memset( buf, 0x5A, sizeof(buf) ); + fd_txncache_snapin_stage( tc, last_off, buf, sizeof(buf), 1 ); + /* Inserts clobber the staging area. */ fd_txncache_fork_id_t root = fd_txncache_attach_child( tc, NULL_FORK ); fd_txncache_finalize_fork( tc, root, 0UL, BLOCKHASH(1UL) ); @@ -504,14 +507,14 @@ test_scratch( uchar * scratch0, FD_TEST( !fd_txncache_query( tc, slot1, BLOCKHASH(1UL), TXNHASH(2UL) ) ); FD_TEST( fd_txncache_query( tc, slot1, BLOCKHASH(1UL), TXNHASH(5UL) ) ); + /* Reset does not touch staged contents. */ fd_txncache_reset( tc ); - fd_memset( scratch, 0xA5, 4096UL ); + fd_memset( buf, 0xA5, sizeof(buf) ); + fd_txncache_snapin_stage( tc, 0UL, buf, sizeof(buf), 1 ); fd_txncache_reset( tc ); - for( ulong i=0UL; i<4096UL; i++ ) FD_TEST( scratch[ i ]==0xA5 ); - - ulong sz2 = 0UL; - FD_TEST( fd_txncache_snapin_scratch( tc, &sz2 )==scratch ); - FD_TEST( sz2==sz ); + fd_memset( buf, 0, sizeof(buf) ); + fd_txncache_snapin_stage( tc, 0UL, buf, sizeof(buf), 0 ); + for( ulong i=0UL; i