From 2b85c87930a998ba9a711900d8946248baeae1a2 Mon Sep 17 00:00:00 2001 From: Manik Jain Date: Wed, 28 Oct 2026 20:27:41 +0000 Subject: [PATCH 1/7] restore: ignore an incremental built on an older full --- src/discof/restore/utils/fd_http_resolver.c | 21 +++++++------- src/discof/restore/utils/test_http_resolver.c | 28 +++++++++++++++++++ 2 files changed, 39 insertions(+), 10 deletions(-) diff --git a/src/discof/restore/utils/fd_http_resolver.c b/src/discof/restore/utils/fd_http_resolver.c index 938d751c815..49266389c39 100644 --- a/src/discof/restore/utils/fd_http_resolver.c +++ b/src/discof/restore/utils/fd_http_resolver.c @@ -36,6 +36,7 @@ struct fd_ssresolve_peer { int is_https; ulong full_slot; ulong incr_slot; + ulong incr_base_slot; /* the full slot that the incremental is built upon */ uchar full_hash[ FD_HASH_FOOTPRINT ]; uchar incr_hash[ FD_HASH_FOOTPRINT ]; @@ -74,8 +75,9 @@ typedef struct fd_ssresolve_peer fd_ssresolve_peer_t; static inline void clear_peer_snapshot_data( fd_ssresolve_peer_t * peer ) { - peer->full_slot = FD_SSPEER_SLOT_UNKNOWN; - peer->incr_slot = FD_SSPEER_SLOT_UNKNOWN; + peer->full_slot = FD_SSPEER_SLOT_UNKNOWN; + peer->incr_slot = FD_SSPEER_SLOT_UNKNOWN; + peer->incr_base_slot = FD_SSPEER_SLOT_UNKNOWN; fd_memset( peer->full_hash, 0, FD_HASH_FOOTPRINT ); fd_memset( peer->incr_hash, 0, FD_HASH_FOOTPRINT ); } @@ -276,10 +278,7 @@ fd_http_resolver_add( fd_http_resolver_t * resolver, peer->addr = addr; peer->is_https = is_https; peer->fd.idx = ULONG_MAX; - peer->full_slot = FD_SSPEER_SLOT_UNKNOWN; - peer->incr_slot = FD_SSPEER_SLOT_UNKNOWN; - fd_memset( peer->full_hash, 0, FD_HASH_FOOTPRINT ); - fd_memset( peer->incr_hash, 0, FD_HASH_FOOTPRINT ); + clear_peer_snapshot_data( peer ); /* Create the selector entry now. Latency, full/incr slot, and full/incr hash are unknown at this point, so the peer only @@ -483,8 +482,8 @@ poll_resolve( fd_http_resolver_t * resolver, peer->full_slot = resolve_result.slot; fd_memcpy( peer->full_hash, resolve_result.hash, FD_HASH_FOOTPRINT ); } else { - peer->full_slot = resolve_result.base_slot; - peer->incr_slot = resolve_result.slot; + peer->incr_base_slot = resolve_result.base_slot; + peer->incr_slot = resolve_result.slot; fd_memcpy( peer->incr_hash, resolve_result.hash, FD_HASH_FOOTPRINT ); } } @@ -541,8 +540,10 @@ poll_advance( fd_http_resolver_t * resolver, deadline_list_ele_push_tail( resolver->valid, peer, resolver->pool ); remove_peer( resolver, peer->fd.idx ); - resolver->on_resolve_cb( resolver->cb_arg, &peer->key, peer->addr, peer->full_slot, peer->incr_slot, peer->full_hash, - peer->incr_slot!=FD_SSPEER_SLOT_UNKNOWN ? peer->incr_hash : NULL ); + int has_incr = peer->incr_slot!=FD_SSPEER_SLOT_UNKNOWN && peer->incr_base_slot==peer->full_slot; + resolver->on_resolve_cb( resolver->cb_arg, &peer->key, peer->addr, peer->full_slot, + has_incr ? peer->incr_slot : FD_SSPEER_SLOT_UNKNOWN, peer->full_hash, + has_incr ? peer->incr_hash : NULL ); } } } diff --git a/src/discof/restore/utils/test_http_resolver.c b/src/discof/restore/utils/test_http_resolver.c index 9fda89463e1..0a00ed88b33 100644 --- a/src/discof/restore/utils/test_http_resolver.c +++ b/src/discof/restore/utils/test_http_resolver.c @@ -246,6 +246,33 @@ test_full_absent_peer_rejected( void ) { FD_LOG_NOTICE(( "pass: test_full_absent_peer_rejected" )); } +/* The server has moved on to a newer full, but its newest incremental + is still built on the previous one. That incremental cannot be + paired with the full the server now offers. */ + +static void +test_incremental_on_older_full( void ) { + static char const * FULL_303_NEWER = + "HTTP/1.1 303 See Other\r\n" + "Location: /snapshot-300-" HASH_B58 ".tar.zst\r\n" + "Content-Length: 0\r\n\r\n"; + + resolve_record_t rec; + fd_sspeer_t best_full, best_incr; + run_resolve( FULL_303_NEWER, INCR_303, 1, &rec, &best_full, &best_incr ); + + FD_TEST( rec.called==1 ); + FD_TEST( rec.full_slot==300UL ); + FD_TEST( rec.incr_slot==FD_SSPEER_SLOT_UNKNOWN ); + FD_TEST( rec.incr_hash_null ); + + FD_TEST( best_full.addr.l ); + FD_TEST( best_full.full_slot==300UL ); + FD_TEST( !best_incr.addr.l ); + + FD_LOG_NOTICE(( "pass: test_incremental_on_older_full" )); +} + static void test_no_incremental_fetch( void ) { resolve_record_t rec; @@ -270,6 +297,7 @@ main( int argc, test_full_only_peer(); test_full_and_incremental_peer(); test_full_absent_peer_rejected(); + test_incremental_on_older_full(); test_no_incremental_fetch(); FD_LOG_NOTICE(( "pass" )); From 0f56123d7f001189eede52bf7f171c73fa5859c1 Mon Sep 17 00:00:00 2001 From: Manik Jain Date: Wed, 28 Oct 2026 20:27:41 +0000 Subject: [PATCH 2/7] snapct: request the incremental snapshot by file name --- src/discof/restore/fd_snapct_tile.c | 55 ++++++-- src/discof/restore/test_snapct_tile.c | 175 +++++++++++++++++++++++++- 2 files changed, 214 insertions(+), 16 deletions(-) diff --git a/src/discof/restore/fd_snapct_tile.c b/src/discof/restore/fd_snapct_tile.c index 380f590365d..75963e9d8e9 100644 --- a/src/discof/restore/fd_snapct_tile.c +++ b/src/discof/restore/fd_snapct_tile.c @@ -207,10 +207,11 @@ struct fd_snapct_tile { uchar in_kind[ FD_TOPO_MAX_TILE_IN_LINKS ]; struct { - ulong full_slot; - ulong slot; - int pending; - int committed; + ulong full_slot; + ulong slot; + int pending; + int committed; + fd_sspeer_t peer; /* newest peer seen with an incremental on full_slot */ } predicted_incremental; struct { @@ -408,6 +409,7 @@ predict_incremental( fd_snapct_tile_t * ctx ) { fd_sspeer_t best = fd_sspeer_selector_best( ctx->selector, 1, ctx->predicted_incremental.full_slot, wfs_floor( ctx ) ); if( FD_LIKELY( best.addr.l ) ) { + ctx->predicted_incremental.peer = best; if( FD_UNLIKELY( ctx->predicted_incremental.slot!=best.incr_slot ) ) { ctx->predicted_incremental.slot = best.incr_slot; ctx->predicted_incremental.pending = 1; @@ -415,6 +417,22 @@ predict_incremental( fd_snapct_tile_t * ctx ) { } } +/* best_incremental_peer picks the peer to download the incremental + from. A server that made a new full while our full loaded no + longer offers an incremental on ours, but the newest one we saw is + still on its disk. */ + +static fd_sspeer_t +best_incremental_peer( fd_snapct_tile_t * ctx ) { + fd_sspeer_t best = fd_sspeer_selector_best( ctx->selector, 1, ctx->predicted_incremental.full_slot, wfs_floor( ctx ) ); + if( FD_UNLIKELY( !best.addr.l && + ctx->predicted_incremental.peer.addr.l && + ctx->predicted_incremental.peer.full_slot==ctx->predicted_incremental.full_slot ) ) { + best = ctx->predicted_incremental.peer; + } + return best; +} + static void on_resolve( void * _ctx, fd_sspeer_key_t const * key, @@ -701,7 +719,7 @@ init_load( fd_snapct_tile_t * ctx, else fd_memcpy( out->snapshot_hash, ctx->peer.incr_hash, FD_HASH_FOOTPRINT ); } - out->is_redirect = !file; /* always use redirect for HTTP downloads */ + out->is_redirect = !file && full; if( file ) out->file_sz = full ? ctx->local_in.full_snapshot_size : ctx->local_in.incremental_snapshot_size; else out->file_sz = 0UL; @@ -713,8 +731,7 @@ init_load( fd_snapct_tile_t * ctx, ctx->load_complete = 0; if( !file ) { - FD_TEST( fd_cstr_printf_check( full ? ctx->http_full_snapshot_name : ctx->http_incr_snapshot_name, - PATH_MAX, NULL, full ? "snapshot.tar.bz2" : "incremental-snapshot.tar.bz2" ) ); + if( full ) FD_TEST( fd_cstr_printf_check( ctx->http_full_snapshot_name, PATH_MAX, NULL, "snapshot.tar.bz2" ) ); snapshot_output_prepare( ctx, full ); } @@ -788,6 +805,9 @@ blacklist_peer( fd_snapct_tile_t * ctx ) { fd_ssping_invalidate( ctx->ssping, ctx->peer.addr, fd_clock_tile_now( ctx->clock ) ); fd_sspeer_selector_remove_by_addr( ctx->selector, ctx->peer.addr ); fd_sspeer_selector_process_cluster_slot( ctx->selector ); + + if( ctx->predicted_incremental.peer.addr.l==ctx->peer.addr.l ) ctx->predicted_incremental.peer.addr.l = 0UL; + if( FD_UNLIKELY( blacklist_map_ele_query( ctx->blacklist_map, &ctx->peer.key, NULL, ctx->blacklist_pool ) ) ) return; if( FD_LIKELY( blacklist_pool_free( ctx->blacklist_pool ) ) ) { fd_sspeer_blacklist_entry_t * bl = blacklist_pool_ele_acquire( ctx->blacklist_pool ); @@ -948,7 +968,11 @@ after_credit( fd_snapct_tile_t * ctx, if( full ) { FD_TEST( fd_cstr_printf_check( out->path, PATH_MAX, &out->path_len, "/snapshot.tar.bz2" ) ); } else { - FD_TEST( fd_cstr_printf_check( out->path, PATH_MAX, &out->path_len, "/incremental-snapshot.tar.bz2" ) ); + char hash_b58[ FD_BASE58_ENCODED_32_SZ ]; + fd_base58_encode_32( ctx->peer.incr_hash, NULL, hash_b58 ); + FD_TEST( fd_cstr_printf_check( ctx->http_incr_snapshot_name, PATH_MAX, NULL, "incremental-snapshot-%lu-%lu-%s.tar.zst", + ctx->predicted_incremental.full_slot, ctx->predicted_incremental.slot, hash_b58 ) ); + FD_TEST( fd_cstr_printf_check( out->path, PATH_MAX, &out->path_len, "/%s", ctx->http_incr_snapshot_name ) ); } /* Peers outside the config list use HTTP without a hostname. */ @@ -1236,7 +1260,7 @@ after_credit( fd_snapct_tile_t * ctx, case FD_SNAPCT_STATE_COLLECTING_PEERS_INCREMENTAL: { if( FD_UNLIKELY( nowdeadline_nanos ) ) break; - fd_sspeer_t best = fd_sspeer_selector_best( ctx->selector, 1, ctx->predicted_incremental.full_slot, wfs_floor( ctx ) ); + fd_sspeer_t best = best_incremental_peer( ctx ); if( FD_UNLIKELY( !best.addr.l ) ) { if( !ctx->download_enabled ) { if( FD_UNLIKELY( wfs_short_of_target( ctx, ctx->predicted_incremental.full_slot ) ) ) { @@ -1471,7 +1495,7 @@ after_credit( fd_snapct_tile_t * ctx, } /* Get the best incremental peer to download from */ - fd_sspeer_t best = fd_sspeer_selector_best( ctx->selector, 1, ctx->predicted_incremental.full_slot, wfs_floor( ctx ) ); + fd_sspeer_t best = best_incremental_peer( ctx ); if( FD_UNLIKELY( !best.addr.l ) ) { ctx->deadline_nanos = now; set_state( ctx, FD_SNAPCT_STATE_COLLECTING_PEERS_INCREMENTAL ); @@ -1933,7 +1957,15 @@ snapld_frag( fd_snapct_tile_t * ctx, } } else { if( full ) fd_cstr_ncpy( ctx->http_full_snapshot_name, meta->resolved_name, PATH_MAX ); - else fd_cstr_ncpy( ctx->http_incr_snapshot_name, meta->resolved_name, PATH_MAX ); + + /* The server sent a newer full than the one we picked. The + incremental has to be built on the full we actually load. */ + if( FD_UNLIKELY( full && meta->resolved_slot!=ULONG_MAX && meta->resolved_slot!=ctx->predicted_incremental.full_slot ) ) { + FD_LOG_NOTICE(( "full snapshot resolved to slot %lu instead of %lu, pairing the incremental snapshot with it", + meta->resolved_slot, ctx->predicted_incremental.full_slot )); + ctx->predicted_incremental.full_slot = meta->resolved_slot; + predict_incremental( ctx ); + } if( FD_LIKELY( !!ctx->out_gui.mem ) ) { char snapshot_path[ PATH_MAX+30UL ]; /* 30 is fd_cstr_nlen( "https://255.255.255.255:65536/", ULONG_MAX ) */ @@ -2495,6 +2527,7 @@ unprivileged_init( fd_topo_t const * topo, ctx->predicted_incremental.slot = FD_SSPEER_SLOT_UNKNOWN; ctx->predicted_incremental.pending = 0; ctx->predicted_incremental.committed = 0; + fd_memset( &ctx->predicted_incremental.peer, 0, sizeof(fd_sspeer_t) ); fd_memset( &ctx->metrics, 0, sizeof(ctx->metrics) ); diff --git a/src/discof/restore/test_snapct_tile.c b/src/discof/restore/test_snapct_tile.c index 4aa1bd83eed..a5a147e7b09 100644 --- a/src/discof/restore/test_snapct_tile.c +++ b/src/discof/restore/test_snapct_tile.c @@ -438,6 +438,36 @@ test_blacklist_peer_basic( fd_ssping_t * ssping ) { free( scratch ); } +/* A banned peer is not kept as the incremental fallback. Banning a + different peer leaves it alone. */ + +static void +test_blacklist_forgets_remembered_peer( fd_ssping_t * ssping ) { + void * scratch = aligned_alloc( scratch_align(), scratch_footprint( NULL ) ); FD_TEST( scratch ); + + fd_snapct_tile_t * ctx; + setup_blacklist_snapct( scratch, ssping, TOTAL_PEERS_MAX, &ctx ); + + fd_ip4_port_t remembered = test_addr( 0x01020304, 8899 ); + fd_ip4_port_t other = test_addr( 0x01020305, 8899 ); + ctx->predicted_incremental.peer.addr = remembered; + + ctx->peer.addr = other; + ctx->peer.key = test_key( 0xAB ); + blacklist_peer( ctx ); + FD_TEST( ctx->predicted_incremental.peer.addr.l==remembered.l ); + + ctx->peer.addr = remembered; + ctx->peer.key = test_key( 0xAC ); + blacklist_peer( ctx ); + FD_TEST( !ctx->predicted_incremental.peer.addr.l ); + + fd_ssping_remove( ctx->ssping, other ); + fd_ssping_remove( ctx->ssping, remembered ); + + free( scratch ); +} + static void test_blacklist_peer_dedup( fd_ssping_t * ssping ) { void * scratch = aligned_alloc( scratch_align(), scratch_footprint( NULL ) ); FD_TEST( scratch ); @@ -764,10 +794,13 @@ test_start_after_init_acks( void ) { /* Committing an incremental source also queues an expected slot, drained by the first after_credit. */ ulong pre_start_cnt = full ? 1UL : 2UL; - ctx->peer.addr = test_addr( 0x7f000001U, 8899 ); - ctx->resolved_servers_cnt = 1UL; - ctx->resolved_servers[ 0 ].addr = ctx->peer.addr; - ctx->resolved_servers[ 0 ].is_https = 1; + ctx->peer.addr = test_addr( 0x7f000001U, 8899 ); + ctx->peer.incr_hash[ 0 ] = 1; + ctx->predicted_incremental.full_slot = 100UL; + ctx->predicted_incremental.slot = 200UL; + ctx->resolved_servers_cnt = 1UL; + ctx->resolved_servers[ 0 ].addr = ctx->peer.addr; + ctx->resolved_servers[ 0 ].is_https = 1; fd_cstr_ncpy( ctx->resolved_servers[ 0 ].hostname, "snapshots.example", FD_FQDN_BUF_MAX ); ulong sig = full ? FD_SNAPSHOT_MSG_CTRL_INIT_FULL : FD_SNAPSHOT_MSG_CTRL_INIT_INCR; int busy = 0; @@ -786,7 +819,18 @@ test_start_after_init_acks( void ) { if( !file ) { FD_TEST( test_start.addr.l==ctx->peer.addr.l && test_start.is_https ); FD_TEST( !strcmp( test_start.hostname, "snapshots.example" ) ); - FD_TEST( !strcmp( test_start.path, full ? "/snapshot.tar.bz2" : "/incremental-snapshot.tar.bz2" ) ); + if( full ) { + FD_TEST( !strcmp( test_start.path, "/snapshot.tar.bz2" ) ); + } else { + /* The incremental is asked for by name, which is also the + name the downloaded file gets. */ + char hash_b58[ FD_BASE58_ENCODED_32_SZ ]; + fd_base58_encode_32( ctx->peer.incr_hash, NULL, hash_b58 ); + char expected[ PATH_MAX ]; + FD_TEST( fd_cstr_printf_check( expected, PATH_MAX, NULL, "/incremental-snapshot-100-200-%s.tar.zst", hash_b58 ) ); + FD_TEST( !strcmp( test_start.path, expected ) ); + FD_TEST( !strcmp( ctx->http_incr_snapshot_name, expected+1 ) ); + } FD_TEST( test_start.path_len==strlen( test_start.path ) ); } after_credit( ctx, NULL, NULL, &busy ); @@ -799,6 +843,124 @@ test_start_after_init_acks( void ) { free( scratch ); } +/* Once no peer offers an incremental on our full any more, the newest + one seen on it is still used. One remembered on a different full + is not. */ + +static void +test_best_incremental_peer( void ) { + static struct { + char const * name; + int live; /* a peer offers (100, 200) now */ + ulong remembered_full; /* full slot of the remembered peer */ + ulong expect_incr; /* incremental slot picked, or none */ + } const cases[] = { + { "live peer wins", 1, 100UL, 200UL }, + { "remembered on our full", 0, 100UL, 150UL }, + { "remembered on another full", 0, 50UL, ULONG_MAX }, + { "nothing remembered", 0, ULONG_MAX, ULONG_MAX }, + }; + + fd_snapct_tile_t ctx[1]; + void * sel = aligned_alloc( fd_sspeer_selector_align(), fd_sspeer_selector_footprint( TOTAL_PEERS_MAX ) ); FD_TEST( sel ); + fd_ip4_port_t remembered_addr = test_addr( 0x0A000001U, 8899 ); + + for( ulong i=0UL; iselector = fd_sspeer_selector_join( fd_sspeer_selector_new( sel, TOTAL_PEERS_MAX, TEST_SELECTOR_SEED ) ); + FD_TEST( ctx->selector ); + + if( cases[ i ].live ) { + fd_sspeer_key_t key = test_key( 0xC0 ); + fd_ip4_port_t addr = test_addr( 0x0C0D0E0FU, 9900 ); + fd_sspeer_selector_add( ctx->selector, &key, addr, 5000UL, 100UL, 200UL, NULL, NULL ); + fd_sspeer_selector_process_cluster_slot( ctx->selector ); + } + + ctx->predicted_incremental.full_slot = 100UL; + if( cases[ i ].remembered_full!=ULONG_MAX ) { + ctx->predicted_incremental.peer.addr = remembered_addr; + ctx->predicted_incremental.peer.full_slot = cases[ i ].remembered_full; + ctx->predicted_incremental.peer.incr_slot = 150UL; + } + + fd_sspeer_t best = best_incremental_peer( ctx ); + ulong incr = best.addr.l ? best.incr_slot : ULONG_MAX; + if( FD_UNLIKELY( incr!=cases[ i ].expect_incr ) ) { + FD_LOG_ERR(( "case `%s`: picked incremental slot %lu, expected %lu", cases[ i ].name, incr, cases[ i ].expect_incr )); + } + if( incr==150UL ) FD_TEST( best.addr.l==remembered_addr.l ); + } + + free( sel ); + FD_LOG_NOTICE(( "pass: test_best_incremental_peer" )); +} + +/* A full that resolves to a newer slot than the one picked moves the + incremental onto that full. An incremental keeps the name it was + requested by. */ + +static void +test_meta_repins_full( void ) { + fd_snapct_tile_t ctx[1]; + void * sel = aligned_alloc( fd_sspeer_selector_align(), fd_sspeer_selector_footprint( TOTAL_PEERS_MAX ) ); FD_TEST( sel ); + static uchar meta_mem[ sizeof(fd_ssctrl_meta_t) ] __attribute__((aligned(FD_CHUNK_ALIGN))); + fd_ssctrl_meta_t * meta = (fd_ssctrl_meta_t *)meta_mem; + + fd_memset( ctx, 0, sizeof(*ctx) ); + test_ctx_wake_init( ctx ); + ctx->selector = fd_sspeer_selector_join( fd_sspeer_selector_new( sel, TOTAL_PEERS_MAX, TEST_SELECTOR_SEED ) ); + FD_TEST( ctx->selector ); + ctx->snapld_in_mem = meta_mem; + + /* The server has moved on: it offers (300, 400) now. */ + fd_sspeer_key_t key = test_key( 0xD0 ); + fd_ip4_port_t addr = test_addr( 0x0D0E0F10U, 9900 ); + fd_sspeer_selector_add( ctx->selector, &key, addr, 5000UL, 300UL, 400UL, NULL, NULL ); + fd_sspeer_selector_process_cluster_slot( ctx->selector ); + + ctx->state = FD_SNAPCT_STATE_READING_FULL_HTTP; + ctx->load_incremental = 1; + ctx->predicted_incremental.full_slot = 100UL; + ctx->predicted_incremental.slot = 200UL; + ctx->predicted_incremental.peer.addr = test_addr( 0x0A000001U, 8899 ); + ctx->predicted_incremental.peer.full_slot = 100UL; + ctx->predicted_incremental.peer.incr_slot = 200UL; + + /* The full we picked: nothing changes. */ + fd_memset( meta, 0, sizeof(*meta) ); + meta->total_sz = 1UL; + meta->resolved_slot = 100UL; + fd_cstr_ncpy( meta->resolved_name, "snapshot-100-a.tar.zst", PATH_MAX ); + snapld_frag( ctx, FD_SNAPSHOT_MSG_META, sizeof(*meta), 0UL, NULL ); + FD_TEST( ctx->predicted_incremental.full_slot==100UL ); + FD_TEST( ctx->predicted_incremental.slot==200UL ); + FD_TEST( ctx->predicted_incremental.peer.full_slot==100UL ); + FD_TEST( !strcmp( ctx->http_full_snapshot_name, "snapshot-100-a.tar.zst" ) ); + + /* A newer full: the incremental follows it. */ + meta->resolved_slot = 300UL; + fd_cstr_ncpy( meta->resolved_name, "snapshot-300-b.tar.zst", PATH_MAX ); + snapld_frag( ctx, FD_SNAPSHOT_MSG_META, sizeof(*meta), 0UL, NULL ); + FD_TEST( ctx->predicted_incremental.full_slot==300UL ); + FD_TEST( ctx->predicted_incremental.slot==400UL ); + FD_TEST( ctx->predicted_incremental.pending==1 ); + FD_TEST( ctx->predicted_incremental.peer.full_slot==300UL && ctx->predicted_incremental.peer.incr_slot==400UL ); + FD_TEST( !strcmp( ctx->http_full_snapshot_name, "snapshot-300-b.tar.zst" ) ); + + /* The incremental was asked for by name; META does not rename it. */ + ctx->state = FD_SNAPCT_STATE_READING_INCREMENTAL_HTTP; + fd_cstr_ncpy( ctx->http_incr_snapshot_name, "incremental-snapshot-300-400-c.tar.zst", PATH_MAX ); + meta->resolved_slot = ULONG_MAX; + fd_cstr_ncpy( meta->resolved_name, "incremental-snapshot-300-500-d.tar.zst", PATH_MAX ); + snapld_frag( ctx, FD_SNAPSHOT_MSG_META, sizeof(*meta), 0UL, NULL ); + FD_TEST( !strcmp( ctx->http_incr_snapshot_name, "incremental-snapshot-300-400-c.tar.zst" ) ); + + free( sel ); + FD_LOG_NOTICE(( "pass: test_meta_repins_full" )); +} + static void test_incr_use_local( void ) { static struct { @@ -1435,6 +1597,8 @@ main( int argc, test_contact_info_slot_reuse_after_unallowed_peer_expires(); test_load_complete_signal(); test_start_after_init_acks(); + test_best_incremental_peer(); + test_meta_repins_full(); test_snapct_needs_incr(); test_incr_use_local(); test_wfs_short_of_target(); @@ -1454,6 +1618,7 @@ main( int argc, test_rpc_flap_to_invalidated_addr_evicts_selector( ssping ); test_blacklist_peer_basic( ssping ); + test_blacklist_forgets_remembered_peer( ssping ); test_blacklist_peer_dedup( ssping ); test_blacklist_peer_cluster_slot_regression( ssping ); test_blacklist_peer_readd_blocked( ssping ); From 24517651bf88d01dd028cbc37d27d87c011bf01a Mon Sep 17 00:00:00 2001 From: Manik Jain Date: Thu, 29 Oct 2026 01:28:42 +0000 Subject: [PATCH 3/7] snapct: harden the remembered incremental peer --- src/discof/restore/fd_snapct_tile.c | 11 +++++++-- src/discof/restore/test_snapct_tile.c | 32 ++++++++++++++++++++++++++- 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/src/discof/restore/fd_snapct_tile.c b/src/discof/restore/fd_snapct_tile.c index 75963e9d8e9..26f5ea6e072 100644 --- a/src/discof/restore/fd_snapct_tile.c +++ b/src/discof/restore/fd_snapct_tile.c @@ -732,6 +732,7 @@ init_load( fd_snapct_tile_t * ctx, if( !file ) { if( full ) FD_TEST( fd_cstr_printf_check( ctx->http_full_snapshot_name, PATH_MAX, NULL, "snapshot.tar.bz2" ) ); + else fd_cstr_fini( ctx->http_incr_snapshot_name ); snapshot_output_prepare( ctx, full ); } @@ -806,7 +807,10 @@ blacklist_peer( fd_snapct_tile_t * ctx ) { fd_sspeer_selector_remove_by_addr( ctx->selector, ctx->peer.addr ); fd_sspeer_selector_process_cluster_slot( ctx->selector ); - if( ctx->predicted_incremental.peer.addr.l==ctx->peer.addr.l ) ctx->predicted_incremental.peer.addr.l = 0UL; + if( ctx->predicted_incremental.peer.addr.l==ctx->peer.addr.l || + fd_sspeer_key_eq( &ctx->predicted_incremental.peer.key, &ctx->peer.key ) ) { + ctx->predicted_incremental.peer.addr.l = 0UL; + } if( FD_UNLIKELY( blacklist_map_ele_query( ctx->blacklist_map, &ctx->peer.key, NULL, ctx->blacklist_pool ) ) ) return; if( FD_LIKELY( blacklist_pool_free( ctx->blacklist_pool ) ) ) { @@ -1212,6 +1216,7 @@ after_credit( fd_snapct_tile_t * ctx, FD_LOG_INFO(( "reading full snapshot at slot %lu (cluster full %lu incremental %lu) from local file `%s`", ctx->local_in.full_snapshot_slot, cluster.full, cluster.incremental, ctx->local_in.full_snapshot_path )); ctx->predicted_incremental.full_slot = ctx->local_in.full_snapshot_slot; + predict_incremental( ctx ); set_state( ctx, FD_SNAPCT_STATE_READING_FULL_FILE ); init_load( ctx, stem, 1, 1 ); } else { @@ -1251,6 +1256,7 @@ after_credit( fd_snapct_tile_t * ctx, ctx->peer = best; set_state( ctx, FD_SNAPCT_STATE_READING_FULL_HTTP ); ctx->predicted_incremental.full_slot = best.full_slot; + predict_incremental( ctx ); init_load( ctx, stem, 1, 0 ); } break; @@ -1426,7 +1432,7 @@ after_credit( fd_snapct_tile_t * ctx, ulong fetch_incr = ULONG_MAX; if( FD_LIKELY( ctx->download_enabled ) ) { - fd_sspeer_t best_incr = fd_sspeer_selector_best( ctx->selector, 1, ctx->predicted_incremental.full_slot, wfs_floor( ctx ) ); + fd_sspeer_t best_incr = best_incremental_peer( ctx ); if( FD_LIKELY( best_incr.addr.l ) ) fetch_incr = best_incr.incr_slot; } @@ -1964,6 +1970,7 @@ snapld_frag( fd_snapct_tile_t * ctx, FD_LOG_NOTICE(( "full snapshot resolved to slot %lu instead of %lu, pairing the incremental snapshot with it", meta->resolved_slot, ctx->predicted_incremental.full_slot )); ctx->predicted_incremental.full_slot = meta->resolved_slot; + ctx->predicted_incremental.slot = FD_SSPEER_SLOT_UNKNOWN; predict_incremental( ctx ); } diff --git a/src/discof/restore/test_snapct_tile.c b/src/discof/restore/test_snapct_tile.c index a5a147e7b09..b201bd383f4 100644 --- a/src/discof/restore/test_snapct_tile.c +++ b/src/discof/restore/test_snapct_tile.c @@ -438,7 +438,8 @@ test_blacklist_peer_basic( fd_ssping_t * ssping ) { free( scratch ); } -/* A banned peer is not kept as the incremental fallback. Banning a +/* A banned peer is not kept as the incremental fallback, whether it is + matched by address or, after it moved, by identity. Banning a different peer leaves it alone. */ static void @@ -462,8 +463,18 @@ test_blacklist_forgets_remembered_peer( fd_ssping_t * ssping ) { blacklist_peer( ctx ); FD_TEST( !ctx->predicted_incremental.peer.addr.l ); + /* Same identity, new address: still forgotten. */ + fd_ip4_port_t moved = test_addr( 0x01020306, 8899 ); + ctx->predicted_incremental.peer.addr = remembered; + ctx->predicted_incremental.peer.key = test_key( 0xAD ); + ctx->peer.addr = moved; + ctx->peer.key = test_key( 0xAD ); + blacklist_peer( ctx ); + FD_TEST( !ctx->predicted_incremental.peer.addr.l ); + fd_ssping_remove( ctx->ssping, other ); fd_ssping_remove( ctx->ssping, remembered ); + fd_ssping_remove( ctx->ssping, moved ); free( scratch ); } @@ -949,6 +960,17 @@ test_meta_repins_full( void ) { FD_TEST( ctx->predicted_incremental.peer.full_slot==300UL && ctx->predicted_incremental.peer.incr_slot==400UL ); FD_TEST( !strcmp( ctx->http_full_snapshot_name, "snapshot-300-b.tar.zst" ) ); + /* A full nobody offers an incremental on yet: no stale prediction + is kept, and the remembered peer no longer qualifies. */ + ctx->predicted_incremental.pending = 0; + meta->resolved_slot = 500UL; + fd_cstr_ncpy( meta->resolved_name, "snapshot-500-e.tar.zst", PATH_MAX ); + snapld_frag( ctx, FD_SNAPSHOT_MSG_META, sizeof(*meta), 0UL, NULL ); + FD_TEST( ctx->predicted_incremental.full_slot==500UL ); + FD_TEST( ctx->predicted_incremental.slot==FD_SSPEER_SLOT_UNKNOWN ); + FD_TEST( ctx->predicted_incremental.pending==0 ); + FD_TEST( !best_incremental_peer( ctx ).addr.l ); + /* The incremental was asked for by name; META does not rename it. */ ctx->state = FD_SNAPCT_STATE_READING_INCREMENTAL_HTTP; fd_cstr_ncpy( ctx->http_incr_snapshot_name, "incremental-snapshot-300-400-c.tar.zst", PATH_MAX ); @@ -1450,6 +1472,14 @@ test_collecting_peers_wfs_selection( void ) { FD_LOG_ERR(( "case `%s`: predicted incremental slot %lu, expected %lu", cases[ i ].name, ctx->predicted_incremental.slot, cases[ i ].expected_predicted_slot )); } + /* A predicted incremental comes with the peer that offers it, + remembered for the pinned full. */ + int picked = cases[ i ].expected_predicted_slot!=FD_SSPEER_SLOT_UNKNOWN; + FD_TEST( !!ctx->predicted_incremental.peer.addr.l==picked ); + if( picked ) { + FD_TEST( ctx->predicted_incremental.peer.full_slot==ctx->predicted_incremental.full_slot ); + FD_TEST( ctx->predicted_incremental.peer.incr_slot==cases[ i ].expected_predicted_slot ); + } } free( sel ); From 4538ef5222f540d06cc540597eb815c753d64883 Mon Sep 17 00:00:00 2001 From: Manik Jain Date: Tue, 3 Nov 2026 20:18:39 +0000 Subject: [PATCH 4/7] snapct: skip a ping-banned server in the incremental fallback --- src/discof/restore/fd_snapct_tile.c | 3 ++- src/discof/restore/test_snapct_tile.c | 19 +++++++++++++++++-- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/src/discof/restore/fd_snapct_tile.c b/src/discof/restore/fd_snapct_tile.c index 26f5ea6e072..4fc361f1434 100644 --- a/src/discof/restore/fd_snapct_tile.c +++ b/src/discof/restore/fd_snapct_tile.c @@ -427,7 +427,8 @@ best_incremental_peer( fd_snapct_tile_t * ctx ) { fd_sspeer_t best = fd_sspeer_selector_best( ctx->selector, 1, ctx->predicted_incremental.full_slot, wfs_floor( ctx ) ); if( FD_UNLIKELY( !best.addr.l && ctx->predicted_incremental.peer.addr.l && - ctx->predicted_incremental.peer.full_slot==ctx->predicted_incremental.full_slot ) ) { + ctx->predicted_incremental.peer.full_slot==ctx->predicted_incremental.full_slot && + !fd_ssping_is_invalidated( ctx->ssping, ctx->predicted_incremental.peer.addr ) ) ) { best = ctx->predicted_incremental.peer; } return best; diff --git a/src/discof/restore/test_snapct_tile.c b/src/discof/restore/test_snapct_tile.c index b201bd383f4..de5ec8cc327 100644 --- a/src/discof/restore/test_snapct_tile.c +++ b/src/discof/restore/test_snapct_tile.c @@ -859,7 +859,7 @@ test_start_after_init_acks( void ) { is not. */ static void -test_best_incremental_peer( void ) { +test_best_incremental_peer( fd_ssping_t * ssping ) { static struct { char const * name; int live; /* a peer offers (100, 200) now */ @@ -879,6 +879,7 @@ test_best_incremental_peer( void ) { for( ulong i=0UL; issping = ssping; ctx->selector = fd_sspeer_selector_join( fd_sspeer_selector_new( sel, TOTAL_PEERS_MAX, TEST_SELECTOR_SEED ) ); FD_TEST( ctx->selector ); @@ -904,6 +905,20 @@ test_best_incremental_peer( void ) { if( incr==150UL ) FD_TEST( best.addr.l==remembered_addr.l ); } + /* A remembered server whose ping timed out is not asked either. */ + fd_memset( ctx, 0, sizeof(*ctx) ); + test_ctx_wake_init( ctx ); + ctx->ssping = ssping; + ctx->selector = fd_sspeer_selector_join( fd_sspeer_selector_new( sel, TOTAL_PEERS_MAX, TEST_SELECTOR_SEED ) ); + ctx->predicted_incremental.full_slot = 100UL; + ctx->predicted_incremental.peer.addr = remembered_addr; + ctx->predicted_incremental.peer.full_slot = 100UL; + ctx->predicted_incremental.peer.incr_slot = 150UL; + fd_ssping_add( ssping, remembered_addr ); + fd_ssping_invalidate( ssping, remembered_addr, fd_clock_tile_now( ctx->clock ) ); + FD_TEST( !best_incremental_peer( ctx ).addr.l ); + fd_ssping_remove( ssping, remembered_addr ); + free( sel ); FD_LOG_NOTICE(( "pass: test_best_incremental_peer" )); } @@ -1627,7 +1642,6 @@ main( int argc, test_contact_info_slot_reuse_after_unallowed_peer_expires(); test_load_complete_signal(); test_start_after_init_acks(); - test_best_incremental_peer(); test_meta_repins_full(); test_snapct_needs_incr(); test_incr_use_local(); @@ -1649,6 +1663,7 @@ main( int argc, test_blacklist_peer_basic( ssping ); test_blacklist_forgets_remembered_peer( ssping ); + test_best_incremental_peer( ssping ); test_blacklist_peer_dedup( ssping ); test_blacklist_peer_cluster_slot_regression( ssping ); test_blacklist_peer_readd_blocked( ssping ); From 7f44f35a427cca1261a4d4dfde7ddd4fa0455a96 Mon Sep 17 00:00:00 2001 From: Manik Jain Date: Tue, 3 Nov 2026 20:26:54 +0000 Subject: [PATCH 5/7] snapct: use the incremental fallback while waiting for peers --- src/discof/restore/fd_snapct_tile.c | 2 +- src/discof/restore/test_snapct_tile.c | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/src/discof/restore/fd_snapct_tile.c b/src/discof/restore/fd_snapct_tile.c index 4fc361f1434..b268ff4eb4b 100644 --- a/src/discof/restore/fd_snapct_tile.c +++ b/src/discof/restore/fd_snapct_tile.c @@ -1075,7 +1075,7 @@ after_credit( fd_snapct_tile_t * ctx, } FD_TEST( ctx->predicted_incremental.full_slot!=FD_SSPEER_SLOT_UNKNOWN ); - fd_sspeer_t best = fd_sspeer_selector_best( ctx->selector, 1, ctx->predicted_incremental.full_slot, wfs_floor( ctx ) ); + fd_sspeer_t best = best_incremental_peer( ctx ); if( FD_LIKELY( best.addr.l ) ) { set_state( ctx, FD_SNAPCT_STATE_COLLECTING_PEERS_INCREMENTAL ); ctx->deadline_nanos = now; diff --git a/src/discof/restore/test_snapct_tile.c b/src/discof/restore/test_snapct_tile.c index de5ec8cc327..9952150b699 100644 --- a/src/discof/restore/test_snapct_tile.c +++ b/src/discof/restore/test_snapct_tile.c @@ -905,6 +905,24 @@ test_best_incremental_peer( fd_ssping_t * ssping ) { if( incr==150UL ) FD_TEST( best.addr.l==remembered_addr.l ); } + /* While waiting for peers, a remembered server on our full ends the + wait once it is usable again. */ + fd_memset( ctx, 0, sizeof(*ctx) ); + test_ctx_wake_init( ctx ); + ctx->ssping = ssping; + ctx->selector = fd_sspeer_selector_join( fd_sspeer_selector_new( sel, TOTAL_PEERS_MAX, TEST_SELECTOR_SEED ) ); + ctx->state = FD_SNAPCT_STATE_WAITING_FOR_PEERS_INCREMENTAL; + ctx->deadline_nanos = LONG_MAX; + ctx->download_enabled = 1; + ctx->load_incremental = 1; + ctx->predicted_incremental.full_slot = 100UL; + ctx->predicted_incremental.peer.addr = remembered_addr; + ctx->predicted_incremental.peer.full_slot = 100UL; + ctx->predicted_incremental.peer.incr_slot = 150UL; + int busy = 0; + after_credit( ctx, NULL, NULL, &busy ); + FD_TEST( ctx->state==FD_SNAPCT_STATE_COLLECTING_PEERS_INCREMENTAL ); + /* A remembered server whose ping timed out is not asked either. */ fd_memset( ctx, 0, sizeof(*ctx) ); test_ctx_wake_init( ctx ); From 2811ed11aaabf40f92f3fe9f5258d6ff2e1f2828 Mon Sep 17 00:00:00 2001 From: Manik Jain Date: Tue, 3 Nov 2026 20:41:57 +0000 Subject: [PATCH 6/7] snapct: tidy the re-pin and local full prediction --- src/discof/restore/fd_snapct_tile.c | 25 ++++++++++++------------- src/discof/restore/test_snapct_tile.c | 5 ++++- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/src/discof/restore/fd_snapct_tile.c b/src/discof/restore/fd_snapct_tile.c index b268ff4eb4b..307a1ae5163 100644 --- a/src/discof/restore/fd_snapct_tile.c +++ b/src/discof/restore/fd_snapct_tile.c @@ -211,7 +211,7 @@ struct fd_snapct_tile { ulong slot; int pending; int committed; - fd_sspeer_t peer; /* newest peer seen with an incremental on full_slot */ + fd_sspeer_t peer; /* peer last picked for an incremental on full_slot */ } predicted_incremental; struct { @@ -731,18 +731,14 @@ init_load( fd_snapct_tile_t * ctx, ctx->start_sent = 0; ctx->load_complete = 0; - if( !file ) { - if( full ) FD_TEST( fd_cstr_printf_check( ctx->http_full_snapshot_name, PATH_MAX, NULL, "snapshot.tar.bz2" ) ); - else fd_cstr_fini( ctx->http_incr_snapshot_name ); - snapshot_output_prepare( ctx, full ); - } + /* The full learns its name from the redirect, the incremental gets + its name at START, and a local file has none. Clear stale names + before the rename functions run. */ + if( !full ) fd_cstr_fini( ctx->http_incr_snapshot_name ); + else if( file ) fd_cstr_fini( ctx->http_full_snapshot_name ); + else FD_TEST( fd_cstr_printf_check( ctx->http_full_snapshot_name, PATH_MAX, NULL, "snapshot.tar.bz2" ) ); - /* Clear stale http_*_snapshot_name for file loads before rename - functions run. GUI publish is deferred to the META handler. */ - if( file ) { - if( full ) fd_cstr_fini( ctx->http_full_snapshot_name ); - else fd_cstr_fini( ctx->http_incr_snapshot_name ); - } + if( !file ) snapshot_output_prepare( ctx, full ); if( !full ) { if( file ) ctx->predicted_incremental.slot = ctx->local_in.incremental_snapshot_slot; @@ -1217,7 +1213,7 @@ after_credit( fd_snapct_tile_t * ctx, FD_LOG_INFO(( "reading full snapshot at slot %lu (cluster full %lu incremental %lu) from local file `%s`", ctx->local_in.full_snapshot_slot, cluster.full, cluster.incremental, ctx->local_in.full_snapshot_path )); ctx->predicted_incremental.full_slot = ctx->local_in.full_snapshot_slot; - predict_incremental( ctx ); + if( !local_incr_on_disk ) predict_incremental( ctx ); set_state( ctx, FD_SNAPCT_STATE_READING_FULL_FILE ); init_load( ctx, stem, 1, 1 ); } else { @@ -1973,6 +1969,9 @@ snapld_frag( fd_snapct_tile_t * ctx, ctx->predicted_incremental.full_slot = meta->resolved_slot; ctx->predicted_incremental.slot = FD_SSPEER_SLOT_UNKNOWN; predict_incremental( ctx ); + /* Until a peer offers an incremental on it, the full itself is + the slot to expect. */ + if( ctx->predicted_incremental.slot==FD_SSPEER_SLOT_UNKNOWN ) send_expected_slot( ctx, stem, meta->resolved_slot ); } if( FD_LIKELY( !!ctx->out_gui.mem ) ) { diff --git a/src/discof/restore/test_snapct_tile.c b/src/discof/restore/test_snapct_tile.c index 9952150b699..660946aaf75 100644 --- a/src/discof/restore/test_snapct_tile.c +++ b/src/discof/restore/test_snapct_tile.c @@ -994,8 +994,10 @@ test_meta_repins_full( void ) { FD_TEST( !strcmp( ctx->http_full_snapshot_name, "snapshot-300-b.tar.zst" ) ); /* A full nobody offers an incremental on yet: no stale prediction - is kept, and the remembered peer no longer qualifies. */ + is kept, the remembered peer no longer qualifies, and the full's + own slot is sent as the one to expect. */ ctx->predicted_incremental.pending = 0; + test_publish_cnt = 0UL; meta->resolved_slot = 500UL; fd_cstr_ncpy( meta->resolved_name, "snapshot-500-e.tar.zst", PATH_MAX ); snapld_frag( ctx, FD_SNAPSHOT_MSG_META, sizeof(*meta), 0UL, NULL ); @@ -1003,6 +1005,7 @@ test_meta_repins_full( void ) { FD_TEST( ctx->predicted_incremental.slot==FD_SSPEER_SLOT_UNKNOWN ); FD_TEST( ctx->predicted_incremental.pending==0 ); FD_TEST( !best_incremental_peer( ctx ).addr.l ); + FD_TEST( test_publish_cnt==1UL && test_publish_sig==FD_SSMSG_EXPECTED_SLOT ); /* The incremental was asked for by name; META does not rename it. */ ctx->state = FD_SNAPCT_STATE_READING_INCREMENTAL_HTTP; From 32f26d1deab997fdc11d2953994518105989cb54 Mon Sep 17 00:00:00 2001 From: Manik Jain Date: Tue, 3 Nov 2026 21:06:56 +0000 Subject: [PATCH 7/7] snapct: drop a comment --- src/discof/restore/fd_snapct_tile.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/discof/restore/fd_snapct_tile.c b/src/discof/restore/fd_snapct_tile.c index 307a1ae5163..b20a1c0a994 100644 --- a/src/discof/restore/fd_snapct_tile.c +++ b/src/discof/restore/fd_snapct_tile.c @@ -1969,8 +1969,6 @@ snapld_frag( fd_snapct_tile_t * ctx, ctx->predicted_incremental.full_slot = meta->resolved_slot; ctx->predicted_incremental.slot = FD_SSPEER_SLOT_UNKNOWN; predict_incremental( ctx ); - /* Until a peer offers an incremental on it, the full itself is - the slot to expect. */ if( ctx->predicted_incremental.slot==FD_SSPEER_SLOT_UNKNOWN ) send_expected_slot( ctx, stem, meta->resolved_slot ); }