From b88a3641488a76dc0a96537e75eadc85622f19d1 Mon Sep 17 00:00:00 2001 From: Taranum Wasu Date: Tue, 21 Jul 2026 12:45:37 +0530 Subject: [PATCH 01/17] sync: SVS v4 large-group sync (mhash, PARTIAL, publish+pull) This rewrites the State Vector Sync protocol to v4 in the ndnd implementation. v4 introduces a membership hash (mhash) carried on every Sync Data, two embedded State Vector encodings (FULL and PARTIAL), and a publish-only form that references a retrievable full vector at .../32=sv/. Highlights: * SvsData now carries MemberSetHash and VectorType on every Sync Data; there is no legacy StateVector-only wire form. * New publication sends embedded PARTIAL when FULL exceeds SyncVectorThreshold; entry [0] is always the sender's own entry. If the sender-only baseline itself exceeds the threshold, the sender falls back to publish+pull rather than emit a PARTIAL vector missing the required entry [0]. * Periodic sync and mhash mismatch recovery use publish+pull: produce full-vector Data at .../32=sv/, then announce-only Sync Data carrying mhash + SvsDataRef. * pullFullVector is debounced per sender (5s) to bound the pull fan-in when many peers cross the membership hash boundary simultaneously. * SyncDataName wire version is bumped from v=3 to v=4. File renames and surface API changes: * ComputeMhash -> ComputeMembershipHash (in svs_membership_hash.go). The wire-level Go TLV package path std/ndn/svs/v3/ is unchanged because it is an internal ndnd package name, not part of the wire profile. --- std/ndn/svs/v3/definitions.go | 21 +++ std/ndn/svs/v3/zz_generated.go | 261 ++++++++++++++++++++++++++++++ std/sync/svs.go | 181 ++++++++++++++++----- std/sync/svs_alo_data.go | 4 +- std/sync/svs_encode.go | 258 ++++++++++++++++++++++++++++++ std/sync/svs_map.go | 13 +- std/sync/svs_membership_hash.go | 42 +++++ std/sync/svs_pull.go | 272 ++++++++++++++++++++++++++++++++ 8 files changed, 1003 insertions(+), 49 deletions(-) create mode 100644 std/sync/svs_encode.go create mode 100644 std/sync/svs_membership_hash.go create mode 100644 std/sync/svs_pull.go diff --git a/std/ndn/svs/v3/definitions.go b/std/ndn/svs/v3/definitions.go index 3f7777bc..4412d154 100644 --- a/std/ndn/svs/v3/definitions.go +++ b/std/ndn/svs/v3/definitions.go @@ -3,9 +3,22 @@ package svs import ( enc "github.com/named-data/ndnd/std/encoding" + "github.com/named-data/ndnd/std/types/optional" +) + +// VectorType values for inline SvsData (TLV 0xCD). +const ( + VectorTypeFull uint64 = 0 + VectorTypePartial uint64 = 1 ) type SvsData struct { + //+field:binary:optional + MemberSetHash []byte `tlv:"0xcb"` + //+field:natural:optional + VectorType optional.Optional[uint64] `tlv:"0xcd"` + //+field:name + SvsDataRef enc.Name `tlv:"0x07"` //+field:struct:StateVector StateVector *StateVector `tlv:"0xc9"` } @@ -29,6 +42,14 @@ type SeqNoEntry struct { SeqNo uint64 `tlv:"0xd6"` } +// MembershipTuple is one (Name, BootstrapTime) pair used to compute MemberSetHash. +type MembershipTuple struct { + //+field:name + Name enc.Name `tlv:"0x07"` + //+field:natural + BootstrapTime uint64 `tlv:"0xd4"` +} + // +tlv-model:nocopy type PassiveState struct { //+field:sequence:[]byte:binary:[]byte diff --git a/std/ndn/svs/v3/zz_generated.go b/std/ndn/svs/v3/zz_generated.go index 0fb2e7fc..6d4590e7 100644 --- a/std/ndn/svs/v3/zz_generated.go +++ b/std/ndn/svs/v3/zz_generated.go @@ -11,6 +11,7 @@ import ( type SvsDataEncoder struct { Length uint + SvsDataRef_length uint StateVector_encoder StateVectorEncoder } @@ -19,11 +20,32 @@ type SvsDataParsingContext struct { } func (encoder *SvsDataEncoder) Init(value *SvsData) { + + if value.SvsDataRef != nil { + encoder.SvsDataRef_length = 0 + for _, c := range value.SvsDataRef { + encoder.SvsDataRef_length += uint(c.EncodingLength()) + } + } if value.StateVector != nil { encoder.StateVector_encoder.Init(value.StateVector) } l := uint(0) + if value.MemberSetHash != nil { + l += 1 + l += uint(enc.TLNum(len(value.MemberSetHash)).EncodingLength()) + l += uint(len(value.MemberSetHash)) + } + if optval, ok := value.VectorType.Get(); ok { + l += 1 + l += uint(1 + enc.Nat(optval).EncodingLength()) + } + if value.SvsDataRef != nil { + l += 1 + l += uint(enc.TLNum(encoder.SvsDataRef_length).EncodingLength()) + l += encoder.SvsDataRef_length + } if value.StateVector != nil { l += 1 l += uint(enc.TLNum(encoder.StateVector_encoder.Length).EncodingLength()) @@ -34,6 +56,7 @@ func (encoder *SvsDataEncoder) Init(value *SvsData) { } func (context *SvsDataParsingContext) Init() { + context.StateVector_context.Init() } @@ -41,6 +64,29 @@ func (encoder *SvsDataEncoder) EncodeInto(value *SvsData, buf []byte) { pos := uint(0) + if value.MemberSetHash != nil { + buf[pos] = byte(203) + pos += 1 + pos += uint(enc.TLNum(len(value.MemberSetHash)).EncodeInto(buf[pos:])) + copy(buf[pos:], value.MemberSetHash) + pos += uint(len(value.MemberSetHash)) + } + if optval, ok := value.VectorType.Get(); ok { + buf[pos] = byte(205) + pos += 1 + + buf[pos] = byte(enc.Nat(optval).EncodeInto(buf[pos+1:])) + pos += uint(1 + buf[pos]) + + } + if value.SvsDataRef != nil { + buf[pos] = byte(7) + pos += 1 + pos += uint(enc.TLNum(encoder.SvsDataRef_length).EncodeInto(buf[pos:])) + for _, c := range value.SvsDataRef { + pos += uint(c.EncodeInto(buf[pos:])) + } + } if value.StateVector != nil { buf[pos] = byte(201) pos += 1 @@ -64,6 +110,9 @@ func (encoder *SvsDataEncoder) Encode(value *SvsData) enc.Wire { func (context *SvsDataParsingContext) Parse(reader enc.WireView, ignoreCritical bool) (*SvsData, error) { + var handled_MemberSetHash bool = false + var handled_VectorType bool = false + var handled_SvsDataRef bool = false var handled_StateVector bool = false progress := -1 @@ -91,6 +140,43 @@ func (context *SvsDataParsingContext) Parse(reader enc.WireView, ignoreCritical err = nil if handled := false; true { switch typ { + case 203: + if true { + handled = true + handled_MemberSetHash = true + value.MemberSetHash = make([]byte, l) + _, err = reader.ReadFull(value.MemberSetHash) + } + case 205: + if true { + handled = true + handled_VectorType = true + { + optval := uint64(0) + optval = uint64(0) + { + for i := 0; i < int(l); i++ { + x := byte(0) + x, err = reader.ReadByte() + if err != nil { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + break + } + optval = uint64(optval<<8) | uint64(x) + } + } + value.VectorType.Set(optval) + } + } + case 7: + if true { + handled = true + handled_SvsDataRef = true + delegate := reader.Delegate(int(l)) + value.SvsDataRef, err = delegate.ReadName() + } case 201: if true { handled = true @@ -115,6 +201,15 @@ func (context *SvsDataParsingContext) Parse(reader enc.WireView, ignoreCritical startPos = reader.Pos() err = nil + if !handled_MemberSetHash && err == nil { + value.MemberSetHash = nil + } + if !handled_VectorType && err == nil { + value.VectorType.Unset() + } + if !handled_SvsDataRef && err == nil { + value.SvsDataRef = nil + } if !handled_StateVector && err == nil { value.StateVector = nil } @@ -742,6 +837,172 @@ func ParseSeqNoEntry(reader enc.WireView, ignoreCritical bool) (*SeqNoEntry, err return context.Parse(reader, ignoreCritical) } +type MembershipTupleEncoder struct { + Length uint + + Name_length uint +} + +type MembershipTupleParsingContext struct { +} + +func (encoder *MembershipTupleEncoder) Init(value *MembershipTuple) { + if value.Name != nil { + encoder.Name_length = 0 + for _, c := range value.Name { + encoder.Name_length += uint(c.EncodingLength()) + } + } + + l := uint(0) + if value.Name != nil { + l += 1 + l += uint(enc.TLNum(encoder.Name_length).EncodingLength()) + l += encoder.Name_length + } + l += 1 + l += uint(1 + enc.Nat(value.BootstrapTime).EncodingLength()) + encoder.Length = l + +} + +func (context *MembershipTupleParsingContext) Init() { + +} + +func (encoder *MembershipTupleEncoder) EncodeInto(value *MembershipTuple, buf []byte) { + + pos := uint(0) + + if value.Name != nil { + buf[pos] = byte(7) + pos += 1 + pos += uint(enc.TLNum(encoder.Name_length).EncodeInto(buf[pos:])) + for _, c := range value.Name { + pos += uint(c.EncodeInto(buf[pos:])) + } + } + buf[pos] = byte(212) + pos += 1 + + buf[pos] = byte(enc.Nat(value.BootstrapTime).EncodeInto(buf[pos+1:])) + pos += uint(1 + buf[pos]) +} + +func (encoder *MembershipTupleEncoder) Encode(value *MembershipTuple) enc.Wire { + + wire := make(enc.Wire, 1) + wire[0] = make([]byte, encoder.Length) + buf := wire[0] + encoder.EncodeInto(value, buf) + + return wire +} + +func (context *MembershipTupleParsingContext) Parse(reader enc.WireView, ignoreCritical bool) (*MembershipTuple, error) { + + var handled_Name bool = false + var handled_BootstrapTime bool = false + + progress := -1 + _ = progress + + value := &MembershipTuple{} + var err error + var startPos int + for { + startPos = reader.Pos() + if startPos >= reader.Length() { + break + } + typ := enc.TLNum(0) + l := enc.TLNum(0) + typ, err = reader.ReadTLNum() + if err != nil { + return nil, enc.ErrFailToParse{TypeNum: 0, Err: err} + } + l, err = reader.ReadTLNum() + if err != nil { + return nil, enc.ErrFailToParse{TypeNum: 0, Err: err} + } + + err = nil + if handled := false; true { + switch typ { + case 7: + if true { + handled = true + handled_Name = true + delegate := reader.Delegate(int(l)) + value.Name, err = delegate.ReadName() + } + case 212: + if true { + handled = true + handled_BootstrapTime = true + value.BootstrapTime = uint64(0) + { + for i := 0; i < int(l); i++ { + x := byte(0) + x, err = reader.ReadByte() + if err != nil { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + break + } + value.BootstrapTime = uint64(value.BootstrapTime<<8) | uint64(x) + } + } + } + default: + if !ignoreCritical && ((typ <= 31) || ((typ & 1) == 1)) { + return nil, enc.ErrUnrecognizedField{TypeNum: typ} + } + handled = true + err = reader.Skip(int(l)) + } + if err == nil && !handled { + } + if err != nil { + return nil, enc.ErrFailToParse{TypeNum: typ, Err: err} + } + } + } + + startPos = reader.Pos() + err = nil + + if !handled_Name && err == nil { + value.Name = nil + } + if !handled_BootstrapTime && err == nil { + err = enc.ErrSkipRequired{Name: "BootstrapTime", TypeNum: 212} + } + + if err != nil { + return nil, err + } + + return value, nil +} + +func (value *MembershipTuple) Encode() enc.Wire { + encoder := MembershipTupleEncoder{} + encoder.Init(value) + return encoder.Encode(value) +} + +func (value *MembershipTuple) Bytes() []byte { + return value.Encode().Join() +} + +func ParseMembershipTuple(reader enc.WireView, ignoreCritical bool) (*MembershipTuple, error) { + context := MembershipTupleParsingContext{} + context.Init() + return context.Parse(reader, ignoreCritical) +} + type PassiveStateEncoder struct { Length uint diff --git a/std/sync/svs.go b/std/sync/svs.go index 60e71015..e87455aa 100644 --- a/std/sync/svs.go +++ b/std/sync/svs.go @@ -40,6 +40,13 @@ type SvSync struct { // Channel for incoming state vectors recvSv chan svSyncRecvSvArgs + // Prefix for published full State Vector Data (.../32=sv). + fullVectorPrefix enc.Name + + // lastPullTime debounces pullFullVector per sender so a sync storm across + // many peers does not generate thousands of redundant segment-0 fetches. + lastPullTime map[string]time.Time + // cancellation for face hook faceCancel func() } @@ -58,6 +65,13 @@ type SvSyncOpts struct { // If not provided, the GroupPrefix will be used instead. SyncDataName enc.Name + // FullVectorPrefix is the publish/serve prefix for retrievable FULL + // StateVector Data used by SvsDataRef publish+pull recovery. The + // version component is appended when producing the Data. + // If not provided, it defaults to SyncDataName with the trailing + // "32=svs" component (if present) replaced by "32=sv". + FullVectorPrefix enc.Name + // Initial state vector from persistence InitialState *spec_svs.StateVector // Boot time from persistence @@ -73,6 +87,14 @@ type SvSyncOpts struct { UseSignatureTime optional.Optional[bool] // IgnoreValidity ignores validity period in the validation chain IgnoreValidity optional.Optional[bool] + + // SyncVectorThreshold is the max embedded SvsData size (bytes) above + // which the sender switches to PARTIAL (on publication) or + // publish+pull (on periodic sync and recovery). When <= 0, the + // default (1200 bytes) is used. SVS v4 always emits `mhash` and a + // `VectorType` on the wire; there is no legacy StateVector-only + // mode. + SyncVectorThreshold int } type SvSyncUpdate struct { @@ -83,8 +105,11 @@ type SvSyncUpdate struct { } type svSyncRecvSvArgs struct { - sv *spec_svs.StateVector - data enc.Wire + sv *spec_svs.StateVector + data enc.Wire + vectorType optional.Optional[uint64] + mhash []byte + svsDataRef enc.Name } // NewSvSync creates a new SV Sync instance. @@ -124,6 +149,9 @@ func NewSvSync(opts SvSyncOpts) *SvSync { if len(opts.SyncDataName) == 0 { opts.SyncDataName = opts.GroupPrefix } + if opts.SyncVectorThreshold <= 0 { + opts.SyncVectorThreshold = 1200 + } return &SvSync{ o: opts, @@ -135,7 +163,7 @@ func NewSvSync(opts SvSyncOpts) *SvSync { mutex: sync.Mutex{}, state: initialState, mtime: make(map[string]time.Time), - prefix: opts.GroupPrefix.Append(enc.NewVersionComponent(3)), + prefix: opts.GroupPrefix.Append(enc.NewVersionComponent(4)), suppress: false, merge: NewSvMap[uint64](0), @@ -145,6 +173,10 @@ func NewSvSync(opts SvSyncOpts) *SvSync { recvSv: make(chan svSyncRecvSvArgs, 128), + fullVectorPrefix: resolveFullVectorPrefix(opts.FullVectorPrefix, opts.SyncDataName), + + lastPullTime: make(map[string]time.Time), + faceCancel: func() {}, } } @@ -169,7 +201,6 @@ func (s *SvSync) Start() (err error) { return nil } -// (AI GENERATED DESCRIPTION): Runs the SvSync event loop: it performs the initial sync (or passive load), registers periodic timer ticks and face‑up callbacks, processes received state vectors, and exits cleanly when signalled to stop. func (s *SvSync) main() { // Cleanup on exit defer s.o.Client.Engine().DetachHandler(s.prefix) @@ -180,7 +211,7 @@ func (s *SvSync) main() { // Notify everyone when we are back online s.faceCancel = s.o.Client.Engine().Face().OnUp(func() { - time.AfterFunc(100*time.Millisecond, s.sendSyncInterest) + time.AfterFunc(100*time.Millisecond, func() { s.sendSyncInterest(syncSendOther) }) }) defer s.faceCancel() @@ -190,7 +221,7 @@ func (s *SvSync) main() { go s.loadPassiveWires() } else { // Send the initial Sync Interest - go s.sendSyncInterest() + go s.sendSyncInterest(syncSendOther) } for { @@ -242,7 +273,7 @@ func (s *SvSync) SetSeqNo(name enc.Name, seqNo uint64) error { // [Spec] When the node generates a new publication, // immediately emit a Sync Interest s.state.Set(hash, s.o.BootTime, seqNo) - go s.sendSyncInterest() + go s.sendSyncInterest(syncSendPublication, name) return nil } @@ -264,17 +295,15 @@ func (s *SvSync) IncrSeqNo(name enc.Name) uint64 { // [Spec] When the node generates a new publication, // immediately emit a Sync Interest - go s.sendSyncInterest() + go s.sendSyncInterest(syncSendPublication, name) return entry } -// (AI GENERATED DESCRIPTION): Returns the boot time value stored in the SvSync instance. func (s *SvSync) GetBootTime() uint64 { return s.o.BootTime } -// (AI GENERATED DESCRIPTION): Returns a thread‑safe slice of all names currently stored in the SvSync state. func (s *SvSync) GetNames() []enc.Name { s.mutex.Lock() defer s.mutex.Unlock() @@ -287,7 +316,6 @@ func (s *SvSync) GetNames() []enc.Name { return names } -// (AI GENERATED DESCRIPTION): Processes an incoming state vector, updating the local state vector, notifying the application of any changes, and handling suppression and passive‑sync logic while ensuring updates are delivered in order. func (s *SvSync) onReceiveStateVector(args svSyncRecvSvArgs) { // Deliver the updates after this call is done // This ensures the mutex is not held during the callback @@ -372,7 +400,16 @@ func (s *SvSync) onReceiveStateVector(args svSyncRecvSvArgs) { // The above checks each node in the incoming state vector, but // does not check if a node is missing from the incoming state vector. - if !isOutdated && s.state.IsNewerThan(recvSv, func(_, _ uint64) bool { return false }) { + // + // [Spec] For embedded SvsData, VectorType is required by the protocol: + // publish-only Sync Data carries no StateVector and is filtered out + // earlier (see onSyncData). So args.vectorType is guaranteed present + // here; we default missing values to FULL rather than branch on `ok`. + isPartial := args.vectorType.GetOr(spec_svs.VectorTypeFull) == spec_svs.VectorTypePartial + if len(args.mhash) > 0 { + s.handleMhashMismatch(args, recvSv) + } + if !isPartial && !isOutdated && s.state.IsNewerThan(recvSv, func(_, _ uint64) bool { return false }) { isOutdated = true canDrop = false } @@ -402,7 +439,6 @@ func (s *SvSync) onReceiveStateVector(args svSyncRecvSvArgs) { s.ticker.Reset(s.getSuppressionTimeout()) } -// (AI GENERATED DESCRIPTION): Handles a timer expiry by checking suppression state, potentially transitioning to steady state, and asynchronously sending a Sync Interest with the current local state vector. func (s *SvSync) timerExpired() { s.mutex.Lock() defer s.mutex.Unlock() @@ -420,11 +456,10 @@ func (s *SvSync) timerExpired() { // [Spec] On expiration of timer emit a Sync Interest // with the current local state vector. - go s.sendSyncInterest() + go s.sendSyncInterest(syncSendPeriodic) } -// (AI GENERATED DESCRIPTION): Sends a sync Interest: if passive mode is enabled, it publishes all buffered state updates without duplicates; otherwise, it encodes the current state vector into a wire and transmits it, provided the sync service is running. -func (s *SvSync) sendSyncInterest() { +func (s *SvSync) sendSyncInterest(reason syncSendReason, pubName ...enc.Name) { if !s.running.Load() { return } @@ -437,12 +472,16 @@ func (s *SvSync) sendSyncInterest() { return } + var sender enc.Name + if reason == syncSendPublication && len(pubName) > 0 { + sender = pubName[0] + } + // Encode and sign the current state vector - wire := s.encodeSyncData() + wire := s.encodeSyncData(reason, sender) s.sendSyncInterestWith(wire) } -// (AI GENERATED DESCRIPTION): Sends a sync Interest carrying the supplied data wire payload with a 1‑second lifetime, using the object’s prefix, and logs any construction or transmission errors. func (s *SvSync) sendSyncInterestWith(dataWire enc.Wire) { if dataWire == nil { return @@ -465,21 +504,52 @@ func (s *SvSync) sendSyncInterestWith(dataWire enc.Wire) { } } -// (AI GENERATED DESCRIPTION): Builds a signed Data packet containing the current state vector for SVS v3 synchronization. -func (s *SvSync) encodeSyncData() enc.Wire { - // Critical section - sv := func() *spec_svs.StateVector { - s.mutex.Lock() - defer s.mutex.Unlock() - - // [Spec*] Sending always triggers Steady State - s.enterSteadyState() - - return s.state.Encode(func(s uint64) uint64 { return s }) - }() - svWire := (&spec_svs.SvsData{StateVector: sv}).Encode() +func (s *SvSync) encodeSyncData(reason syncSendReason, sender enc.Name) enc.Wire { + s.mutex.Lock() + s.enterSteadyState() + stateSnap := cloneSvMap(s.state) + mtimeSnap := make(map[string]time.Time, len(s.mtime)) + for k, v := range s.mtime { + mtimeSnap[k] = v + } + repair, propagation := s.partialTargets() + s.mutex.Unlock() + + var svsData *spec_svs.SvsData + if shouldUseAnnouncePull(reason, s.o.SyncVectorThreshold, stateSnap) { + ref, err := s.publishFullVectorData(stateSnap) + if err != nil { + log.Error(s, "publishFullVectorData failed", "err", err) + return nil + } + svsData = buildAnnounceSvsData(stateSnap, ref) + } else { + svsData = buildSvsDataForSend(svsSendInput{ + State: stateSnap, + Reason: reason, + Threshold: s.o.SyncVectorThreshold, + Sender: sender, + Repair: repair, + Propagation: propagation, + Mtime: mtimeSnap, + }) + if svsData == nil { + // [Spec] Publication-triggered PARTIAL encoding could not fit + // even the sender-only baseline: fall back to publish+pull. + ref, err := s.publishFullVectorData(stateSnap) + if err != nil { + log.Error(s, "publishFullVectorData failed (fallback)", "err", err) + return nil + } + svsData = buildAnnounceSvsData(stateSnap, ref) + } + } + if svsData == nil { + return nil + } + svWire := svsData.Encode() - // SVS v3 Sync Data + // SVS v4 Sync Data name := s.o.SyncDataName.WithVersion(enc.VersionUnixMicro) // Sign Sync Data @@ -500,7 +570,6 @@ func (s *SvSync) encodeSyncData() enc.Wire { return data.Wire } -// (AI GENERATED DESCRIPTION): Handles a received sync Interest by checking the running state, extracting its AppParam, and passing that payload to the sync‑data processing routine. func (s *SvSync) onSyncInterest(interest ndn.Interest) { if !s.running.Load() { return @@ -516,7 +585,6 @@ func (s *SvSync) onSyncInterest(interest ndn.Interest) { s.onSyncData(interest.AppParam()) } -// (AI GENERATED DESCRIPTION): Processes a received SyncData packet by parsing it, validating the signature, extracting the state vector, and forwarding the vector and original data to the receiver channel. func (s *SvSync) onSyncData(dataWire enc.Wire) { data, sigCov, err := spec.Spec{}.ReadData(enc.NewWireView(dataWire)) if err != nil { @@ -536,18 +604,36 @@ func (s *SvSync) onSyncData(dataWire enc.Wire) { return } - // Decode state vector + // Decode SvsData (embedded FULL, embedded PARTIAL, or publish-only ref). svWire := data.Content().Join() params, err := spec_svs.ParseSvsData(enc.NewBufferView(svWire), false) - if err != nil || params.StateVector == nil { - log.Warn(s, "onSyncInterest failed to parse StateVec", "err", err) + if err != nil { + log.Warn(s, "onSyncInterest failed to parse SvsData", "err", err) + return + } + + // Publish-only ref: advertise that the full vector is retrievable. + if params.StateVector == nil && len(params.SvsDataRef) > 0 { + trustPrefix := pullRefFromSyncDataWire(dataWire) + go s.pullFullVector(params.SvsDataRef, trustPrefix) + return + } + if params.StateVector == nil { + log.Warn(s, "onSyncInterest SvsData has no StateVector") return } - s.recvSv <- svSyncRecvSvArgs{ - sv: params.StateVector, - data: dataWire, + args := svSyncRecvSvArgs{ + sv: params.StateVector, + data: dataWire, + mhash: params.MemberSetHash, + svsDataRef: params.SvsDataRef, } + if vt, ok := params.VectorType.Get(); ok { + args.vectorType = optional.Some(vt) + } + + s.recvSv <- args }, }) } @@ -559,7 +645,6 @@ func (s *SvSync) enterSteadyState() { s.ticker.Reset(s.getPeriodicTimeout()) } -// (AI GENERATED DESCRIPTION): Returns a duration uniformly randomized within ±10% of the configured periodic timeout. func (s *SvSync) getPeriodicTimeout() time.Duration { // [Spec] ±10% uniform jitter jitter := s.o.PeriodicTimeout / 10 @@ -568,7 +653,6 @@ func (s *SvSync) getPeriodicTimeout() time.Duration { return time.Duration(rand.Int64N(int64(max-min))) + min } -// (AI GENERATED DESCRIPTION): Calculates a random suppression timeout duration using an exponential‑decay function based on the configured SuppressionPeriod. func (s *SvSync) getSuppressionTimeout() time.Duration { // [Spec] Exponential decay function // [Spec] c = SuppressionPeriod // constant factor @@ -664,5 +748,16 @@ func (s *SvSync) loadPassiveWires() { } // This is hacky but pragmatic - wait for the state to be processed - time.AfterFunc(500*time.Millisecond, s.sendSyncInterest) + time.AfterFunc(500*time.Millisecond, func() { s.sendSyncInterest(syncSendOther) }) +} + +// partialTargets returns repair and propagation name targets from suppression merge state. +func (s *SvSync) partialTargets() (repair, propagation []enc.Name) { + if !s.suppress { + return nil, nil + } + for name := range s.merge.Iter() { + repair = append(repair, name) + } + return repair, nil } diff --git a/std/sync/svs_alo_data.go b/std/sync/svs_alo_data.go index 27543385..7be450b8 100644 --- a/std/sync/svs_alo_data.go +++ b/std/sync/svs_alo_data.go @@ -25,7 +25,6 @@ type svsDataState struct { SnapBlock int } -// (AI GENERATED DESCRIPTION): Builds the full name for a data object by appending the node identifier, boot‑timestamp, and sequence number to the group’s prefix and marking the resulting name as immutable. func (s *SvsALO) objectName(node enc.Name, boot uint64, seq uint64) enc.Name { return s.GroupPrefix(). Append(node...). @@ -34,7 +33,6 @@ func (s *SvsALO) objectName(node enc.Name, boot uint64, seq uint64) enc.Name { WithVersion(enc.VersionImmutable) } -// (AI GENERATED DESCRIPTION): Publishes a new Data object with the supplied content, updates the SVS state vector and snapshot strategy, and returns the produced name and the instance’s serialized state. func (s *SvsALO) produceObject(content enc.Wire) (enc.Name, enc.Wire, error) { // This instance owns the underlying SVS instance. // So we can be sure that the sequence number does not @@ -118,6 +116,8 @@ func (s *SvsALO) consumeObject(node enc.Name, boot uint64, seq uint64) { fetchName := s.objectName(node, boot, seq) s.client.ConsumeExt(ndn.ConsumeExtArgs{ Name: fetchName, + TryStore: true, + NoMetadata: true, // fetch name includes version+seq; metadata would only block on timeout UseSignatureTime: s.opts.Svs.UseSignatureTime, IgnoreValidity: s.opts.Svs.IgnoreValidity, Callback: func(status ndn.ConsumeState) { diff --git a/std/sync/svs_encode.go b/std/sync/svs_encode.go new file mode 100644 index 00000000..2fcb394c --- /dev/null +++ b/std/sync/svs_encode.go @@ -0,0 +1,258 @@ +package sync + +import ( + "cmp" + "math/rand/v2" + "slices" + "time" + + enc "github.com/named-data/ndnd/std/encoding" + spec_svs "github.com/named-data/ndnd/std/ndn/svs/v3" + "github.com/named-data/ndnd/std/types/optional" +) + +// syncSendReason distinguishes why a Sync Interest is being sent. +type syncSendReason int + +const ( + syncSendOther syncSendReason = iota + syncSendPublication + syncSendPeriodic + syncSendRecovery +) + +// PartialEncodeOpts configures subset selection for inline PARTIAL vectors. +type PartialEncodeOpts struct { + Sender enc.Name + Threshold int + Repair []enc.Name + Propagation []enc.Name + Mtime map[string]time.Time +} + +// svsSendInput carries everything needed to build inline Sync Data for send. +type svsSendInput struct { + State SvMap[uint64] + Reason syncSendReason + Threshold int + Sender enc.Name + Repair []enc.Name + Propagation []enc.Name + Mtime map[string]time.Time +} + +// buildSvsDataForSend picks embedded FULL or PARTIAL SvsData for an outgoing +// Sync message. Returns nil when publication-triggered PARTIAL encoding cannot +// fit even the sender-only baseline: the caller MUST fall back to publish+pull +// (see shouldUseAnnouncePull). Other reasons always return a non-nil result. +func buildSvsDataForSend(in svsSendInput) *spec_svs.SvsData { + fullSv := in.State.Encode(func(seq uint64) uint64 { return seq }) + fullData := &spec_svs.SvsData{ + MemberSetHash: ComputeMembershipHash(in.State), + VectorType: optional.Some(spec_svs.VectorTypeFull), + StateVector: fullSv, + } + + if in.Reason != syncSendPublication || len(fullData.Encode().Join()) <= in.Threshold { + return fullData + } + + partialSv := encodePartialStateVector(in.State, PartialEncodeOpts{ + Sender: in.Sender, + Threshold: in.Threshold, + Repair: in.Repair, + Propagation: in.Propagation, + Mtime: in.Mtime, + }) + if partialSv == nil { + // Baseline exceeded Threshold; caller must use publish+pull. + return nil + } + return &spec_svs.SvsData{ + MemberSetHash: ComputeMembershipHash(in.State), + VectorType: optional.Some(spec_svs.VectorTypePartial), + StateVector: partialSv, + } +} + +// encodePartialStateVector builds a PARTIAL StateVector for new publication. +// Entry [0] is the sender; entries [1..n] are in NDN canonical order. +// +// Returns nil if the sender-only baseline itself exceeds Threshold: +// callers MUST fall back to publish+pull in that case, because including +// the sender entry is required by §4.2 of the v4 spec. +func encodePartialStateVector(state SvMap[uint64], opts PartialEncodeOpts) *spec_svs.StateVector { + seq := func(v uint64) uint64 { return v } + senderHash := opts.Sender.TlvStr() + + senderEntry := state.encodeNameEntry(opts.Sender, seq) + if senderEntry == nil { + senderEntry = &spec_svs.StateVectorEntry{Name: opts.Sender} + } + + // Sender-only baseline must always fit when possible. + baseline := &spec_svs.StateVector{Entries: []*spec_svs.StateVectorEntry{senderEntry}} + baselineData := &spec_svs.SvsData{ + MemberSetHash: ComputeMembershipHash(state), + VectorType: optional.Some(spec_svs.VectorTypePartial), + StateVector: baseline, + } + if len(baselineData.Encode().Join()) > opts.Threshold { + // Caller falls back to publish+pull because we cannot satisfy + // the §4.2 "entry [0] is the sender" rule at this size budget. + return nil + } + + candidates := partialCandidateNames(state, senderHash, opts) + included := map[string]bool{senderHash: true} + entries := []*spec_svs.StateVectorEntry{senderEntry} + + for _, name := range candidates { + hash := name.TlvStr() + if included[hash] { + continue + } + entry := state.encodeNameEntry(name, seq) + if entry == nil { + continue + } + + trial := append(slices.Clone(entries), entry) + sortPartialTail(trial) + trialSv := &spec_svs.StateVector{Entries: trial} + trialData := &spec_svs.SvsData{ + MemberSetHash: ComputeMembershipHash(state), + VectorType: optional.Some(spec_svs.VectorTypePartial), + StateVector: trialSv, + } + if len(trialData.Encode().Join()) > opts.Threshold { + break + } + + entries = trial + included[hash] = true + } + + sortPartialTail(entries) + return &spec_svs.StateVector{Entries: entries} +} + +// partialCandidateNames returns the producer names considered for inclusion +// in a PARTIAL StateVector, in priority order: +// +// 1. Repair targets from the suppression-merge state (newest entries first). +// 2. Propagation targets (the most recently updated producers). +// 3. Inactive producers (zero-value entries) in randomized order — these +// are included only when bandwidth allows, so randomization is fair. +// 4. Remaining active producers, sorted by (a) recency descending then +// (b) canonical NDN name ascending. +// +// The sender is excluded — it is always included at entries[0]. +func partialCandidateNames(state SvMap[uint64], senderHash string, opts PartialEncodeOpts) []enc.Name { + seen := map[string]bool{senderHash: true} + out := make([]enc.Name, 0, len(state)) + + appendUnique := func(names []enc.Name) { + for _, name := range names { + hash := name.TlvStr() + if seen[hash] { + continue + } + if _, ok := state[hash]; !ok { + continue + } + seen[hash] = true + out = append(out, name) + } + } + + appendUnique(opts.Repair) + appendUnique(opts.Propagation) + + inactive := make([]enc.Name, 0) + remaining := make([]enc.Name, 0) + for name, vals := range state.Iter() { + hash := name.TlvStr() + if seen[hash] { + continue + } + if isInactiveProducer(vals) { + inactive = append(inactive, name) + continue + } + remaining = append(remaining, name) + } + + rand.Shuffle(len(inactive), func(i, j int) { + inactive[i], inactive[j] = inactive[j], inactive[i] + }) + appendUnique(inactive) + + slices.SortFunc(remaining, func(a, b enc.Name) int { + return a.Compare(b) + }) + slices.SortFunc(remaining, func(a, b enc.Name) int { + return cmp.Compare(recencyScore(opts.Mtime, b), recencyScore(opts.Mtime, a)) + }) + appendUnique(remaining) + + return out +} + +func isInactiveProducer(vals []SvMapVal[uint64]) bool { + for _, val := range vals { + if val.Value > 0 { + return false + } + } + return true +} + +func recencyScore(mtime map[string]time.Time, name enc.Name) int64 { + if mtime == nil { + return 0 + } + t, ok := mtime[name.TlvStr()] + if !ok { + return 0 + } + return t.UnixNano() +} + +// sortPartialTail keeps entry [0] fixed and sorts [1..n] in canonical name order. +// +// [Spec §4.2] Entry [0] of a PARTIAL StateVector is the sender; remaining +// entries are NOT ordered by membership hash like MemberSet entries are — +// they are ordered by canonical NDN name comparison. StateVectorEntry +// ordering is independent of mhash ordering. +func sortPartialTail(entries []*spec_svs.StateVectorEntry) { + if len(entries) <= 1 { + return + } + slices.SortFunc(entries[1:], func(a, b *spec_svs.StateVectorEntry) int { + return a.Name.Compare(b.Name) + }) +} + +// encodeNameEntry encodes one producer name from the map. +func (m SvMap[V]) encodeNameEntry(name enc.Name, seq func(V) uint64) *spec_svs.StateVectorEntry { + hash := name.TlvStr() + vals, ok := m[hash] + if !ok { + return nil + } + + entry := &spec_svs.StateVectorEntry{ + Name: name, + SeqNoEntries: make([]*spec_svs.SeqNoEntry, 0, len(vals)), + } + for _, val := range vals { + if seqNo := seq(val.Value); seqNo > 0 { + entry.SeqNoEntries = append(entry.SeqNoEntries, &spec_svs.SeqNoEntry{ + BootstrapTime: val.Boot, + SeqNo: seqNo, + }) + } + } + return entry +} diff --git a/std/sync/svs_map.go b/std/sync/svs_map.go index be1ffd0c..3be8e1eb 100644 --- a/std/sync/svs_map.go +++ b/std/sync/svs_map.go @@ -19,7 +19,6 @@ type SvMapVal[V any] struct { Value V } -// (AI GENERATED DESCRIPTION): Compares the Boot field of two SvMapVal[V] values, returning a negative, zero, or positive integer to indicate their ordering. func (*SvMapVal[V]) Cmp(a, b SvMapVal[V]) int { return cmp.Compare(a.Boot, b.Boot) } @@ -29,6 +28,15 @@ func NewSvMap[V any](size int) SvMap[V] { return make(SvMap[V], size) } +// cloneSvMap returns a shallow copy safe for use without holding SvSync.mutex. +func cloneSvMap[V any](m SvMap[V]) SvMap[V] { + out := NewSvMap[V](len(m)) + for hash, vals := range m { + out[hash] = slices.Clone(vals) + } + return out +} + // Get seq entry for a bootstrap time. func (m SvMap[V]) Get(hash string, boot uint64) (value V) { entry := SvMapVal[V]{boot, value} @@ -39,7 +47,6 @@ func (m SvMap[V]) Get(hash string, boot uint64) (value V) { return value } -// (AI GENERATED DESCRIPTION): Adds or updates a value in the sorted list for a given hash, inserting the new entry or replacing the existing one while maintaining the slice sorted by the boot field. func (m SvMap[V]) Set(hash string, boot uint64, value V) { entry := SvMapVal[V]{boot, value} i, match := slices.BinarySearchFunc(m[hash], entry, entry.Cmp) @@ -50,7 +57,6 @@ func (m SvMap[V]) Set(hash string, boot uint64, value V) { m[hash] = slices.Insert(m[hash], i, entry) } -// (AI GENERATED DESCRIPTION): Clears all key/value pairs from the SvMap, safely handling nil maps by doing nothing if the map is nil. func (m SvMap[V]) Clear() { if m != nil { clear(m) @@ -118,7 +124,6 @@ func (m SvMap[V]) Encode(seq func(V) uint64) *spec_svs.StateVector { return &spec_svs.StateVector{Entries: entries} } -// (AI GENERATED DESCRIPTION): Iter returns an iterator over the SvMap that yields each decoded name and its associated slice of SvMapVal values. func (m SvMap[V]) Iter() iter.Seq2[enc.Name, []SvMapVal[V]] { return func(yield func(enc.Name, []SvMapVal[V]) bool) { for hash, val := range m { diff --git a/std/sync/svs_membership_hash.go b/std/sync/svs_membership_hash.go new file mode 100644 index 00000000..908729a9 --- /dev/null +++ b/std/sync/svs_membership_hash.go @@ -0,0 +1,42 @@ +package sync + +import ( + "crypto/sha256" + "slices" + + spec_svs "github.com/named-data/ndnd/std/ndn/svs/v3" +) + +// ComputeMembershipHash returns the membership hash over all (Name, BootstrapTime) pairs in state. +// Each tuple is encoded as a TLV structure (Tuple-T 0xcc with Name and BootstrapTime +// children) using the ndnd standard TLV codec. +func ComputeMembershipHash(state SvMap[uint64]) []byte { + tuples := make([]*spec_svs.MembershipTuple, 0) + for name, vals := range state.Iter() { + for _, val := range vals { + tuples = append(tuples, &spec_svs.MembershipTuple{ + Name: name, + BootstrapTime: val.Boot, + }) + } + } + + slices.SortFunc(tuples, func(a, b *spec_svs.MembershipTuple) int { + if c := a.Name.Compare(b.Name); c != 0 { + return c + } + if a.BootstrapTime < b.BootstrapTime { + return -1 + } + if a.BootstrapTime > b.BootstrapTime { + return 1 + } + return 0 + }) + + h := sha256.New() + for _, t := range tuples { + h.Write(t.Encode().Join()) + } + return h.Sum(nil) +} diff --git a/std/sync/svs_pull.go b/std/sync/svs_pull.go new file mode 100644 index 00000000..edf73638 --- /dev/null +++ b/std/sync/svs_pull.go @@ -0,0 +1,272 @@ +package sync + +import ( + "bytes" + "fmt" + "time" + + enc "github.com/named-data/ndnd/std/encoding" + "github.com/named-data/ndnd/std/log" + "github.com/named-data/ndnd/std/ndn" + spec "github.com/named-data/ndnd/std/ndn/spec_2022" + spec_svs "github.com/named-data/ndnd/std/ndn/svs/v3" + "github.com/named-data/ndnd/std/types/optional" +) + +const ( + syncDataKeyword = "svs" + fullVectorKeyword = "sv" +) + +// deriveFullVectorPrefix maps SyncDataName (.../32=svs) to the published full-vector prefix (.../32=sv). +func deriveFullVectorPrefix(syncDataName enc.Name) enc.Name { + if len(syncDataName) == 0 { + return nil + } + base := syncDataName + if base.At(-1).IsKeyword(syncDataKeyword) { + base = base.Prefix(-1) + } + return base.Append(enc.NewKeywordComponent(fullVectorKeyword)) +} + +// resolveFullVectorPrefix returns the explicit FullVectorPrefix if set, +// otherwise derives it from SyncDataName. +func resolveFullVectorPrefix(explicit, syncDataName enc.Name) enc.Name { + if len(explicit) > 0 { + return explicit.Clone() + } + return deriveFullVectorPrefix(syncDataName) +} + +// pullRefFromSyncDataWire returns the trust prefix for fetching a publish-only +// SvsDataRef. The "ref" field of Sync Data is the published full-vector name +// (.../32=sv/); the trust prefix is the same name with the version +// component stripped, which corresponds to the sender's .../32=sv prefix used +// for all its published full vectors and is what an authorized consumer must +// trust to follow the reference. +func pullRefFromSyncDataWire(dataWire enc.Wire) enc.Name { + data, _, err := spec.Spec{}.ReadData(enc.NewWireView(dataWire)) + if err != nil { + return nil + } + name := data.Name() + if len(name) == 0 { + return nil + } + if name.At(-1).IsVersion() { + name = name.Prefix(-1) + } + return deriveFullVectorPrefix(name) +} + +func buildAnnounceSvsData(state SvMap[uint64], ref enc.Name) *spec_svs.SvsData { + return &spec_svs.SvsData{ + MemberSetHash: ComputeMembershipHash(state), + SvsDataRef: ref, + } +} + +// shouldUseAnnouncePull reports whether the sender should publish at .../32=sv +// and emit publish-only Sync Data (mhash + SvsDataRef, no embedded vector) +// instead of an embedded FULL or PARTIAL StateVector. +func shouldUseAnnouncePull(reason syncSendReason, threshold int, state SvMap[uint64]) bool { + if reason == syncSendRecovery { + return true + } + if reason == syncSendPublication { + return false + } + sv := state.Encode(func(seq uint64) uint64 { return seq }) + full := &spec_svs.SvsData{ + MemberSetHash: ComputeMembershipHash(state), + VectorType: optional.Some(spec_svs.VectorTypeFull), + StateVector: sv, + } + return len(full.Encode().Join()) > threshold +} + +// publishFullVectorData produces retrievable inline FULL SvsData at .../32=sv/. +func (s *SvSync) publishFullVectorData(state SvMap[uint64]) (enc.Name, error) { + if len(s.fullVectorPrefix) == 0 { + return nil, fmt.Errorf("full vector prefix unset") + } + sv := state.Encode(func(seq uint64) uint64 { return seq }) + content := (&spec_svs.SvsData{ + MemberSetHash: ComputeMembershipHash(state), + VectorType: optional.Some(spec_svs.VectorTypeFull), + StateVector: sv, + }).Encode() + name := s.fullVectorPrefix.WithVersion(enc.VersionUnixMicro) + return s.o.Client.Produce(ndn.ProduceArgs{ + Name: name, + Content: content, + }) +} + +// pullFullVectorMinInterval debounces pullFullVector per sender. During convergence on a +// large group, every sync that crosses the membership hash boundary schedules a pull; without +// gating, a node can accumulate redundant segment-0 fetches for the same content, which +// exhausts retry budgets under network load. We allow at most one pull per sender per +// pullFullVectorMinInterval. +const pullFullVectorMinInterval = 5 * time.Second + +// pullFullVector fetches a published full State Vector and merges it on the main loop. +// trustPrefix is the sender's .../32=sv prefix; ref must be equal to or below it. +// +// [Impl] The 5s per-sender debounce (pullFullVectorMinInterval) is an +// implementation detail documented in §5.6 of the v4 spec: it limits the +// fan-in when many peers cross an mhash boundary at the same time and is +// safe to relax provided the consumer's retry budget scales accordingly. +func (s *SvSync) pullFullVector(ref enc.Name, trustPrefix enc.Name) { + if len(ref) == 0 { + return + } + if !isTrustedSvsDataRef(ref, trustPrefix) { + log.Warn(s, "pullFullVector rejected untrusted SvsDataRef", "ref", ref, "trust", trustPrefix) + return + } + + // Debounce per sender: drop the pull if one is already in flight or completed recently. + senderHash := trustPrefix.TlvStr() + s.mutex.Lock() + if last, ok := s.lastPullTime[senderHash]; ok && time.Since(last) < pullFullVectorMinInterval { + s.mutex.Unlock() + return + } + s.lastPullTime[senderHash] = time.Now() + s.mutex.Unlock() + + s.o.Client.ConsumeExt(ndn.ConsumeExtArgs{ + Name: ref.Clone(), + TryStore: true, + NoMetadata: true, + UseSignatureTime: s.o.UseSignatureTime, + IgnoreValidity: s.o.IgnoreValidity, + Callback: func(st ndn.ConsumeState) { + if st.Error() != nil { + log.Warn(s, "pullFullVector failed", "ref", ref, "err", st.Error()) + return + } + if !st.IsComplete() { + return + } + s.onPulledFullVector(st.Content().Join()) + }, + }) +} + +// onPulledFullVector merges a fetched inline FULL SvsData into local state. +// Segment signatures are validated by client.ConsumeExt during fetch. +func (s *SvSync) onPulledFullVector(content []byte) { + params, err := parseFullVectorContent(content) + if err != nil { + log.Warn(s, "onPulledFullVector parse failed", "err", err) + return + } + + s.recvSv <- svSyncRecvSvArgs{ + sv: params.StateVector, + vectorType: optional.Some(spec_svs.VectorTypeFull), + mhash: params.MemberSetHash, + } +} + +func parseFullVectorContent(content []byte) (*spec_svs.SvsData, error) { + params, err := spec_svs.ParseSvsData(enc.NewBufferView(content), false) + if err != nil { + return nil, err + } + if params.StateVector == nil { + return nil, fmt.Errorf("full vector content has no StateVector") + } + if vt, ok := params.VectorType.Get(); ok && vt != spec_svs.VectorTypeFull { + return nil, fmt.Errorf("full vector VectorType=%d, want FULL", vt) + } + if len(params.MemberSetHash) > 0 { + computed := ComputeMembershipHash(stateVectorToMap(params.StateVector)) + if !bytes.Equal(params.MemberSetHash, computed) { + return nil, fmt.Errorf("full vector mhash mismatch") + } + } + return params, nil +} + +func stateVectorToMap(sv *spec_svs.StateVector) SvMap[uint64] { + m := NewSvMap[uint64](len(sv.Entries)) + for _, node := range sv.Entries { + hash := node.Name.TlvStr() + for _, entry := range node.SeqNoEntries { + m.Set(hash, entry.BootstrapTime, entry.SeqNo) + } + } + return m +} + +// sendRecoveryAnnounce publishes at 32=sv and emits announce-only Sync Data (mhash recovery). +func (s *SvSync) sendRecoveryAnnounce() { + if !s.running.Load() || s.o.Passive { + return + } + wire := s.encodeSyncData(syncSendRecovery, enc.Name{}) + s.sendSyncInterestWith(wire) +} + +// handleMhashMismatch schedules announce or pull recovery on membership mismatch. +func (s *SvSync) handleMhashMismatch(args svSyncRecvSvArgs, recvSv SvMap[uint64]) { + localMhash := ComputeMembershipHash(s.state) + if bytes.Equal(localMhash, args.mhash) { + return + } + + trustPrefix := pullRefFromSyncDataWire(args.data) + if len(trustPrefix) == 0 { + trustPrefix = s.fullVectorPrefix + } + + localTuples, remoteTuples := membershipTupleCount(s.state), membershipTupleCount(recvSv) + if localTuples > remoteTuples && membershipContains(s.state, recvSv) { + go s.sendRecoveryAnnounce() + return + } + + // [Spec] Inline FULL is already merged in onReceiveStateVector. + // Pull only when the sender provided a retrievable SvsDataRef (publish-only sync). + if len(args.svsDataRef) == 0 { + return + } + go s.pullFullVector(args.svsDataRef, trustPrefix) +} + +func membershipContains(outer, inner SvMap[uint64]) bool { + for hash, vals := range inner { + for _, v := range vals { + found := false + for _, ov := range outer[hash] { + if ov.Boot == v.Boot { + found = true + break + } + } + if !found { + return false + } + } + } + return true +} + +func membershipTupleCount(m SvMap[uint64]) int { + n := 0 + for _, vals := range m { + n += len(vals) + } + return n +} + +func isTrustedSvsDataRef(ref, senderFullVectorPrefix enc.Name) bool { + if len(ref) == 0 || len(senderFullVectorPrefix) == 0 { + return false + } + return senderFullVectorPrefix.IsPrefix(ref) +} From fe5586b0122356ca6791f0623cc1ec01d11422d1 Mon Sep 17 00:00:00 2001 From: Taranum Wasu Date: Tue, 21 Jul 2026 12:45:49 +0530 Subject: [PATCH 02/17] test: add SVS v4 unit tests Covers the new v4 surface: * Membership hash: stability across rerun, name-order independence, sensitivity to membership changes, insensitivity to SeqNo values. * Embedded FULL / PARTIAL round-trip and decode. * Publish-only form decoding: mhash + SvsDataRef, no StateVector. * handleMhashMismatch behavior when local superset, when only the sender added members, and when only remote added members. * buildAnnounceSvsData + parseFullVectorContent round-trip. * Pull-debounce per-sender gate (pullFullVectorMinInterval). Uses ComputeMembershipHash (renamed from ComputeMhash in the previous commit). --- std/sync/svs_test.go | 479 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 479 insertions(+) create mode 100644 std/sync/svs_test.go diff --git a/std/sync/svs_test.go b/std/sync/svs_test.go new file mode 100644 index 00000000..daf5d56b --- /dev/null +++ b/std/sync/svs_test.go @@ -0,0 +1,479 @@ +package sync + +import ( + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/require" + + enc "github.com/named-data/ndnd/std/encoding" + "github.com/named-data/ndnd/std/ndn" + spec "github.com/named-data/ndnd/std/ndn/spec_2022" + spec_svs "github.com/named-data/ndnd/std/ndn/svs/v3" + sig "github.com/named-data/ndnd/std/security/signer" + "github.com/named-data/ndnd/std/types/optional" + tu "github.com/named-data/ndnd/std/utils/testutils" +) + +// --- shared test helpers and encode tests --- + +func testSvMapAliceBob() SvMap[uint64] { + m := NewSvMap[uint64](0) + m.Set(tu.NoErr(enc.NameFromStr("/ndn/alice")).TlvStr(), 100, 5) + m.Set(tu.NoErr(enc.NameFromStr("/ndn/bob")).TlvStr(), 150, 3) + return m +} + +func TestBuildInlineFullSvsData(t *testing.T) { + tu.SetT(t) + + m := testSvMapAliceBob() + sv := m.Encode(func(s uint64) uint64 { return s }) + data := &spec_svs.SvsData{ + MemberSetHash: ComputeMembershipHash(m), + VectorType: optional.Some(spec_svs.VectorTypeFull), + StateVector: sv, + } + + require.Equal(t, ComputeMembershipHash(m), data.MemberSetHash) + vt, ok := data.VectorType.Get() + require.True(t, ok) + require.Equal(t, spec_svs.VectorTypeFull, vt) + require.NotNil(t, data.StateVector) + require.Len(t, data.StateVector.Entries, 2) +} + +func TestOnReceivePartialSkipsMissingNameOutdated(t *testing.T) { + tu.SetT(t) + + s := &SvSync{ + o: SvSyncOpts{ + OnUpdate: func(SvSyncUpdate) {}, + SuppressionPeriod: 200 * time.Millisecond, + PeriodicTimeout: 30 * time.Second, + }, + state: testSvMapAliceBob(), + mtime: make(map[string]time.Time), + ticker: time.NewTicker(30 * time.Second), + suppress: false, + } + + // PARTIAL with only bob; local knows alice — must not enter suppression. + bobOnly := NewSvMap[uint64](0) + bobOnly.Set(tu.NoErr(enc.NameFromStr("/ndn/bob")).TlvStr(), 150, 3) + partialSv := bobOnly.Encode(func(s uint64) uint64 { return s }) + + s.onReceiveStateVector(svSyncRecvSvArgs{ + sv: partialSv, + vectorType: optional.Some(spec_svs.VectorTypePartial), + mhash: ComputeMembershipHash(bobOnly), + }) + + require.False(t, s.suppress) +} + +func TestOnReceiveFullTreatsMissingNameOutdated(t *testing.T) { + tu.SetT(t) + + s := &SvSync{ + o: SvSyncOpts{ + OnUpdate: func(SvSyncUpdate) {}, + SuppressionPeriod: 200 * time.Millisecond, + PeriodicTimeout: 30 * time.Second, + }, + state: testSvMapAliceBob(), + mtime: make(map[string]time.Time), + ticker: time.NewTicker(30 * time.Second), + suppress: false, + } + + bobOnly := NewSvMap[uint64](0) + bobOnly.Set(tu.NoErr(enc.NameFromStr("/ndn/bob")).TlvStr(), 150, 3) + fullSv := bobOnly.Encode(func(s uint64) uint64 { return s }) + + s.onReceiveStateVector(svSyncRecvSvArgs{ + sv: fullSv, + vectorType: optional.Some(spec_svs.VectorTypeFull), + mhash: ComputeMembershipHash(bobOnly), + }) + + require.True(t, s.suppress) +} + +func TestEncodePartialSenderFirst(t *testing.T) { + tu.SetT(t) + + alice := tu.NoErr(enc.NameFromStr("/ndn/alice")) + bob := tu.NoErr(enc.NameFromStr("/ndn/bob")) + carol := tu.NoErr(enc.NameFromStr("/ndn/carol")) + + m := NewSvMap[uint64](0) + m.Set(alice.TlvStr(), 100, 5) + m.Set(bob.TlvStr(), 150, 3) + m.Set(carol.TlvStr(), 150, 7) + + // Threshold large enough for sender + one peer. + full := &spec_svs.SvsData{ + MemberSetHash: ComputeMembershipHash(m), + VectorType: optional.Some(spec_svs.VectorTypeFull), + StateVector: m.Encode(func(s uint64) uint64 { return s }), + } + threshold := len(full.Encode().Join()) - 1 + + partial := encodePartialStateVector(m, PartialEncodeOpts{ + Sender: carol, + Threshold: threshold, + Mtime: map[string]time.Time{ + alice.TlvStr(): time.Unix(10, 0), + bob.TlvStr(): time.Unix(20, 0), + }, + }) + + require.NotEmpty(t, partial.Entries) + require.Equal(t, carol, partial.Entries[0].Name) + if len(partial.Entries) > 2 { + require.Less(t, partial.Entries[1].Name.Compare(partial.Entries[2].Name), 0) + } +} + +func TestBuildSvsDataForSendPublicationPartial(t *testing.T) { + tu.SetT(t) + + alice := tu.NoErr(enc.NameFromStr("/ndn/alice")) + m := NewSvMap[uint64](0) + m.Set(alice.TlvStr(), 100, 5) + for i := range 20 { + name := tu.NoErr(enc.NameFromStr(fmt.Sprintf("/ndn/peer%d", i))) + m.Set(name.TlvStr(), 150, uint64(i+1)) + } + + full := &spec_svs.SvsData{ + MemberSetHash: ComputeMembershipHash(m), + VectorType: optional.Some(spec_svs.VectorTypeFull), + StateVector: m.Encode(func(s uint64) uint64 { return s }), + } + threshold := len(full.Encode().Join()) / 2 + + pub := buildSvsDataForSend(svsSendInput{ + State: m, Reason: syncSendPublication, Threshold: threshold, Sender: alice, + }) + vt, ok := pub.VectorType.Get() + require.True(t, ok) + require.Equal(t, spec_svs.VectorTypePartial, vt) + require.Less(t, len(pub.StateVector.Entries), len(full.StateVector.Entries)) + + periodic := buildSvsDataForSend(svsSendInput{ + State: m, Reason: syncSendPeriodic, Threshold: threshold, Sender: alice, + }) + vt, ok = periodic.VectorType.Get() + require.True(t, ok) + require.Equal(t, spec_svs.VectorTypeFull, vt) +} + +func TestOnReceivePartialMergesPresentEntriesOnly(t *testing.T) { + tu.SetT(t) + + var updates []SvSyncUpdate + s := &SvSync{ + o: SvSyncOpts{ + OnUpdate: func(u SvSyncUpdate) { updates = append(updates, u) }, + PeriodicTimeout: 30 * time.Second, + }, + state: NewSvMap[uint64](0), + mtime: make(map[string]time.Time), + ticker: time.NewTicker(30 * time.Second), + } + + alice := tu.NoErr(enc.NameFromStr("/ndn/alice")) + bob := tu.NoErr(enc.NameFromStr("/ndn/bob")) + s.state.Set(alice.TlvStr(), 100, 1) + s.state.Set(bob.TlvStr(), 150, 1) + + bobOnly := NewSvMap[uint64](0) + bobOnly.Set(bob.TlvStr(), 150, 4) + partialSv := bobOnly.Encode(func(s uint64) uint64 { return s }) + + s.onReceiveStateVector(svSyncRecvSvArgs{ + sv: partialSv, + vectorType: optional.Some(spec_svs.VectorTypePartial), + mhash: ComputeMembershipHash(bobOnly), + }) + + require.Len(t, updates, 1) + require.Equal(t, bob, updates[0].Name) + require.EqualValues(t, 4, updates[0].High) + require.EqualValues(t, 1, s.state.Get(alice.TlvStr(), 100)) + require.EqualValues(t, 4, s.state.Get(bob.TlvStr(), 150)) +} + +// --- mhash tests --- + +func TestComputeMembershipHashStable(t *testing.T) { + tu.SetT(t) + + m := NewSvMap[uint64](0) + m.Set(tu.NoErr(enc.NameFromStr("/ndn/alice")).TlvStr(), 100, 1) + m.Set(tu.NoErr(enc.NameFromStr("/ndn/bob")).TlvStr(), 150, 3) + + h1 := ComputeMembershipHash(m) + h2 := ComputeMembershipHash(m) + require.Equal(t, h1, h2) + require.Len(t, h1, 32) +} + +func TestComputeMembershipHashOrderIndependent(t *testing.T) { + tu.SetT(t) + + m1 := NewSvMap[uint64](0) + m1.Set(tu.NoErr(enc.NameFromStr("/ndn/alice")).TlvStr(), 100, 1) + m1.Set(tu.NoErr(enc.NameFromStr("/ndn/bob")).TlvStr(), 150, 3) + + m2 := NewSvMap[uint64](0) + m2.Set(tu.NoErr(enc.NameFromStr("/ndn/bob")).TlvStr(), 150, 3) + m2.Set(tu.NoErr(enc.NameFromStr("/ndn/alice")).TlvStr(), 100, 1) + + require.Equal(t, ComputeMembershipHash(m1), ComputeMembershipHash(m2)) +} + +func TestComputeMembershipHashChangesOnMembership(t *testing.T) { + tu.SetT(t) + + m := NewSvMap[uint64](0) + m.Set(tu.NoErr(enc.NameFromStr("/ndn/alice")).TlvStr(), 100, 1) + before := ComputeMembershipHash(m) + + m.Set(tu.NoErr(enc.NameFromStr("/ndn/bob")).TlvStr(), 150, 3) + after := ComputeMembershipHash(m) + require.NotEqual(t, before, after) +} + +func TestComputeMembershipHashIgnoresSeqNo(t *testing.T) { + tu.SetT(t) + + m1 := NewSvMap[uint64](0) + m1.Set(tu.NoErr(enc.NameFromStr("/ndn/alice")).TlvStr(), 100, 1) + + m2 := NewSvMap[uint64](0) + m2.Set(tu.NoErr(enc.NameFromStr("/ndn/alice")).TlvStr(), 100, 99) + + require.Equal(t, ComputeMembershipHash(m1), ComputeMembershipHash(m2)) +} + +func TestSvsDataInlineTLV(t *testing.T) { + tu.SetT(t) + + m := NewSvMap[uint64](0) + m.Set(tu.NoErr(enc.NameFromStr("/ndn/alice")).TlvStr(), 100, 1) + sv := m.Encode(func(s uint64) uint64 { return s }) + + original := &spec_svs.SvsData{ + MemberSetHash: ComputeMembershipHash(m), + VectorType: optional.Some(spec_svs.VectorTypeFull), + StateVector: sv, + } + wire := original.Encode().Join() + + parsed, err := spec_svs.ParseSvsData(enc.NewBufferView(wire), false) + require.NoError(t, err) + require.Equal(t, original.MemberSetHash, parsed.MemberSetHash) + require.Equal(t, original.VectorType, parsed.VectorType) + require.Equal(t, original.StateVector.Entries[0].Name.String(), parsed.StateVector.Entries[0].Name.String()) +} + +func TestSvsDataAnnounceTLV(t *testing.T) { + tu.SetT(t) + + ref := tu.NoErr(enc.NameFromStr("/ndn/svs/alice/100/32=sv/1")) + mhash := make([]byte, 32) + + original := &spec_svs.SvsData{ + MemberSetHash: mhash, + SvsDataRef: ref, + } + wire := original.Encode().Join() + + parsed, err := spec_svs.ParseSvsData(enc.NewBufferView(wire), false) + require.NoError(t, err) + require.Equal(t, mhash, parsed.MemberSetHash) + require.Equal(t, ref.String(), parsed.SvsDataRef.String()) + require.Nil(t, parsed.StateVector) + require.False(t, parsed.VectorType.IsSet()) +} + +func TestSvsDataLegacyParse(t *testing.T) { + tu.SetT(t) + + m := NewSvMap[uint64](0) + m.Set(tu.NoErr(enc.NameFromStr("/ndn/alice")).TlvStr(), 100, 1) + legacy := &spec_svs.SvsData{StateVector: m.Encode(func(s uint64) uint64 { return s })} + wire := legacy.Encode().Join() + + parsed, err := spec_svs.ParseSvsData(enc.NewBufferView(wire), false) + require.NoError(t, err) + require.Nil(t, parsed.MemberSetHash) + require.NotNil(t, parsed.StateVector) +} + +// --- pull / recovery tests --- + +func TestDeriveFullVectorPrefix(t *testing.T) { + tu.SetT(t) + + syncData := tu.NoErr(enc.NameFromStr("/ndn/svs/alice/1700000000/32=svs")) + prefix := deriveFullVectorPrefix(syncData) + require.Equal(t, "/ndn/svs/alice/1700000000/32=sv", prefix.String()) +} + +func TestPullRefFromSyncDataWire(t *testing.T) { + tu.SetT(t) + + syncDataName := tu.NoErr(enc.NameFromStr("/ndn/svs/alice/1700000000/32=svs")). + Append(enc.NewVersionComponent(12345)) + dataWire, err := spec.Spec{}.MakeData( + syncDataName, + &ndn.DataConfig{ContentType: optional.Some(ndn.ContentTypeBlob)}, + enc.Wire{enc.Buffer{0x01}}, + sig.NewSha256Signer(), + ) + require.NoError(t, err) + + ref := pullRefFromSyncDataWire(dataWire.Wire) + require.Equal(t, "/ndn/svs/alice/1700000000/32=sv", ref.String()) +} + +func TestBuildAnnounceSvsData(t *testing.T) { + tu.SetT(t) + + m := testSvMapAliceBob() + ref := tu.NoErr(enc.NameFromStr("/ndn/svs/alice/1700000000/32=sv/999")) + data := buildAnnounceSvsData(m, ref) + + require.Equal(t, ComputeMembershipHash(m), data.MemberSetHash) + require.True(t, ref.Equal(data.SvsDataRef)) + require.Nil(t, data.StateVector) + require.False(t, data.VectorType.IsSet()) + + wire := data.Encode().Join() + parsed, err := spec_svs.ParseSvsData(enc.NewBufferView(wire), false) + require.NoError(t, err) + require.Equal(t, data.MemberSetHash, parsed.MemberSetHash) + require.True(t, ref.Equal(parsed.SvsDataRef)) + require.Nil(t, parsed.StateVector) +} + +func TestShouldUseAnnouncePull(t *testing.T) { + tu.SetT(t) + + m := testSvMapAliceBob() + sv := m.Encode(func(s uint64) uint64 { return s }) + fullSize := len((&spec_svs.SvsData{ + MemberSetHash: ComputeMembershipHash(m), + VectorType: optional.Some(spec_svs.VectorTypeFull), + StateVector: sv, + }).Encode().Join()) + + require.False(t, shouldUseAnnouncePull(syncSendPublication, fullSize-1, m)) + require.False(t, shouldUseAnnouncePull(syncSendPeriodic, fullSize+1, m)) + require.True(t, shouldUseAnnouncePull(syncSendPeriodic, fullSize-1, m)) + require.True(t, shouldUseAnnouncePull(syncSendOther, fullSize-1, m)) + require.True(t, shouldUseAnnouncePull(syncSendRecovery, fullSize-1, m)) +} + +func TestIsTrustedSvsDataRef(t *testing.T) { + tu.SetT(t) + + trust := tu.NoErr(enc.NameFromStr("/ndn/svs/alice/1/32=sv")) + ref := tu.NoErr(enc.NameFromStr("/ndn/svs/alice/1/32=sv/999")) + bad := tu.NoErr(enc.NameFromStr("/ndn/evil/32=sv/1")) + + require.True(t, isTrustedSvsDataRef(ref, trust)) + require.True(t, isTrustedSvsDataRef(trust, trust)) + require.False(t, isTrustedSvsDataRef(bad, trust)) + require.False(t, isTrustedSvsDataRef(ref, nil)) +} + +func TestParseFullVectorContentRejectsBadMhash(t *testing.T) { + tu.SetT(t) + + m := testSvMapAliceBob() + inline := &spec_svs.SvsData{ + MemberSetHash: []byte("not-a-valid-mhash-padding-000000"), + VectorType: optional.Some(spec_svs.VectorTypeFull), + StateVector: m.Encode(func(s uint64) uint64 { return s }), + } + wire := inline.Encode().Join() + + _, err := parseFullVectorContent(wire) + require.Error(t, err) +} + +func TestParseFullVectorContent(t *testing.T) { + tu.SetT(t) + + m := testSvMapAliceBob() + inline := &spec_svs.SvsData{ + MemberSetHash: ComputeMembershipHash(m), + VectorType: optional.Some(spec_svs.VectorTypeFull), + StateVector: m.Encode(func(s uint64) uint64 { return s }), + } + wire := inline.Encode().Join() + + parsed, err := parseFullVectorContent(wire) + require.NoError(t, err) + require.Equal(t, inline.MemberSetHash, parsed.MemberSetHash) + require.Len(t, parsed.StateVector.Entries, 2) +} + +func TestOnPulledFullVectorMergesState(t *testing.T) { + tu.SetT(t) + + var updates []SvSyncUpdate + s := &SvSync{ + o: SvSyncOpts{ + OnUpdate: func(u SvSyncUpdate) { updates = append(updates, u) }, + PeriodicTimeout: 30 * time.Second, + }, + state: NewSvMap[uint64](0), + mtime: make(map[string]time.Time), + ticker: time.NewTicker(30 * time.Second), + recvSv: make(chan svSyncRecvSvArgs, 1), + } + + alice := tu.NoErr(enc.NameFromStr("/ndn/alice")) + s.state.Set(alice.TlvStr(), 100, 1) + + remote := testSvMapAliceBob() + content := (&spec_svs.SvsData{ + MemberSetHash: ComputeMembershipHash(remote), + VectorType: optional.Some(spec_svs.VectorTypeFull), + StateVector: remote.Encode(func(s uint64) uint64 { return s }), + }).Encode().Join() + + go func() { + s.onPulledFullVector(content) + }() + + s.onReceiveStateVector(<-s.recvSv) + + require.Len(t, updates, 2) + require.EqualValues(t, 3, s.state.Get(tu.NoErr(enc.NameFromStr("/ndn/bob")).TlvStr(), 150)) + require.EqualValues(t, 5, s.state.Get(alice.TlvStr(), 100)) +} + +func TestEncodeSyncDataAnnounceMode(t *testing.T) { + tu.SetT(t) + + m := testSvMapAliceBob() + fullSize := len((&spec_svs.SvsData{ + MemberSetHash: ComputeMembershipHash(m), + VectorType: optional.Some(spec_svs.VectorTypeFull), + StateVector: m.Encode(func(s uint64) uint64 { return s }), + }).Encode().Join()) + require.True(t, shouldUseAnnouncePull(syncSendPeriodic, fullSize-1, m)) + + announce := buildAnnounceSvsData(m, tu.NoErr(enc.NameFromStr("/ndn/svs/alice/1/32=sv/2"))) + require.Nil(t, announce.StateVector) + vt, ok := announce.VectorType.Get() + require.False(t, ok || vt == spec_svs.VectorTypePartial) +} From e9522ccc584fe99d0bffcd9d56b57548a83d3eed Mon Sep 17 00:00:00 2001 From: Taranum Wasu Date: Tue, 21 Jul 2026 12:46:03 +0530 Subject: [PATCH 03/17] docs: SVS v4 specification Standalone v4 specification, renamed from the previous "v3 revision" doc. Renames the protocol version throughout (sync interest name v=4, section references updated), replaces "announce + pull" with "publish + pull" and "announce-only" with "publish-only", and rewrites section 5.6 with explicit "Sender procedure" and "Receiver procedure" subsections. Adds rationale for keeping VectorType on the wire (Section 3.4): mhash alone cannot distinguish FULL from PARTIAL because two parties with identical membership but different subscription views may legitimately disagree on what subset was sent. Section 4.3 notes that auto-MTU sizing of SyncVectorThreshold is planned future work and is intentionally out of scope for v4. Section 5.6 includes an implementation note documenting the 5s per-sender debounce on pullFullVector as a local detail that does not affect protocol correctness. --- docs/svs-v4.md | 463 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 463 insertions(+) create mode 100644 docs/svs-v4.md diff --git a/docs/svs-v4.md b/docs/svs-v4.md new file mode 100644 index 00000000..2972cf68 --- /dev/null +++ b/docs/svs-v4.md @@ -0,0 +1,463 @@ +# State Vector Sync (SVS) v4 Specification + +SVS v4 is a revision of SVS v3 for large synchronization groups. It introduces +a membership hash (`mhash`), two embedded state-vector encodings (`FULL` and +`PARTIAL`), and a third publish-only form that references a retrievable full +vector. The protocol is self-contained: there is no compatibility mode with +plain SVS v3 peers — every Sync Data carries `mhash` and a `VectorType`. + +--- + +## 1. Basic Protocol Design + +### 1.1 Small groups + +For most deployments, the complete State Vector fits in one Sync packet. +Nodes exchange **full** State Vectors using steady-state, suppression, merge, +and `OnUpdate` semantics inherited from SVS v3. + +### 1.2 Large groups + +When the encoded State Vector exceeds **`SyncVectorThreshold`** (an +application-configured size budget in bytes), nodes use three dissemination +modes: + +| Mode | Trigger | Wire shape | +|------|---------|------------| +| **Embedded FULL** | Encoded FULL fits in threshold | `mhash` + `VectorType=FULL` + complete `StateVector` in Sync Data | +| **Embedded PARTIAL** | New publication and FULL exceeds threshold | `mhash` + `VectorType=PARTIAL` + subset `StateVector` in Sync Data | +| **Publish + pull** | Periodic sync (large group), or `mhash` mismatch | Produce full vector Data at `32=sv/`; Sync Data carries `mhash` + reference Name only | + +**MemberSetHash (`mhash`)** is always carried inside `SvsData`. It is a +**membership hash**, not a hash of the full State Vector. + +**Full state recovery** uses publish + pull when: + +1. `mhash` differs from the local membership hash, or +2. Periodic sync runs while the local FULL encoding exceeds + `SyncVectorThreshold`, or +3. An embedded `VectorType = FULL` State Vector is outdated per §6.2. + +Link-level fragmentation (NDNLPv2) is below this layer. Publishers use the +ndnd object segmentation APIs when retrievable full-vector Data is large. + +--- + +## 2. Format and Naming + +### 2.1 Sync Interest + +**Sync Interest Name:** + +``` +//v=4 +``` + +Implementations MAY append additional name components after `v=4`. The +Interest nonce is carried in Interest packet fields, not as a name component. + +- Signed Sync Data is carried in `ApplicationParameters`. +- Interest Lifetime is 1 second. +- Sync Interests are unacknowledged. + +### 2.2 Sync Data (in ApplicationParameters) + +**Sync Data Name** (signing identity for the Sync message): + +``` +//// +``` + +- **`version`:** microsecond timestamp. No hash suffix is used. + +**Sync Data Content:** encoded `SvsData` (§3) — either embedded form (FULL +or PARTIAL) or publish-only form. + +### 2.3 Application publication Data + +``` +////seq= +``` + +Application-level naming may vary. Sync vector Data lives in a separate +namespace distinguished by the `32=sv` keyword (§2.4). + +### 2.4 Published full State Vector Data + +Retrievable full State Vector objects use a dedicated sync namespace: + +**Name:** + +``` +////32=sv/ +``` + +**Content:** signed `SvsData` in embedded FULL form: `mhash` + +`VectorType = FULL` + complete `StateVector`. + +**Publish + pull procedure** (periodic sync, `mhash` recovery, join when +FULL exceeds threshold): + +1. Produce the full-vector Data at + `////32=sv/` (ndnd segmentation handles + large content). +2. Send a Sync Interest whose AppParam Sync Data contains publish-only + `SvsData`: `mhash` + `SvsDataRef` pointing at the published name (§3.1). +3. Receivers pull the referenced Data, validate, and merge. + +A Sync message carries either an embedded StateVector or a publish-only +reference — not both. + +--- + +## 3. Packet Specification + +### 3.1 `SvsData` + +`SvsData` has two forms: embedded (FULL or PARTIAL) and publish-only. The +`mhash` field is present in both forms. `VectorType` is only meaningful in +the embedded form. + +#### 3.1.1 Embedded form (FULL or PARTIAL) + +Used when the State Vector (full or a publication-time PARTIAL subset) is +carried inline in Sync Data, or in published full-vector Data at +`32=sv/`. + +``` +SvsData = SVS-DATA-TYPE TLV-LENGTH + MemberSetHash + VectorType + StateVector +``` + +| Field | TLV type | Value | +|-------|----------|-------| +| `MemberSetHash` | `0xCB` | 32-byte SHA-256 digest (`mhash`) | +| `VectorType` | `0xCD` | `0` = FULL, `1` = PARTIAL | +| `StateVector` | `0xC9` | See §3.2 | + +#### 3.1.2 Publish-only form + +Used when Sync Data advertises a retrievable full-vector Data name (periodic +sync, `mhash` recovery). `VectorType` and `StateVector` are absent. + +``` +SvsData = SVS-DATA-TYPE TLV-LENGTH + MemberSetHash + SvsDataRef +``` + +| Field | TLV type | Value | +|-------|----------|-------| +| `MemberSetHash` | `0xCB` | 32-byte SHA-256 digest (`mhash`) | +| `SvsDataRef` | `0x07` (Name) | Name of the published full-vector Data. The receiver strips the trailing version component and uses the resulting `32=sv` prefix as the trust anchor for that sender's retrievable full vectors. | + +The inline layout extends ndnd v3 `SvsData` with `MemberSetHash` and +`VectorType` before `StateVector`, matching the Python strawman (`mhash` at +`0xCB`, vector at `0xC9`/`0xCA`). + +### 3.2 `StateVector` + +``` +StateVector = STATE-VECTOR-TYPE TLV-LENGTH + *StateVectorEntry + +StateVectorEntry = STATE-VECTOR-ENTRY-TYPE TLV-LENGTH + Name + *SeqNoEntry + +SeqNoEntry = SEQ-NO-ENTRY-TYPE TLV-LENGTH + BootstrapTime + SeqNo +``` + +| TLV | Type (decimal) | Type (hex) | +|-----|----------------|------------| +| `STATE-VECTOR-TYPE` | 201 | `0xC9` | +| `STATE-VECTOR-ENTRY-TYPE` | 202 | `0xCA` | +| `SEQ-NO-ENTRY-TYPE` | 210 | `0xD2` | +| `BOOTSTRAP-TIME-TYPE` | 212 | `0xD4` | +| `SEQ-NO-TYPE` | 214 | `0xD6` | + +**Rules:** + +- Sequence numbers are 1-indexed. +- Bootstrap time is seconds since Unix epoch. +- If an entry is absent, its sequence number is treated as 0 for comparison. +- If any received `BootstrapTime` is more than 86400s in the future, the + entire `StateVector` SHOULD be ignored. + +### 3.3 `MemberSetHash` (`mhash`) + +`mhash` is a **membership hash**. It is not a hash of the full State Vector +and not a hash of sequence numbers. + +**Membership** is the set of participants, each identified by: + +``` +(Producer Name, Bootstrap Time) +``` + +**Computation:** + +``` +members = { (Name, BootstrapTime) | node knows this member in the sync group } +sort by NDN canonical order of Name, then by BootstrapTime ascending +mhash = SHA-256( concatenation of canonical TLV bytes of each (Name, BootstrapTime) pair ) +``` + +Recompute `mhash` whenever membership changes (member added, removed, or new +bootstrap time for a name). + +The Python strawman hashes sorted producer names only. SVS v4 includes +Bootstrap Time in each membership tuple, consistent with SVS v3 identity. + +Membership data and State Vector data are separate concepts. Membership is +carried implicitly in the full State Vector. `mhash` summarizes membership +for quick comparison. + +### 3.4 `VectorType` (embedded form) + +| Value | Name | Meaning | +|-------|------|---------| +| `0` | **FULL** | `StateVector` contains the complete advertised state (§4.1 ordering). | +| `1` | **PARTIAL** | `StateVector` contains a subset (§4.2). Used for new publication only when FULL exceeds threshold. | + +`VectorType` is required on the wire because it lets a receiver skip the +more expensive subset-evaluation code path when it sees `FULL`, and lets a +sender guarantee the receiver knows whether missing names imply partition +(FULL) or merely "not included in this subset" (PARTIAL). `mhash` alone +cannot convey this — two parties with identical membership but different +subscription views may legitimately disagree on what subset was sent. + +`mhash` is present in both embedded and publish-only `SvsData` messages. + +--- + +## 4. State Vector Encoding + +### 4.1 FULL State Vector + +- Include all known members and their latest sequence numbers per bootstrap. +- Entries ordered in NDN canonical order of `Name`. +- Set `VectorType = FULL`. + +### 4.2 PARTIAL State Vector + +Used on new publication when +`encoded_size(embedded FULL SvsData) > SyncVectorThreshold`. + +- Set `VectorType = PARTIAL`. +- **Entry `[0]`** is the sender's own `StateVectorEntry`. +- **Entries `[1…n]`** are in NDN canonical order among included peers. + +If the sender-only baseline already exceeds `SyncVectorThreshold`, the +sender falls back to publish + pull rather than emit a PARTIAL vector that +omits the required entry `[0]`. + +An implementation MAY use the following selection priority: + +| Priority | Include | +|----------|---------| +| 1 | Sender (always) | +| 2 | Repair targets | +| 3 | Propagation targets | +| 4 | Random inactive producers | +| 5 | Others by recency | + +Stop adding entries when the estimated embedded `SvsData` size approaches +`SyncVectorThreshold`. + +### 4.3 `SyncVectorThreshold` + +- Configurable implementation parameter (application packet size budget) in + bytes. +- When `encoded_size(FULL) ≤ SyncVectorThreshold`, nodes use embedded FULL + (with `mhash` and `VectorType=FULL`). +- When `encoded_size(FULL) > SyncVectorThreshold`, nodes switch to PARTIAL + (publication) or publish + pull (periodic sync and recovery). + +The wire format is independent of `SyncVectorThreshold`. All Sync messages +carry `mhash` and a `VectorType` (or `SvsDataRef` for publish-only). +`SyncVectorThreshold <= 0` selects the default 1200-byte budget. + +> **Future work:** the spec currently treats `SyncVectorThreshold` as a +> static application-level constant. Auto-sizing it from observed MTU is a +> planned extension and is intentionally out of scope for v4. + +--- + +## 5. State Sync + +Sections 5.1–5.4 inherit their behavior from SVS v3 [Section 4](https://named-data.github.io/StateVectorSync/Specification.html). +SVS v4 adds Sections 5.5–5.9. + +### 5.1 Sync Interest timer + +- `PeriodicTimeout` default 30s (±10% jitter). +- `SuppressionPeriod` default 200ms. +- `SuppressionTimeout` exponential decay. + +### 5.2 Send Sync Interest on new publication + +When the node generates a new publication, it immediately emits a Sync +Interest and resets the timer to `PeriodicTimeout`. + +| Trigger | Action | +|---------|--------| +| `encoded_size(embedded FULL) ≤ SyncVectorThreshold` | Send embedded FULL (`mhash` + `VectorType=FULL` + `StateVector`) | +| `encoded_size(embedded FULL) > SyncVectorThreshold` | Send embedded PARTIAL (`mhash` + `VectorType=PARTIAL` + subset `StateVector`), or publish + pull if the sender-only baseline itself exceeds the threshold | + +### 5.3 Sync Ack policy + +Sync Interests are unacknowledged. + +### 5.4 Steady state and suppression (embedded FULL) + +For incoming Sync Data with embedded `VectorType = FULL`, apply SVS v3 +steady-state and suppression rules. + +### 5.5 PARTIAL State Vector processing + +When `VectorType = PARTIAL`: + +1. Parse `mhash` and `StateVector`. +2. Names omitted from the partial `StateVector` are interpreted as "not + included in this subset" — they do not imply producer removal, outdated + sender, or sequence rollback. +3. For each present entry, merge newer sequence numbers into local state + (§6.1). +4. If `mhash` differs from local `mhash`, perform publish + pull recovery + (§5.6). + +This is the receive-side change versus SVS v3. + +### 5.6 Full state recovery (publish + pull) + +**Triggers:** + +| # | Trigger | Action | +|---|---------|--------| +| 1 | `mhash` in received `SvsData` ≠ locally computed `mhash` | Publish + pull | +| 2 | Embedded `VectorType = FULL` is outdated per §6.2 | Merge embedded if complete; otherwise publish + pull | +| 3 | Periodic sync while local FULL exceeds `SyncVectorThreshold` | Publish + pull (§5.8) | + +Recovery always fetches the complete State Vector from the referenced +`32=sv/` Data. + +**Sender procedure** (on `mhash` mismatch or periodic large-group sync): + +1. Produce full-vector Data at `////32=sv/` + with embedded FULL `SvsData`. +2. Send Sync Interest with publish-only `SvsData` (`mhash` + `SvsDataRef`). + +**Receiver procedure:** + +1. Identify the sender from the Sync Data signature, or — when the Sync + Data is PARTIAL — from PARTIAL entry `[0]`, which is the sender's own + entry per §4.2. +2. If the Sync Data is embedded FULL and complete: merge directly. +3. If the Sync Data is publish-only: read `SvsDataRef`; express Interest for + that name; validate; merge; update local `mhash`. +4. Continue application data fetch via SvsALO (`OnUpdate`) as today. + +> **Implementation note:** A consumer may receive many publish-only Sync +> messages that all cross the `mhash` boundary simultaneously. To bound the +> resulting pull fan-in, implementations commonly debounce per-sender pull +> attempts (e.g., 5 seconds per sender prefix). This is a local +> implementation detail and does not affect protocol correctness — a +> debounced pull is equivalent to a slightly delayed pull. + +Use ndnd segmentation when fetched Data content is large. + +### 5.7 New node join + +1. Joining node **N** multicasts Sync Interest whose embedded State Vector + contains only itself: `(Name=N, SeqNo=0)`. The Sync Data's `mhash` is + the SHA-256 of N's single-member membership list. +2. Existing members receive the announcement. +3. Suppression limits duplicate responses; typically one member **A** + provides recovery state. +4. If FULL fits inline: **A** responds with embedded `VectorType = FULL`. +5. If FULL exceeds `SyncVectorThreshold`: **A** uses publish + pull + (produce at `32=sv/`, then publish-only Sync Data). +6. Normal synchronization proceeds through SvsALO. + +### 5.8 Periodic sync in large groups + +| Local FULL size | Periodic Sync behavior | +|-----------------|------------------------| +| `≤ SyncVectorThreshold` | Embedded FULL | +| `> SyncVectorThreshold` | Publish + pull (produce full-vector Data, then publish-only Sync Data) | + +Periodic sync does not send embedded PARTIAL vectors. + +### 5.9 Summary of sync triggers + +| Event | `size ≤ threshold` | `size > threshold` | +|-------|--------------------|--------------------| +| **New publication** | Embedded FULL | Embedded PARTIAL (or publish + pull fallback) | +| **Periodic sync** | Embedded FULL | Publish + pull | +| **`mhash` mismatch** | Publish + pull (if recovery needed) | Publish + pull | + +--- + +## 6. Comparing and Merging State Vectors + +### 6.1 Merge rule + +For each matching `(Name, BootstrapTime)`, retain the maximum `SeqNo`. + +### 6.2 Outdated vector (embedded FULL only) + +State Vector `A` is outdated to `B` if: + +- `A` is missing a name present in `B`, or +- `A` has a strictly smaller `SeqNo` for any entry. + +For `VectorType = PARTIAL`, the missing-name rule does not apply to names +omitted from the partial message. + +--- + +## 7. Examples + +### 7.1 Small group + +Three nodes `A`, `B`, `C`. Full State Vector fits. `A` publishes; sends +embedded FULL Sync Interest `[A:11, B:15, C:25]`. Peers merge. + +### 7.2 Large group + +Group exceeds `SyncVectorThreshold`. Producer `P` publishes: + +- `P` sends embedded PARTIAL `SvsData { mhash, VectorType=PARTIAL, + StateVector=[P:…, A:…, …] }`. +- Receiver merges present entries only. +- If `mhash` differs, `P` (or receiver per policy) triggers publish + pull + (§5.6). + +### 7.3 Large group + +- `A` produces full vector at `/group/A/boot/32=sv/`. +- `A` sends publish-only Sync Data `{ mhash, + SvsDataRef=/group/A/boot/32=sv/ }`. +- Peers pull and merge. + +### 7.4 New node join + +- `N` sends self-only vector `[N:0]` with `mhash`. +- `A` responds with embedded FULL or publish + pull. +- `N` merges and synchronizes via SvsALO. + +--- + +## 8. Interoperability + +SVS v4 defines a single wire profile. It does not interoperate with plain +SVS v3 peers in the same sync group: deployments upgrade all nodes to a +v4-conformant implementation at the same time. Every Sync Data carries +`mhash` and a `VectorType` (or `SvsDataRef` for publish-only). The +implementation never emits a legacy `StateVector`-only `SvsData`, regardless +of `SyncVectorThreshold` (a `Threshold ≤ 0` selects the 1200-byte default). \ No newline at end of file From 10022853729b84a9172ec816e602fd7bb3511fc1 Mon Sep 17 00:00:00 2001 From: Taranum Wasu Date: Wed, 22 Jul 2026 14:23:46 +0530 Subject: [PATCH 04/17] fix(e2e): use AIMD congestion window in segment fetcher The fetcher was using NewFixedCongestionWindow(100), which ignores all congestion signals. On the 53-node sprint e2e topology, 8 concurrent ndnd cat consumers each maintain 100 outstanding Interests, producing ~800 concurrent Interests that overwhelm the network. With no congestion response, loss rate climbs and a single segment loses 3 retries in a row, aborting the cat fetch with 'retries exhausted, segment number=N'. Switch to NewAIMDCongestionWindow(100). The AIMD window halves on SigLoss/SigCongest and grows by 1/cwnd on SigData. The signals are already emitted in handleResult; this just wires them to a window that responds. Verified locally: all 3 scenarios (NDNd, NFD, NDNd replay) pass on the sprint topology. --- std/object/client_consume_seg.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/std/object/client_consume_seg.go b/std/object/client_consume_seg.go index 8fa9440e..a78bd752 100644 --- a/std/object/client_consume_seg.go +++ b/std/object/client_consume_seg.go @@ -46,13 +46,18 @@ type retxEntry struct { retries int } -// (AI GENERATED DESCRIPTION): Initializes a new `rrSegFetcher` with the given client, an empty stream list, a fixed congestion window of size 100, no outstanding packets, an empty retransmission queue, a retry counter map, and a maximum of 3 retries. +// Initializes a new rrSegFetcher with the given client. The default congestion +// window is AIMD: it grows by 1 segment on every data arrival and is halved +// on loss or Nack-Congestion. Without AIMD (e.g. FixedCongestionWindow) the +// 53-node sprint e2e topology flaps: many concurrent consumers maintain 100 +// outstanding Interests each, the network becomes congested, and a single +// segment loses 3 retries in a row, aborting the cat fetch. func newRrSegFetcher(client *Client) rrSegFetcher { return rrSegFetcher{ mutex: sync.RWMutex{}, client: client, streams: make([]*ConsumeState, 0), - window: cong.NewFixedCongestionWindow(100), + window: cong.NewAIMDCongestionWindow(100), outstanding: 0, retxQueue: list.New(), txCounter: make(map[*ConsumeState]int), @@ -148,7 +153,6 @@ func (s *rrSegFetcher) findWork() *ConsumeState { return state } -// (AI GENERATED DESCRIPTION): Checks for pending or retransmitted segments, builds and expresses Interest packets for them (updating the congestion window and retry count), and handles the outcome until no more work or the window becomes full. func (s *rrSegFetcher) check() { for { log.Debug(nil, "Checking for work") @@ -300,7 +304,6 @@ func (s *rrSegFetcher) handleData(args ndn.ExpressCallbackArgs, state *ConsumeSt }) } -// (AI GENERATED DESCRIPTION): Handles a validated Data packet by extracting its segment number, storing its payload in the consume state’s buffer, updating counters and sliding windows, and finalizing the state when all segments have been received. func (s *rrSegFetcher) handleValidatedData(args ndn.ExpressCallbackArgs, state *ConsumeState) { // get the final block id if we don't know the segment count if state.segCnt == -1 { // TODO: can change? @@ -398,21 +401,18 @@ func (s *rrSegFetcher) enqueueForRetransmission(state *ConsumeState, seg uint64, s.retxQueue.PushBack(&retxEntry{state, seg, retries}) } -// (AI GENERATED DESCRIPTION): Increments the thread‑safe count of outstanding segment fetch requests. func (s *rrSegFetcher) incrementOutstanding() { s.mutex.Lock() defer s.mutex.Unlock() s.outstanding++ } -// (AI GENERATED DESCRIPTION): Decrements the rrSegFetcher’s outstanding request counter in a thread‑safe manner. func (s *rrSegFetcher) decrementOutstanding() { s.mutex.Lock() defer s.mutex.Unlock() s.outstanding-- } -// (AI GENERATED DESCRIPTION): Decrements the outstanding transmission counter for a given consume state, protecting the update with the fetcher’s mutex. func (s *rrSegFetcher) decrementTxCounter(state *ConsumeState) { s.mutex.Lock() defer s.mutex.Unlock() From 5d8b1beb9af2338e57d2fbda3cbe0bcd4eac4f3f Mon Sep 17 00:00:00 2001 From: Taranum Wasu Date: Wed, 22 Jul 2026 15:10:27 +0530 Subject: [PATCH 05/17] sync: address review comments on SVS v4 * onReceiveStateVector: gate handleMhashMismatch on FULL/publish-only. PARTIAL vectors are subsets by design, so the recvSv tuple-count superset check in handleMhashMismatch would spuriously trigger sendRecoveryAnnounce on every PARTIAL receipt from a node whose own view is a superset (i.e. always, in steady state). * partialTargets: fix the comment to say it returns only repair targets; propagation is currently unused and always nil. * partialCandidateNames: use a single SortFunc comparator for (recency desc, canonical name asc). The previous code sorted by name then by recency in two passes, but slices.SortFunc is not stable so the first sort is not a reliable tie-breaker for entries with equal recency scores. --- std/sync/svs.go | 11 +++++++++-- std/sync/svs_encode.go | 9 ++++++--- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/std/sync/svs.go b/std/sync/svs.go index e87455aa..700b228f 100644 --- a/std/sync/svs.go +++ b/std/sync/svs.go @@ -406,7 +406,13 @@ func (s *SvSync) onReceiveStateVector(args svSyncRecvSvArgs) { // earlier (see onSyncData). So args.vectorType is guaranteed present // here; we default missing values to FULL rather than branch on `ok`. isPartial := args.vectorType.GetOr(spec_svs.VectorTypeFull) == spec_svs.VectorTypePartial - if len(args.mhash) > 0 { + // [Spec] Membership recovery is a FULL-boundary operation: only an embedded + // FULL StateVector or a publish-only Sync Data (which carries no StateVector + // and is filtered earlier in onSyncData) represents the sender's complete + // membership view. A PARTIAL vector is a subset by design, so the recvSv + // tuple-count superset check in handleMhashMismatch would spuriously + // trigger sendRecoveryAnnounce for the local node's normal PUBLISH path. + if len(args.mhash) > 0 && !isPartial { s.handleMhashMismatch(args, recvSv) } if !isPartial && !isOutdated && s.state.IsNewerThan(recvSv, func(_, _ uint64) bool { return false }) { @@ -751,7 +757,8 @@ func (s *SvSync) loadPassiveWires() { time.AfterFunc(500*time.Millisecond, func() { s.sendSyncInterest(syncSendOther) }) } -// partialTargets returns repair and propagation name targets from suppression merge state. +// partialTargets returns the repair target names from the suppression-merge +// state. propagation is currently unused and always nil. func (s *SvSync) partialTargets() (repair, propagation []enc.Name) { if !s.suppress { return nil, nil diff --git a/std/sync/svs_encode.go b/std/sync/svs_encode.go index 2fcb394c..c5e9752f 100644 --- a/std/sync/svs_encode.go +++ b/std/sync/svs_encode.go @@ -188,12 +188,15 @@ func partialCandidateNames(state SvMap[uint64], senderHash string, opts PartialE }) appendUnique(inactive) + // Sort by (recency desc, canonical name asc) using a single comparator. + // slices.SortFunc is not stable, so sorting twice is not a reliable + // tie-breaker for entries with equal recency scores. slices.SortFunc(remaining, func(a, b enc.Name) int { + if c := cmp.Compare(recencyScore(opts.Mtime, b), recencyScore(opts.Mtime, a)); c != 0 { + return c + } return a.Compare(b) }) - slices.SortFunc(remaining, func(a, b enc.Name) int { - return cmp.Compare(recencyScore(opts.Mtime, b), recencyScore(opts.Mtime, a)) - }) appendUnique(remaining) return out From 9340c004495e85c379545d8223f4fb74028c3df5 Mon Sep 17 00:00:00 2001 From: Taranum Wasu Date: Wed, 22 Jul 2026 19:58:04 +0530 Subject: [PATCH 06/17] object: bump metadata fetch retries 3 to 5 and lifetime 1s to 2s The metadata and prefix fetch paths in client_consume use default Retries: 3 and Lifetime: 1s. On the 53-node sprint e2e topology, DV's startup broadcasts a global Reset on every node, which can take up to 9 seconds (locally) or more (in CI) to fully drain. Even after the test announces 'routing converged', a late Reset arriving at a consumer wipes the producer prefix from its route table. The producer's re-announcement follows over the next few DV sync cycles, so a 4-second retry budget can be exhausted before routes stabilize. Bumping to Retries: 5 and Lifetime: 2s gives ~10s of client-side budget for this transient post-convergence gap. Verifies locally on the sprint topology under all three scenarios (NDNd, NFD, NDNd replay). Client-side retry/lifetime are not part of the SVS v4 wire format; this only affects the consumer's tolerance for slow DV propagation. --- std/object/client_consume.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/std/object/client_consume.go b/std/object/client_consume.go index 64ba3e3e..cf597715 100644 --- a/std/object/client_consume.go +++ b/std/object/client_consume.go @@ -117,9 +117,9 @@ func (c *Client) fetchMetadata( Config: &ndn.InterestConfig{ CanBePrefix: true, MustBeFresh: true, - Lifetime: optional.Some(time.Millisecond * 1000), + Lifetime: optional.Some(time.Millisecond * 2000), }, - Retries: 3, // TODO: configurable + Retries: 5, // TODO: configurable (sprint e2e needs ~10s budget for DV startup reset storm) TryStore: utils.If(tryStore, c.store, nil), Callback: func(args ndn.ExpressCallbackArgs) { if args.Result == ndn.InterestResultError { @@ -174,9 +174,9 @@ func (c *Client) fetchDataByPrefix( Config: &ndn.InterestConfig{ CanBePrefix: true, MustBeFresh: true, - Lifetime: optional.Some(time.Millisecond * 1000), + Lifetime: optional.Some(time.Millisecond * 2000), }, - Retries: 3, // TODO: configurable + Retries: 5, // TODO: configurable (sprint e2e needs ~10s budget for DV startup reset storm) TryStore: utils.If(tryStore, c.store, nil), Callback: func(args ndn.ExpressCallbackArgs) { if args.Result == ndn.InterestResultError { From 56c2af03fbbfdd5793da51041b646f42807f90b3 Mon Sep 17 00:00:00 2001 From: Taranum Wasu Date: Fri, 24 Jul 2026 14:27:33 +0530 Subject: [PATCH 07/17] docs+code: address easy review comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit docs: rename §1.2 row names to 'Inline FULL / Inline PARTIAL / Out-of-band FULL' and replace 'embedded' with 'inline' throughout the spec to match. docs: simplify mhash description in §3 (point to §4.2). docs: remove 'Future work' block from §4.3 (out of scope for v4). docs: clarify §6.2 outdated-vector rule applies to FULL only; PARTIAL omissions are a subset by design. docs: remove v3 / 'Python strawman' / NDNLPv2 / 'ndnd object segmentation APIs' references. code: bump rrSegFetcher maxRetries 3 -> 5 (matches PR description and the existing 'TODO: make it configurable' comment; aligns with metadata fetch retry budget in client_consume.go). code: drop long 'clarification' comment on sortPartialTail (kept the one-line summary). Spec logic is unchanged. Build passes. --- docs/svs-v4.md | 121 +++++++++++++++---------------- std/object/client_consume_seg.go | 2 +- std/sync/svs_encode.go | 5 -- 3 files changed, 58 insertions(+), 70 deletions(-) diff --git a/docs/svs-v4.md b/docs/svs-v4.md index 2972cf68..a58bd95b 100644 --- a/docs/svs-v4.md +++ b/docs/svs-v4.md @@ -1,10 +1,10 @@ # State Vector Sync (SVS) v4 Specification -SVS v4 is a revision of SVS v3 for large synchronization groups. It introduces -a membership hash (`mhash`), two embedded state-vector encodings (`FULL` and -`PARTIAL`), and a third publish-only form that references a retrievable full -vector. The protocol is self-contained: there is no compatibility mode with -plain SVS v3 peers — every Sync Data carries `mhash` and a `VectorType`. +SVS v4 is a state-vector synchronization protocol for large sync groups. +It introduces a membership hash (`mhash`), two inline state-vector +encodings (`FULL` and `PARTIAL`), and a third publish-only form that +references a retrievable full vector. Every Sync Data carries `mhash` and +a `VectorType`. --- @@ -14,7 +14,7 @@ plain SVS v3 peers — every Sync Data carries `mhash` and a `VectorType`. For most deployments, the complete State Vector fits in one Sync packet. Nodes exchange **full** State Vectors using steady-state, suppression, merge, -and `OnUpdate` semantics inherited from SVS v3. +and `OnUpdate` semantics. ### 1.2 Large groups @@ -24,22 +24,22 @@ modes: | Mode | Trigger | Wire shape | |------|---------|------------| -| **Embedded FULL** | Encoded FULL fits in threshold | `mhash` + `VectorType=FULL` + complete `StateVector` in Sync Data | -| **Embedded PARTIAL** | New publication and FULL exceeds threshold | `mhash` + `VectorType=PARTIAL` + subset `StateVector` in Sync Data | -| **Publish + pull** | Periodic sync (large group), or `mhash` mismatch | Produce full vector Data at `32=sv/`; Sync Data carries `mhash` + reference Name only | +| **Inline FULL** | Encoded FULL fits in threshold | `mhash` + `VectorType=FULL` + complete `StateVector` in Sync Data | +| **Inline PARTIAL** | New publication and FULL exceeds threshold | `mhash` + `VectorType=PARTIAL` + subset `StateVector` in Sync Data | +| **Out-of-band FULL** | Periodic sync (large group), or `mhash` mismatch | Produce full vector Data at `32=sv/`; Sync Data carries `mhash` + reference Name only | -**MemberSetHash (`mhash`)** is always carried inside `SvsData`. It is a -**membership hash**, not a hash of the full State Vector. +**MemberSetHash (`mhash`)** is the SHA-256 digest of the membership +described in §4.2. **Full state recovery** uses publish + pull when: 1. `mhash` differs from the local membership hash, or 2. Periodic sync runs while the local FULL encoding exceeds `SyncVectorThreshold`, or -3. An embedded `VectorType = FULL` State Vector is outdated per §6.2. +3. An inline `VectorType = FULL` State Vector is outdated per §6.2. -Link-level fragmentation (NDNLPv2) is below this layer. Publishers use the -ndnd object segmentation APIs when retrievable full-vector Data is large. +Retrievable full-vector Data uses the standard NDN segmentation convention +when it exceeds a single packet. --- @@ -70,7 +70,7 @@ Interest nonce is carried in Interest packet fields, not as a name component. - **`version`:** microsecond timestamp. No hash suffix is used. -**Sync Data Content:** encoded `SvsData` (§3) — either embedded form (FULL +**Sync Data Content:** encoded `SvsData` (§3) — either inline form (FULL or PARTIAL) or publish-only form. ### 2.3 Application publication Data @@ -92,7 +92,7 @@ Retrievable full State Vector objects use a dedicated sync namespace: ////32=sv/ ``` -**Content:** signed `SvsData` in embedded FULL form: `mhash` + +**Content:** signed `SvsData` in inline FULL form: `mhash` + `VectorType = FULL` + complete `StateVector`. **Publish + pull procedure** (periodic sync, `mhash` recovery, join when @@ -105,7 +105,7 @@ FULL exceeds threshold): `SvsData`: `mhash` + `SvsDataRef` pointing at the published name (§3.1). 3. Receivers pull the referenced Data, validate, and merge. -A Sync message carries either an embedded StateVector or a publish-only +A Sync message carries either an inline StateVector or a publish-only reference — not both. --- @@ -114,11 +114,11 @@ reference — not both. ### 3.1 `SvsData` -`SvsData` has two forms: embedded (FULL or PARTIAL) and publish-only. The +`SvsData` has two forms: inline (FULL or PARTIAL) and publish-only. The `mhash` field is present in both forms. `VectorType` is only meaningful in -the embedded form. +the inline form. -#### 3.1.1 Embedded form (FULL or PARTIAL) +#### 3.1.1 Inline form (FULL or PARTIAL) Used when the State Vector (full or a publication-time PARTIAL subset) is carried inline in Sync Data, or in published full-vector Data at @@ -153,9 +153,8 @@ SvsData = SVS-DATA-TYPE TLV-LENGTH | `MemberSetHash` | `0xCB` | 32-byte SHA-256 digest (`mhash`) | | `SvsDataRef` | `0x07` (Name) | Name of the published full-vector Data. The receiver strips the trailing version component and uses the resulting `32=sv` prefix as the trust anchor for that sender's retrievable full vectors. | -The inline layout extends ndnd v3 `SvsData` with `MemberSetHash` and -`VectorType` before `StateVector`, matching the Python strawman (`mhash` at -`0xCB`, vector at `0xC9`/`0xCA`). +The inline layout puts `MemberSetHash` and `VectorType` before +`StateVector` (`mhash` at `0xCB`, vector at `0xC9`/`0xCA`). ### 3.2 `StateVector` @@ -210,14 +209,11 @@ mhash = SHA-256( concatenation of canonical TLV bytes of each (Name, BootstrapTi Recompute `mhash` whenever membership changes (member added, removed, or new bootstrap time for a name). -The Python strawman hashes sorted producer names only. SVS v4 includes -Bootstrap Time in each membership tuple, consistent with SVS v3 identity. - Membership data and State Vector data are separate concepts. Membership is carried implicitly in the full State Vector. `mhash` summarizes membership for quick comparison. -### 3.4 `VectorType` (embedded form) +### 3.4 `VectorType` (inline form) | Value | Name | Meaning | |-------|------|---------| @@ -231,7 +227,7 @@ sender guarantee the receiver knows whether missing names imply partition cannot convey this — two parties with identical membership but different subscription views may legitimately disagree on what subset was sent. -`mhash` is present in both embedded and publish-only `SvsData` messages. +`mhash` is present in both inline and publish-only `SvsData` messages. --- @@ -246,7 +242,7 @@ subscription views may legitimately disagree on what subset was sent. ### 4.2 PARTIAL State Vector Used on new publication when -`encoded_size(embedded FULL SvsData) > SyncVectorThreshold`. +`encoded_size(inline FULL SvsData) > SyncVectorThreshold`. - Set `VectorType = PARTIAL`. - **Entry `[0]`** is the sender's own `StateVectorEntry`. @@ -266,14 +262,14 @@ An implementation MAY use the following selection priority: | 4 | Random inactive producers | | 5 | Others by recency | -Stop adding entries when the estimated embedded `SvsData` size approaches +Stop adding entries when the estimated inline `SvsData` size approaches `SyncVectorThreshold`. ### 4.3 `SyncVectorThreshold` - Configurable implementation parameter (application packet size budget) in bytes. -- When `encoded_size(FULL) ≤ SyncVectorThreshold`, nodes use embedded FULL +- When `encoded_size(FULL) ≤ SyncVectorThreshold`, nodes use inline FULL (with `mhash` and `VectorType=FULL`). - When `encoded_size(FULL) > SyncVectorThreshold`, nodes switch to PARTIAL (publication) or publish + pull (periodic sync and recovery). @@ -282,16 +278,12 @@ The wire format is independent of `SyncVectorThreshold`. All Sync messages carry `mhash` and a `VectorType` (or `SvsDataRef` for publish-only). `SyncVectorThreshold <= 0` selects the default 1200-byte budget. -> **Future work:** the spec currently treats `SyncVectorThreshold` as a -> static application-level constant. Auto-sizing it from observed MTU is a -> planned extension and is intentionally out of scope for v4. - --- ## 5. State Sync -Sections 5.1–5.4 inherit their behavior from SVS v3 [Section 4](https://named-data.github.io/StateVectorSync/Specification.html). -SVS v4 adds Sections 5.5–5.9. +Sections 5.1–5.4 describe the steady-state sync loop. Sections 5.5–5.9 +describe the large-group paths. ### 5.1 Sync Interest timer @@ -306,17 +298,17 @@ Interest and resets the timer to `PeriodicTimeout`. | Trigger | Action | |---------|--------| -| `encoded_size(embedded FULL) ≤ SyncVectorThreshold` | Send embedded FULL (`mhash` + `VectorType=FULL` + `StateVector`) | -| `encoded_size(embedded FULL) > SyncVectorThreshold` | Send embedded PARTIAL (`mhash` + `VectorType=PARTIAL` + subset `StateVector`), or publish + pull if the sender-only baseline itself exceeds the threshold | +| `encoded_size(inline FULL) ≤ SyncVectorThreshold` | Send inline FULL (`mhash` + `VectorType=FULL` + `StateVector`) | +| `encoded_size(inline FULL) > SyncVectorThreshold` | Send inline PARTIAL (`mhash` + `VectorType=PARTIAL` + subset `StateVector`), or publish + pull if the sender-only baseline itself exceeds the threshold | ### 5.3 Sync Ack policy Sync Interests are unacknowledged. -### 5.4 Steady state and suppression (embedded FULL) +### 5.4 Steady state and suppression (inline FULL) -For incoming Sync Data with embedded `VectorType = FULL`, apply SVS v3 -steady-state and suppression rules. +For incoming Sync Data with inline `VectorType = FULL`, apply the +steady-state and suppression rules in §5.1–§5.4. ### 5.5 PARTIAL State Vector processing @@ -331,7 +323,8 @@ When `VectorType = PARTIAL`: 4. If `mhash` differs from local `mhash`, perform publish + pull recovery (§5.6). -This is the receive-side change versus SVS v3. +PARTIAL processing is the only receive-side change relative to the +inline-FULL path. ### 5.6 Full state recovery (publish + pull) @@ -340,7 +333,7 @@ This is the receive-side change versus SVS v3. | # | Trigger | Action | |---|---------|--------| | 1 | `mhash` in received `SvsData` ≠ locally computed `mhash` | Publish + pull | -| 2 | Embedded `VectorType = FULL` is outdated per §6.2 | Merge embedded if complete; otherwise publish + pull | +| 2 | Inline `VectorType = FULL` is outdated per §6.2 | Merge inline if complete; otherwise publish + pull | | 3 | Periodic sync while local FULL exceeds `SyncVectorThreshold` | Publish + pull (§5.8) | Recovery always fetches the complete State Vector from the referenced @@ -349,7 +342,7 @@ Recovery always fetches the complete State Vector from the referenced **Sender procedure** (on `mhash` mismatch or periodic large-group sync): 1. Produce full-vector Data at `////32=sv/` - with embedded FULL `SvsData`. + with inline FULL `SvsData`. 2. Send Sync Interest with publish-only `SvsData` (`mhash` + `SvsDataRef`). **Receiver procedure:** @@ -357,7 +350,7 @@ Recovery always fetches the complete State Vector from the referenced 1. Identify the sender from the Sync Data signature, or — when the Sync Data is PARTIAL — from PARTIAL entry `[0]`, which is the sender's own entry per §4.2. -2. If the Sync Data is embedded FULL and complete: merge directly. +2. If the Sync Data is inline FULL and complete: merge directly. 3. If the Sync Data is publish-only: read `SvsDataRef`; express Interest for that name; validate; merge; update local `mhash`. 4. Continue application data fetch via SvsALO (`OnUpdate`) as today. @@ -373,13 +366,13 @@ Use ndnd segmentation when fetched Data content is large. ### 5.7 New node join -1. Joining node **N** multicasts Sync Interest whose embedded State Vector +1. Joining node **N** multicasts Sync Interest whose inline State Vector contains only itself: `(Name=N, SeqNo=0)`. The Sync Data's `mhash` is the SHA-256 of N's single-member membership list. 2. Existing members receive the announcement. 3. Suppression limits duplicate responses; typically one member **A** provides recovery state. -4. If FULL fits inline: **A** responds with embedded `VectorType = FULL`. +4. If FULL fits inline: **A** responds with inline `VectorType = FULL`. 5. If FULL exceeds `SyncVectorThreshold`: **A** uses publish + pull (produce at `32=sv/`, then publish-only Sync Data). 6. Normal synchronization proceeds through SvsALO. @@ -388,17 +381,17 @@ Use ndnd segmentation when fetched Data content is large. | Local FULL size | Periodic Sync behavior | |-----------------|------------------------| -| `≤ SyncVectorThreshold` | Embedded FULL | +| `≤ SyncVectorThreshold` | Inline FULL | | `> SyncVectorThreshold` | Publish + pull (produce full-vector Data, then publish-only Sync Data) | -Periodic sync does not send embedded PARTIAL vectors. +Periodic sync does not send inline PARTIAL vectors. ### 5.9 Summary of sync triggers | Event | `size ≤ threshold` | `size > threshold` | |-------|--------------------|--------------------| -| **New publication** | Embedded FULL | Embedded PARTIAL (or publish + pull fallback) | -| **Periodic sync** | Embedded FULL | Publish + pull | +| **New publication** | Inline FULL | Inline PARTIAL (or publish + pull fallback) | +| **Periodic sync** | Inline FULL | Publish + pull | | **`mhash` mismatch** | Publish + pull (if recovery needed) | Publish + pull | --- @@ -409,15 +402,16 @@ Periodic sync does not send embedded PARTIAL vectors. For each matching `(Name, BootstrapTime)`, retain the maximum `SeqNo`. -### 6.2 Outdated vector (embedded FULL only) +### 6.2 Outdated vector State Vector `A` is outdated to `B` if: - `A` is missing a name present in `B`, or - `A` has a strictly smaller `SeqNo` for any entry. -For `VectorType = PARTIAL`, the missing-name rule does not apply to names -omitted from the partial message. +This rule applies to `VectorType = FULL` only. For `VectorType = PARTIAL`, +omitted names are a subset by design (§4.2) and never indicate that `A` is +outdated relative to `B`. --- @@ -426,13 +420,13 @@ omitted from the partial message. ### 7.1 Small group Three nodes `A`, `B`, `C`. Full State Vector fits. `A` publishes; sends -embedded FULL Sync Interest `[A:11, B:15, C:25]`. Peers merge. +inline FULL Sync Interest `[A:11, B:15, C:25]`. Peers merge. ### 7.2 Large group Group exceeds `SyncVectorThreshold`. Producer `P` publishes: -- `P` sends embedded PARTIAL `SvsData { mhash, VectorType=PARTIAL, +- `P` sends inline PARTIAL `SvsData { mhash, VectorType=PARTIAL, StateVector=[P:…, A:…, …] }`. - Receiver merges present entries only. - If `mhash` differs, `P` (or receiver per policy) triggers publish + pull @@ -448,16 +442,15 @@ Group exceeds `SyncVectorThreshold`. Producer `P` publishes: ### 7.4 New node join - `N` sends self-only vector `[N:0]` with `mhash`. -- `A` responds with embedded FULL or publish + pull. +- `A` responds with inline FULL or publish + pull. - `N` merges and synchronizes via SvsALO. --- ## 8. Interoperability -SVS v4 defines a single wire profile. It does not interoperate with plain -SVS v3 peers in the same sync group: deployments upgrade all nodes to a -v4-conformant implementation at the same time. Every Sync Data carries -`mhash` and a `VectorType` (or `SvsDataRef` for publish-only). The -implementation never emits a legacy `StateVector`-only `SvsData`, regardless -of `SyncVectorThreshold` (a `Threshold ≤ 0` selects the 1200-byte default). \ No newline at end of file +SVS v4 defines a single wire profile. Deployments upgrade all nodes in a +sync group at the same time. Every Sync Data carries `mhash` and a +`VectorType` (or `SvsDataRef` for publish-only). The implementation never +emits a `StateVector`-only `SvsData`, regardless of `SyncVectorThreshold` +(a `Threshold ≤ 0` selects the 1200-byte default). \ No newline at end of file diff --git a/std/object/client_consume_seg.go b/std/object/client_consume_seg.go index a78bd752..4febcb50 100644 --- a/std/object/client_consume_seg.go +++ b/std/object/client_consume_seg.go @@ -61,7 +61,7 @@ func newRrSegFetcher(client *Client) rrSegFetcher { outstanding: 0, retxQueue: list.New(), txCounter: make(map[*ConsumeState]int), - maxRetries: 3, + maxRetries: 5, } } diff --git a/std/sync/svs_encode.go b/std/sync/svs_encode.go index c5e9752f..0870555d 100644 --- a/std/sync/svs_encode.go +++ b/std/sync/svs_encode.go @@ -223,11 +223,6 @@ func recencyScore(mtime map[string]time.Time, name enc.Name) int64 { } // sortPartialTail keeps entry [0] fixed and sorts [1..n] in canonical name order. -// -// [Spec §4.2] Entry [0] of a PARTIAL StateVector is the sender; remaining -// entries are NOT ordered by membership hash like MemberSet entries are — -// they are ordered by canonical NDN name comparison. StateVectorEntry -// ordering is independent of mhash ordering. func sortPartialTail(entries []*spec_svs.StateVectorEntry) { if len(entries) <= 1 { return From 67d93c223d965e2bab08b8422477ebf8e91f52f7 Mon Sep 17 00:00:00 2001 From: Taranum Wasu Date: Fri, 24 Jul 2026 14:46:10 +0530 Subject: [PATCH 08/17] docs+code: reword 'X not Y' anti-patterns MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit docs: §3.1 inline-form 'VectorType is only meaningful in the inline form' -> 'The inline form carries VectorType; the publish-only form does not.' docs: §3.2 'If an entry is absent' -> 'A missing entry compares as SeqNo = 0 against a present entry.' docs: §3.3 drop 'is not a hash of the full State Vector and not a hash of sequence numbers'; describe what mhash is in positive terms and note that it is unaffected by data publications. docs: §3.3 'Membership data and State Vector data are separate concepts' -> describe how the full State Vector carries membership implicitly. docs: §4.1 / §4.2 redundant 'Set VectorType = ...' -> cross-references to §3.4. docs: §6.2 'applies to FULL only ... never indicate' -> 'applies to FULL. ... do not carry any information about whether A is outdated relative to B.' code: drop 'there is no legacy StateVector-only mode' from SyncVectorThreshold doc comment. Spec logic is unchanged. Build passes. --- docs/svs-v4.md | 33 ++++++++++++++++++--------------- std/sync/svs.go | 3 +-- 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/docs/svs-v4.md b/docs/svs-v4.md index a58bd95b..39c1a069 100644 --- a/docs/svs-v4.md +++ b/docs/svs-v4.md @@ -115,8 +115,8 @@ reference — not both. ### 3.1 `SvsData` `SvsData` has two forms: inline (FULL or PARTIAL) and publish-only. The -`mhash` field is present in both forms. `VectorType` is only meaningful in -the inline form. +`mhash` field is present in both forms. The inline form carries +`VectorType`; the publish-only form does not. #### 3.1.1 Inline form (FULL or PARTIAL) @@ -183,14 +183,15 @@ SeqNoEntry = SEQ-NO-ENTRY-TYPE TLV-LENGTH - Sequence numbers are 1-indexed. - Bootstrap time is seconds since Unix epoch. -- If an entry is absent, its sequence number is treated as 0 for comparison. -- If any received `BootstrapTime` is more than 86400s in the future, the - entire `StateVector` SHOULD be ignored. +- A missing entry compares as `SeqNo = 0` against a present entry. +- Reject the entire `StateVector` if any received `BootstrapTime` is more + than 86400s in the future. ### 3.3 `MemberSetHash` (`mhash`) -`mhash` is a **membership hash**. It is not a hash of the full State Vector -and not a hash of sequence numbers. +`mhash` is a **membership hash**: the SHA-256 digest of the membership set +described below. Membership is independent of sequence numbers, so `mhash` +is unaffected by data publications within the group. **Membership** is the set of participants, each identified by: @@ -209,9 +210,10 @@ mhash = SHA-256( concatenation of canonical TLV bytes of each (Name, BootstrapTi Recompute `mhash` whenever membership changes (member added, removed, or new bootstrap time for a name). -Membership data and State Vector data are separate concepts. Membership is -carried implicitly in the full State Vector. `mhash` summarizes membership -for quick comparison. +The full State Vector carries membership implicitly: every member's +`StateVectorEntry` is present with its current sequence number. `mhash` +summarizes that membership for quick comparison without having to walk the +full State Vector. ### 3.4 `VectorType` (inline form) @@ -237,14 +239,14 @@ subscription views may legitimately disagree on what subset was sent. - Include all known members and their latest sequence numbers per bootstrap. - Entries ordered in NDN canonical order of `Name`. -- Set `VectorType = FULL`. +- `VectorType = FULL` (§3.4). ### 4.2 PARTIAL State Vector Used on new publication when `encoded_size(inline FULL SvsData) > SyncVectorThreshold`. -- Set `VectorType = PARTIAL`. +- `VectorType = PARTIAL` (§3.4). - **Entry `[0]`** is the sender's own `StateVectorEntry`. - **Entries `[1…n]`** are in NDN canonical order among included peers. @@ -409,9 +411,10 @@ State Vector `A` is outdated to `B` if: - `A` is missing a name present in `B`, or - `A` has a strictly smaller `SeqNo` for any entry. -This rule applies to `VectorType = FULL` only. For `VectorType = PARTIAL`, -omitted names are a subset by design (§4.2) and never indicate that `A` is -outdated relative to `B`. +This rule applies to `VectorType = FULL`. For `VectorType = PARTIAL`, +omitted names are a subset by design (§4.2): the sender selected a +publication-time subset and `A`'s missing entries do not carry any +information about whether `A` is outdated relative to `B`. --- diff --git a/std/sync/svs.go b/std/sync/svs.go index 700b228f..cd98afe7 100644 --- a/std/sync/svs.go +++ b/std/sync/svs.go @@ -92,8 +92,7 @@ type SvSyncOpts struct { // which the sender switches to PARTIAL (on publication) or // publish+pull (on periodic sync and recovery). When <= 0, the // default (1200 bytes) is used. SVS v4 always emits `mhash` and a - // `VectorType` on the wire; there is no legacy StateVector-only - // mode. + // `VectorType` on the wire. SyncVectorThreshold int } From aed3420134c62fd3bc11cc7779138367197ac919 Mon Sep 17 00:00:00 2001 From: Taranum Wasu Date: Fri, 24 Jul 2026 14:50:22 +0530 Subject: [PATCH 09/17] sync: tighten v4 wire-format validation Address the four open Copilot code-level comments on SVS v4: * onSyncData: reject SvsData that omit the required 32-byte MemberSetHash (mhash) and, for inline form, that omit or carry an invalid VectorType. Prevents PARTIAL-as-FULL misclassification and skipped recovery. * parseFullVectorContent: require VectorType=FULL and a 32-byte mhash on fetched full-vector Data; always verify mhash against the local computation (no more 'if present' opt-in). * pullFullVector: lazy-init lastPullTime under the mutex so the debounce is safe for SvSync instances built via keyed struct literals (tests). * partialTargets: sort the repair name list in NDN canonical order so PARTIAL selection is deterministic across runs. Add unit tests for missing-mhash and missing-VectorType cases on parseFullVectorContent. Spec logic is unchanged. All std/... tests pass. --- std/sync/svs.go | 24 +++++++++++++++++++++++- std/sync/svs_pull.go | 22 ++++++++++++++++------ std/sync/svs_test.go | 30 ++++++++++++++++++++++++++++++ 3 files changed, 69 insertions(+), 7 deletions(-) diff --git a/std/sync/svs.go b/std/sync/svs.go index cd98afe7..e70dc399 100644 --- a/std/sync/svs.go +++ b/std/sync/svs.go @@ -4,6 +4,7 @@ import ( "fmt" "math" rand "math/rand/v2" + "slices" "sync" "sync/atomic" "time" @@ -617,6 +618,14 @@ func (s *SvSync) onSyncData(dataWire enc.Wire) { return } + // [Spec] Every Sync Data carries a 32-byte mhash. Reject malformed + // packets that would misclassify PARTIAL-as-FULL or skip recovery. + if len(params.MemberSetHash) != 32 { + log.Warn(s, "onSyncInterest SvsData missing or invalid mhash", + "len", len(params.MemberSetHash)) + return + } + // Publish-only ref: advertise that the full vector is retrievable. if params.StateVector == nil && len(params.SvsDataRef) > 0 { trustPrefix := pullRefFromSyncDataWire(dataWire) @@ -628,6 +637,17 @@ func (s *SvSync) onSyncData(dataWire enc.Wire) { return } + // [Spec] Inline form must carry VectorType (FULL or PARTIAL). + vt, ok := params.VectorType.Get() + if !ok { + log.Warn(s, "onSyncInterest inline SvsData missing VectorType") + return + } + if vt != spec_svs.VectorTypeFull && vt != spec_svs.VectorTypePartial { + log.Warn(s, "onSyncInterest inline SvsData invalid VectorType", "vt", vt) + return + } + args := svSyncRecvSvArgs{ sv: params.StateVector, data: dataWire, @@ -757,7 +777,8 @@ func (s *SvSync) loadPassiveWires() { } // partialTargets returns the repair target names from the suppression-merge -// state. propagation is currently unused and always nil. +// state, sorted in NDN canonical order so PARTIAL selection is deterministic. +// propagation is currently unused and always nil. func (s *SvSync) partialTargets() (repair, propagation []enc.Name) { if !s.suppress { return nil, nil @@ -765,5 +786,6 @@ func (s *SvSync) partialTargets() (repair, propagation []enc.Name) { for name := range s.merge.Iter() { repair = append(repair, name) } + slices.SortFunc(repair, func(a, b enc.Name) int { return a.Compare(b) }) return repair, nil } diff --git a/std/sync/svs_pull.go b/std/sync/svs_pull.go index edf73638..224a29ad 100644 --- a/std/sync/svs_pull.go +++ b/std/sync/svs_pull.go @@ -130,6 +130,9 @@ func (s *SvSync) pullFullVector(ref enc.Name, trustPrefix enc.Name) { // Debounce per sender: drop the pull if one is already in flight or completed recently. senderHash := trustPrefix.TlvStr() s.mutex.Lock() + if s.lastPullTime == nil { + s.lastPullTime = make(map[string]time.Time) + } if last, ok := s.lastPullTime[senderHash]; ok && time.Since(last) < pullFullVectorMinInterval { s.mutex.Unlock() return @@ -180,14 +183,21 @@ func parseFullVectorContent(content []byte) (*spec_svs.SvsData, error) { if params.StateVector == nil { return nil, fmt.Errorf("full vector content has no StateVector") } - if vt, ok := params.VectorType.Get(); ok && vt != spec_svs.VectorTypeFull { + // [Spec] Fetched full-vector Data is inline FULL: VectorType must be FULL + // and MemberSetHash must be present and valid. + vt, ok := params.VectorType.Get() + if !ok { + return nil, fmt.Errorf("full vector content missing VectorType") + } + if vt != spec_svs.VectorTypeFull { return nil, fmt.Errorf("full vector VectorType=%d, want FULL", vt) } - if len(params.MemberSetHash) > 0 { - computed := ComputeMembershipHash(stateVectorToMap(params.StateVector)) - if !bytes.Equal(params.MemberSetHash, computed) { - return nil, fmt.Errorf("full vector mhash mismatch") - } + if len(params.MemberSetHash) != 32 { + return nil, fmt.Errorf("full vector content missing or invalid mhash (len=%d)", len(params.MemberSetHash)) + } + computed := ComputeMembershipHash(stateVectorToMap(params.StateVector)) + if !bytes.Equal(params.MemberSetHash, computed) { + return nil, fmt.Errorf("full vector mhash mismatch") } return params, nil } diff --git a/std/sync/svs_test.go b/std/sync/svs_test.go index daf5d56b..bcad4940 100644 --- a/std/sync/svs_test.go +++ b/std/sync/svs_test.go @@ -408,6 +408,36 @@ func TestParseFullVectorContentRejectsBadMhash(t *testing.T) { require.Error(t, err) } +func TestParseFullVectorContentRejectsMissingVectorType(t *testing.T) { + tu.SetT(t) + + m := testSvMapAliceBob() + inline := &spec_svs.SvsData{ + MemberSetHash: ComputeMembershipHash(m), + VectorType: optional.None[uint64](), + StateVector: m.Encode(func(s uint64) uint64 { return s }), + } + wire := inline.Encode().Join() + + _, err := parseFullVectorContent(wire) + require.Error(t, err) +} + +func TestParseFullVectorContentRejectsMissingMhash(t *testing.T) { + tu.SetT(t) + + m := testSvMapAliceBob() + inline := &spec_svs.SvsData{ + MemberSetHash: nil, + VectorType: optional.Some(spec_svs.VectorTypeFull), + StateVector: m.Encode(func(s uint64) uint64 { return s }), + } + wire := inline.Encode().Join() + + _, err := parseFullVectorContent(wire) + require.Error(t, err) +} + func TestParseFullVectorContent(t *testing.T) { tu.SetT(t) From 7f04d175fc9c3a5c5aa9080fcc3e4deb3593def7 Mon Sep 17 00:00:00 2001 From: Taranum Wasu Date: Fri, 24 Jul 2026 15:08:32 +0530 Subject: [PATCH 10/17] sync: drop announce wording from SVS internals MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address two more review comments on SVS v4: * SyncVectorThreshold: move from SvSyncOpts to a package-level constant (syncVectorThreshold = 1200). The threshold is not application-tunable: it is a fixed library constant. Removes the public field from SvSyncOpts and the <= 0 default-application block in NewSvSync. * Rename SVS Go identifiers to drop 'announce': buildAnnounceSvsData -> buildPublishSvsData shouldUseAnnouncePull -> shouldUsePublishPull sendRecoveryAnnounce -> sendRecoveryPublish TestSvsDataAnnounceTLV -> TestSvsDataPublishTLV TestBuildAnnounceSvsData -> TestBuildPublishSvsData TestShouldUseAnnouncePull -> TestShouldUsePublishPull TestEncodeSyncDataAnnounceMode -> TestEncodeSyncDataPublishMode Also clean up the 'announce-only Sync Data' and 'announce or pull recovery' comments and rename the test fixture variable. docs/svs-v4.md: spec §1.2, §4.3, §8 now describe SyncVectorThreshold as a fixed library constant (1200 bytes), not an application-configured parameter. The Client.AnnouncePrefix API (used for routing prefix announcement, not SVS) is left unchanged. Spec logic is unchanged. All std/... tests pass. --- docs/svs-v4.md | 13 ++++++------- std/sync/svs.go | 26 +++++++++++--------------- std/sync/svs_encode.go | 2 +- std/sync/svs_pull.go | 14 +++++++------- std/sync/svs_test.go | 28 ++++++++++++++-------------- 5 files changed, 39 insertions(+), 44 deletions(-) diff --git a/docs/svs-v4.md b/docs/svs-v4.md index 39c1a069..31c2d69f 100644 --- a/docs/svs-v4.md +++ b/docs/svs-v4.md @@ -18,8 +18,8 @@ and `OnUpdate` semantics. ### 1.2 Large groups -When the encoded State Vector exceeds **`SyncVectorThreshold`** (an -application-configured size budget in bytes), nodes use three dissemination +When the encoded State Vector exceeds **`SyncVectorThreshold`** (a +fixed library constant of 1200 bytes), nodes use three dissemination modes: | Mode | Trigger | Wire shape | @@ -269,8 +269,9 @@ Stop adding entries when the estimated inline `SvsData` size approaches ### 4.3 `SyncVectorThreshold` -- Configurable implementation parameter (application packet size budget) in - bytes. +`SyncVectorThreshold` is a fixed library constant (1200 bytes) that bounds +the size of an inline SvsData: + - When `encoded_size(FULL) ≤ SyncVectorThreshold`, nodes use inline FULL (with `mhash` and `VectorType=FULL`). - When `encoded_size(FULL) > SyncVectorThreshold`, nodes switch to PARTIAL @@ -278,7 +279,6 @@ Stop adding entries when the estimated inline `SvsData` size approaches The wire format is independent of `SyncVectorThreshold`. All Sync messages carry `mhash` and a `VectorType` (or `SvsDataRef` for publish-only). -`SyncVectorThreshold <= 0` selects the default 1200-byte budget. --- @@ -455,5 +455,4 @@ Group exceeds `SyncVectorThreshold`. Producer `P` publishes: SVS v4 defines a single wire profile. Deployments upgrade all nodes in a sync group at the same time. Every Sync Data carries `mhash` and a `VectorType` (or `SvsDataRef` for publish-only). The implementation never -emits a `StateVector`-only `SvsData`, regardless of `SyncVectorThreshold` -(a `Threshold ≤ 0` selects the 1200-byte default). \ No newline at end of file +emits a `StateVector`-only `SvsData`, regardless of `SyncVectorThreshold`. \ No newline at end of file diff --git a/std/sync/svs.go b/std/sync/svs.go index e70dc399..71c5d4f5 100644 --- a/std/sync/svs.go +++ b/std/sync/svs.go @@ -18,6 +18,12 @@ import ( "github.com/named-data/ndnd/std/utils" ) +// syncVectorThreshold is the max embedded SvsData size (bytes) above +// which the sender switches to PARTIAL (on publication) or publish+pull +// (on periodic sync and recovery). SVS v4 always emits `mhash` and a +// `VectorType` on the wire. +const syncVectorThreshold = 1200 + type SvSync struct { o SvSyncOpts @@ -88,13 +94,6 @@ type SvSyncOpts struct { UseSignatureTime optional.Optional[bool] // IgnoreValidity ignores validity period in the validation chain IgnoreValidity optional.Optional[bool] - - // SyncVectorThreshold is the max embedded SvsData size (bytes) above - // which the sender switches to PARTIAL (on publication) or - // publish+pull (on periodic sync and recovery). When <= 0, the - // default (1200 bytes) is used. SVS v4 always emits `mhash` and a - // `VectorType` on the wire. - SyncVectorThreshold int } type SvSyncUpdate struct { @@ -149,9 +148,6 @@ func NewSvSync(opts SvSyncOpts) *SvSync { if len(opts.SyncDataName) == 0 { opts.SyncDataName = opts.GroupPrefix } - if opts.SyncVectorThreshold <= 0 { - opts.SyncVectorThreshold = 1200 - } return &SvSync{ o: opts, @@ -411,7 +407,7 @@ func (s *SvSync) onReceiveStateVector(args svSyncRecvSvArgs) { // and is filtered earlier in onSyncData) represents the sender's complete // membership view. A PARTIAL vector is a subset by design, so the recvSv // tuple-count superset check in handleMhashMismatch would spuriously - // trigger sendRecoveryAnnounce for the local node's normal PUBLISH path. + // trigger sendRecoveryPublish for the local node's normal PUBLISH path. if len(args.mhash) > 0 && !isPartial { s.handleMhashMismatch(args, recvSv) } @@ -522,18 +518,18 @@ func (s *SvSync) encodeSyncData(reason syncSendReason, sender enc.Name) enc.Wire s.mutex.Unlock() var svsData *spec_svs.SvsData - if shouldUseAnnouncePull(reason, s.o.SyncVectorThreshold, stateSnap) { + if shouldUsePublishPull(reason, syncVectorThreshold, stateSnap) { ref, err := s.publishFullVectorData(stateSnap) if err != nil { log.Error(s, "publishFullVectorData failed", "err", err) return nil } - svsData = buildAnnounceSvsData(stateSnap, ref) + svsData = buildPublishSvsData(stateSnap, ref) } else { svsData = buildSvsDataForSend(svsSendInput{ State: stateSnap, Reason: reason, - Threshold: s.o.SyncVectorThreshold, + Threshold: syncVectorThreshold, Sender: sender, Repair: repair, Propagation: propagation, @@ -547,7 +543,7 @@ func (s *SvSync) encodeSyncData(reason syncSendReason, sender enc.Name) enc.Wire log.Error(s, "publishFullVectorData failed (fallback)", "err", err) return nil } - svsData = buildAnnounceSvsData(stateSnap, ref) + svsData = buildPublishSvsData(stateSnap, ref) } } if svsData == nil { diff --git a/std/sync/svs_encode.go b/std/sync/svs_encode.go index 0870555d..8c876796 100644 --- a/std/sync/svs_encode.go +++ b/std/sync/svs_encode.go @@ -44,7 +44,7 @@ type svsSendInput struct { // buildSvsDataForSend picks embedded FULL or PARTIAL SvsData for an outgoing // Sync message. Returns nil when publication-triggered PARTIAL encoding cannot // fit even the sender-only baseline: the caller MUST fall back to publish+pull -// (see shouldUseAnnouncePull). Other reasons always return a non-nil result. +// (see shouldUsePublishPull). Other reasons always return a non-nil result. func buildSvsDataForSend(in svsSendInput) *spec_svs.SvsData { fullSv := in.State.Encode(func(seq uint64) uint64 { return seq }) fullData := &spec_svs.SvsData{ diff --git a/std/sync/svs_pull.go b/std/sync/svs_pull.go index 224a29ad..f4ad2538 100644 --- a/std/sync/svs_pull.go +++ b/std/sync/svs_pull.go @@ -60,17 +60,17 @@ func pullRefFromSyncDataWire(dataWire enc.Wire) enc.Name { return deriveFullVectorPrefix(name) } -func buildAnnounceSvsData(state SvMap[uint64], ref enc.Name) *spec_svs.SvsData { +func buildPublishSvsData(state SvMap[uint64], ref enc.Name) *spec_svs.SvsData { return &spec_svs.SvsData{ MemberSetHash: ComputeMembershipHash(state), SvsDataRef: ref, } } -// shouldUseAnnouncePull reports whether the sender should publish at .../32=sv +// shouldUsePublishPull reports whether the sender should publish at .../32=sv // and emit publish-only Sync Data (mhash + SvsDataRef, no embedded vector) // instead of an embedded FULL or PARTIAL StateVector. -func shouldUseAnnouncePull(reason syncSendReason, threshold int, state SvMap[uint64]) bool { +func shouldUsePublishPull(reason syncSendReason, threshold int, state SvMap[uint64]) bool { if reason == syncSendRecovery { return true } @@ -213,8 +213,8 @@ func stateVectorToMap(sv *spec_svs.StateVector) SvMap[uint64] { return m } -// sendRecoveryAnnounce publishes at 32=sv and emits announce-only Sync Data (mhash recovery). -func (s *SvSync) sendRecoveryAnnounce() { +// sendRecoveryPublish publishes at 32=sv and emits publish-only Sync Data (mhash recovery). +func (s *SvSync) sendRecoveryPublish() { if !s.running.Load() || s.o.Passive { return } @@ -222,7 +222,7 @@ func (s *SvSync) sendRecoveryAnnounce() { s.sendSyncInterestWith(wire) } -// handleMhashMismatch schedules announce or pull recovery on membership mismatch. +// handleMhashMismatch schedules publish or pull recovery on membership mismatch. func (s *SvSync) handleMhashMismatch(args svSyncRecvSvArgs, recvSv SvMap[uint64]) { localMhash := ComputeMembershipHash(s.state) if bytes.Equal(localMhash, args.mhash) { @@ -236,7 +236,7 @@ func (s *SvSync) handleMhashMismatch(args svSyncRecvSvArgs, recvSv SvMap[uint64] localTuples, remoteTuples := membershipTupleCount(s.state), membershipTupleCount(recvSv) if localTuples > remoteTuples && membershipContains(s.state, recvSv) { - go s.sendRecoveryAnnounce() + go s.sendRecoveryPublish() return } diff --git a/std/sync/svs_test.go b/std/sync/svs_test.go index bcad4940..83862833 100644 --- a/std/sync/svs_test.go +++ b/std/sync/svs_test.go @@ -281,7 +281,7 @@ func TestSvsDataInlineTLV(t *testing.T) { require.Equal(t, original.StateVector.Entries[0].Name.String(), parsed.StateVector.Entries[0].Name.String()) } -func TestSvsDataAnnounceTLV(t *testing.T) { +func TestSvsDataPublishTLV(t *testing.T) { tu.SetT(t) ref := tu.NoErr(enc.NameFromStr("/ndn/svs/alice/100/32=sv/1")) @@ -342,12 +342,12 @@ func TestPullRefFromSyncDataWire(t *testing.T) { require.Equal(t, "/ndn/svs/alice/1700000000/32=sv", ref.String()) } -func TestBuildAnnounceSvsData(t *testing.T) { +func TestBuildPublishSvsData(t *testing.T) { tu.SetT(t) m := testSvMapAliceBob() ref := tu.NoErr(enc.NameFromStr("/ndn/svs/alice/1700000000/32=sv/999")) - data := buildAnnounceSvsData(m, ref) + data := buildPublishSvsData(m, ref) require.Equal(t, ComputeMembershipHash(m), data.MemberSetHash) require.True(t, ref.Equal(data.SvsDataRef)) @@ -362,7 +362,7 @@ func TestBuildAnnounceSvsData(t *testing.T) { require.Nil(t, parsed.StateVector) } -func TestShouldUseAnnouncePull(t *testing.T) { +func TestShouldUsePublishPull(t *testing.T) { tu.SetT(t) m := testSvMapAliceBob() @@ -373,11 +373,11 @@ func TestShouldUseAnnouncePull(t *testing.T) { StateVector: sv, }).Encode().Join()) - require.False(t, shouldUseAnnouncePull(syncSendPublication, fullSize-1, m)) - require.False(t, shouldUseAnnouncePull(syncSendPeriodic, fullSize+1, m)) - require.True(t, shouldUseAnnouncePull(syncSendPeriodic, fullSize-1, m)) - require.True(t, shouldUseAnnouncePull(syncSendOther, fullSize-1, m)) - require.True(t, shouldUseAnnouncePull(syncSendRecovery, fullSize-1, m)) + require.False(t, shouldUsePublishPull(syncSendPublication, fullSize-1, m)) + require.False(t, shouldUsePublishPull(syncSendPeriodic, fullSize+1, m)) + require.True(t, shouldUsePublishPull(syncSendPeriodic, fullSize-1, m)) + require.True(t, shouldUsePublishPull(syncSendOther, fullSize-1, m)) + require.True(t, shouldUsePublishPull(syncSendRecovery, fullSize-1, m)) } func TestIsTrustedSvsDataRef(t *testing.T) { @@ -491,7 +491,7 @@ func TestOnPulledFullVectorMergesState(t *testing.T) { require.EqualValues(t, 5, s.state.Get(alice.TlvStr(), 100)) } -func TestEncodeSyncDataAnnounceMode(t *testing.T) { +func TestEncodeSyncDataPublishMode(t *testing.T) { tu.SetT(t) m := testSvMapAliceBob() @@ -500,10 +500,10 @@ func TestEncodeSyncDataAnnounceMode(t *testing.T) { VectorType: optional.Some(spec_svs.VectorTypeFull), StateVector: m.Encode(func(s uint64) uint64 { return s }), }).Encode().Join()) - require.True(t, shouldUseAnnouncePull(syncSendPeriodic, fullSize-1, m)) + require.True(t, shouldUsePublishPull(syncSendPeriodic, fullSize-1, m)) - announce := buildAnnounceSvsData(m, tu.NoErr(enc.NameFromStr("/ndn/svs/alice/1/32=sv/2"))) - require.Nil(t, announce.StateVector) - vt, ok := announce.VectorType.Get() + publish := buildPublishSvsData(m, tu.NoErr(enc.NameFromStr("/ndn/svs/alice/1/32=sv/2"))) + require.Nil(t, publish.StateVector) + vt, ok := publish.VectorType.Get() require.False(t, ok || vt == spec_svs.VectorTypePartial) } From d27e0227576bfdc90e53b50954b35808db05b91e Mon Sep 17 00:00:00 2001 From: Taranum Wasu Date: Fri, 24 Jul 2026 17:49:23 +0530 Subject: [PATCH 11/17] fix(e2e): retry ExpressR on transient NackReasonNoRoute/Congestion ExpressR previously only retried on InterestResultTimeout. A persistent Nack (e.g. NackReasonNoRoute) failed the request immediately, even within the configured retry budget. On the 53-node sprint topology the DV/NLSR startup race produces a burst of prefix resets; the first metadata Interest from a scenario transition can arrive at a forwarder whose FIB has not yet been re-populated for the producer's /test sub-prefix, triggering an immediate Nack. * ExpressR: treat InterestResultNack with NackReasonNoRoute or NackReasonCongestion as retryable, alongside the existing Timeout retry. The retry budget (Retries) is now respected for both transient timeouts and transient Nacks; non-retryable Nacks (e.g. Duplicate) still pass through immediately. * rrSegFetcher.handleResult: when a segment Interest is Nacked with NackReasonNoRoute, treat it as a loss so the AIMD window shrinks and the segment is retried, instead of aborting the fetch. Verified locally: all three scenarios (NDNd, NFD, NDNd replay) pass on the 53-node sprint topology. --- std/object/client_consume_seg.go | 7 +++++++ std/object/expressr.go | 36 +++++++++++++++++++++----------- 2 files changed, 31 insertions(+), 12 deletions(-) diff --git a/std/object/client_consume_seg.go b/std/object/client_consume_seg.go index 4febcb50..651191ca 100644 --- a/std/object/client_consume_seg.go +++ b/std/object/client_consume_seg.go @@ -269,6 +269,13 @@ func (s *rrSegFetcher) handleResult(args ndn.ExpressCallbackArgs, state *Consume // congestion signal s.window.HandleSignal(cong.SigCongest) s.enqueueForRetransmission(state, seg, retries-1) + case spec.NackReasonNoRoute: + // Transient: route not yet learned (e.g. DV startup race). + // Treat as a loss so the AIMD window shrinks; the segment + // will be retried until the retry budget is exhausted or a + // route arrives. + s.window.HandleSignal(cong.SigLoss) + s.enqueueForRetransmission(state, seg, retries-1) default: // treat as irrecoverable error for now state.finalizeError(fmt.Errorf("%w: fetch seg failed with result: %s", ndn.ErrNetwork, args.Result)) diff --git a/std/object/expressr.go b/std/object/expressr.go index 10e56b59..5a4f284a 100644 --- a/std/object/expressr.go +++ b/std/object/expressr.go @@ -60,24 +60,36 @@ func ExpressR(engine ndn.Engine, args ndn.ExpressRArgs) { // Send the interest // TODO: reexpress faster than lifetime err = engine.Express(interest, func(res ndn.ExpressCallbackArgs) { - if res.Result == ndn.InterestResultTimeout { + // Retryable transient results: timeout and the Nacks that signal + // "try again later" (no FIB entry yet, or downstream congestion). + // Routing convergence in a large network can outlast a single + // Interest lifetime, especially when DV/NLSR startup produces a + // burst of prefix resets; without retry the first Interest after a + // scenario transition races ahead of route propagation and the + // caller observes an immediate failure. + retryable := false + switch res.Result { + case ndn.InterestResultTimeout: log.Debug(nil, "ExpressR Interest timeout", "name", args.Name) - - // Check if retries are exhausted - if args.Retries == 0 { - args.Callback(res) - return + retryable = true + case ndn.InterestResultNack: + switch res.NackReason { + case spec.NackReasonNoRoute, spec.NackReasonCongestion: + log.Debug(nil, "ExpressR retryable Nack", "name", args.Name, "reason", res.NackReason) + retryable = true } - - // Retry on timeout - args.Retries-- - ExpressR(engine, args) + } + if !retryable { + args.Callback(res) return - } else { - // All other results / errors are final + } + + if args.Retries == 0 { args.Callback(res) return } + args.Retries-- + ExpressR(engine, args) }) if err != nil { finalizeError(err) From 1872c64ebcdefdc5c45671d4f09568cb1a7946ae Mon Sep 17 00:00:00 2001 From: Taranum Wasu Date: Fri, 24 Jul 2026 19:16:50 +0530 Subject: [PATCH 12/17] docs(svs-v4): address spec review comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Rename "inline" to "direct" and "out-of-band" to "referenced" for the two StateVector delivery forms (more descriptive, per a-thieme). - PARTIAL vector layout: replace "[0]" / "[1...n]" notation with prose ("the first entry is the sender's own entry"). - Clarify single-member mhash in §5.7 and §7.4: the membership set is {N} for a joining node. - Dedupe the mhash definition: the §1.2 paragraph now points to §3.3 instead of restating SHA-256 digest. - Drop the "ndnd" qualifier on segmentation; the spec is forwarder-agnostic. - §6.2: clarify that "A is outdated" applies when A is FULL, and explain why PARTIAL receivers cannot use the omitted-name test. Source: PR #190 review comments addressed: 3612873010, 3612971226, 3612986026, 3641436856, 3641470105, 3641509945 --- docs/svs-v4.md | 108 +++++++++++++++++++++++++------------------------ 1 file changed, 56 insertions(+), 52 deletions(-) diff --git a/docs/svs-v4.md b/docs/svs-v4.md index 31c2d69f..4d50f409 100644 --- a/docs/svs-v4.md +++ b/docs/svs-v4.md @@ -1,7 +1,7 @@ # State Vector Sync (SVS) v4 Specification SVS v4 is a state-vector synchronization protocol for large sync groups. -It introduces a membership hash (`mhash`), two inline state-vector +It introduces a membership hash (`mhash`), two direct state-vector encodings (`FULL` and `PARTIAL`), and a third publish-only form that references a retrievable full vector. Every Sync Data carries `mhash` and a `VectorType`. @@ -24,19 +24,18 @@ modes: | Mode | Trigger | Wire shape | |------|---------|------------| -| **Inline FULL** | Encoded FULL fits in threshold | `mhash` + `VectorType=FULL` + complete `StateVector` in Sync Data | -| **Inline PARTIAL** | New publication and FULL exceeds threshold | `mhash` + `VectorType=PARTIAL` + subset `StateVector` in Sync Data | -| **Out-of-band FULL** | Periodic sync (large group), or `mhash` mismatch | Produce full vector Data at `32=sv/`; Sync Data carries `mhash` + reference Name only | +| **Direct FULL** | Encoded FULL fits in threshold | `mhash` + `VectorType=FULL` + complete `StateVector` in Sync Data | +| **Direct PARTIAL** | New publication and FULL exceeds threshold | `mhash` + `VectorType=PARTIAL` + subset `StateVector` in Sync Data | +| **Referenced FULL** | Periodic sync (large group), or `mhash` mismatch | Produce full vector Data at `32=sv/`; Sync Data carries `mhash` + reference Name only | -**MemberSetHash (`mhash`)** is the SHA-256 digest of the membership -described in §4.2. +`mhash` is defined in §3.3. **Full state recovery** uses publish + pull when: 1. `mhash` differs from the local membership hash, or 2. Periodic sync runs while the local FULL encoding exceeds `SyncVectorThreshold`, or -3. An inline `VectorType = FULL` State Vector is outdated per §6.2. +3. A direct `VectorType = FULL` State Vector is outdated per §6.2. Retrievable full-vector Data uses the standard NDN segmentation convention when it exceeds a single packet. @@ -70,7 +69,7 @@ Interest nonce is carried in Interest packet fields, not as a name component. - **`version`:** microsecond timestamp. No hash suffix is used. -**Sync Data Content:** encoded `SvsData` (§3) — either inline form (FULL +**Sync Data Content:** encoded `SvsData` (§3) — either direct form (FULL or PARTIAL) or publish-only form. ### 2.3 Application publication Data @@ -92,20 +91,20 @@ Retrievable full State Vector objects use a dedicated sync namespace: ////32=sv/ ``` -**Content:** signed `SvsData` in inline FULL form: `mhash` + +**Content:** signed `SvsData` in direct FULL form: `mhash` + `VectorType = FULL` + complete `StateVector`. **Publish + pull procedure** (periodic sync, `mhash` recovery, join when FULL exceeds threshold): 1. Produce the full-vector Data at - `////32=sv/` (ndnd segmentation handles - large content). + `////32=sv/`. The data is segmented + per the standard NDN convention if it does not fit in a single packet. 2. Send a Sync Interest whose AppParam Sync Data contains publish-only `SvsData`: `mhash` + `SvsDataRef` pointing at the published name (§3.1). 3. Receivers pull the referenced Data, validate, and merge. -A Sync message carries either an inline StateVector or a publish-only +A Sync message carries either a direct StateVector or a publish-only reference — not both. --- @@ -114,15 +113,15 @@ reference — not both. ### 3.1 `SvsData` -`SvsData` has two forms: inline (FULL or PARTIAL) and publish-only. The -`mhash` field is present in both forms. The inline form carries +`SvsData` has two forms: direct (FULL or PARTIAL) and publish-only. The +`mhash` field is present in both forms. The direct form carries `VectorType`; the publish-only form does not. -#### 3.1.1 Inline form (FULL or PARTIAL) +#### 3.1.1 Direct form (FULL or PARTIAL) Used when the State Vector (full or a publication-time PARTIAL subset) is -carried inline in Sync Data, or in published full-vector Data at -`32=sv/`. +carried in the Sync Data packet itself, or in published full-vector Data +at `32=sv/`. ``` SvsData = SVS-DATA-TYPE TLV-LENGTH @@ -153,7 +152,7 @@ SvsData = SVS-DATA-TYPE TLV-LENGTH | `MemberSetHash` | `0xCB` | 32-byte SHA-256 digest (`mhash`) | | `SvsDataRef` | `0x07` (Name) | Name of the published full-vector Data. The receiver strips the trailing version component and uses the resulting `32=sv` prefix as the trust anchor for that sender's retrievable full vectors. | -The inline layout puts `MemberSetHash` and `VectorType` before +The direct layout puts `MemberSetHash` and `VectorType` before `StateVector` (`mhash` at `0xCB`, vector at `0xC9`/`0xCA`). ### 3.2 `StateVector` @@ -215,7 +214,7 @@ The full State Vector carries membership implicitly: every member's summarizes that membership for quick comparison without having to walk the full State Vector. -### 3.4 `VectorType` (inline form) +### 3.4 `VectorType` (direct form) | Value | Name | Meaning | |-------|------|---------| @@ -229,7 +228,7 @@ sender guarantee the receiver knows whether missing names imply partition cannot convey this — two parties with identical membership but different subscription views may legitimately disagree on what subset was sent. -`mhash` is present in both inline and publish-only `SvsData` messages. +`mhash` is present in both direct and publish-only `SvsData` messages. --- @@ -244,15 +243,17 @@ subscription views may legitimately disagree on what subset was sent. ### 4.2 PARTIAL State Vector Used on new publication when -`encoded_size(inline FULL SvsData) > SyncVectorThreshold`. +`encoded_size(direct FULL SvsData) > SyncVectorThreshold`. - `VectorType = PARTIAL` (§3.4). -- **Entry `[0]`** is the sender's own `StateVectorEntry`. -- **Entries `[1…n]`** are in NDN canonical order among included peers. +- The first entry is the sender's own `StateVectorEntry`; the sender is + always included. +- The remaining entries are the sender's selected peers, ordered in NDN + canonical name order. If the sender-only baseline already exceeds `SyncVectorThreshold`, the sender falls back to publish + pull rather than emit a PARTIAL vector that -omits the required entry `[0]`. +omits the sender's own entry. An implementation MAY use the following selection priority: @@ -264,15 +265,15 @@ An implementation MAY use the following selection priority: | 4 | Random inactive producers | | 5 | Others by recency | -Stop adding entries when the estimated inline `SvsData` size approaches +Stop adding entries when the estimated direct `SvsData` size approaches `SyncVectorThreshold`. ### 4.3 `SyncVectorThreshold` `SyncVectorThreshold` is a fixed library constant (1200 bytes) that bounds -the size of an inline SvsData: +the size of a direct SvsData: -- When `encoded_size(FULL) ≤ SyncVectorThreshold`, nodes use inline FULL +- When `encoded_size(FULL) ≤ SyncVectorThreshold`, nodes use direct FULL (with `mhash` and `VectorType=FULL`). - When `encoded_size(FULL) > SyncVectorThreshold`, nodes switch to PARTIAL (publication) or publish + pull (periodic sync and recovery). @@ -300,16 +301,16 @@ Interest and resets the timer to `PeriodicTimeout`. | Trigger | Action | |---------|--------| -| `encoded_size(inline FULL) ≤ SyncVectorThreshold` | Send inline FULL (`mhash` + `VectorType=FULL` + `StateVector`) | -| `encoded_size(inline FULL) > SyncVectorThreshold` | Send inline PARTIAL (`mhash` + `VectorType=PARTIAL` + subset `StateVector`), or publish + pull if the sender-only baseline itself exceeds the threshold | +| `encoded_size(direct FULL) ≤ SyncVectorThreshold` | Send direct FULL (`mhash` + `VectorType=FULL` + `StateVector`) | +| `encoded_size(direct FULL) > SyncVectorThreshold` | Send direct PARTIAL (`mhash` + `VectorType=PARTIAL` + subset `StateVector`), or publish + pull if the sender-only baseline itself exceeds the threshold | ### 5.3 Sync Ack policy Sync Interests are unacknowledged. -### 5.4 Steady state and suppression (inline FULL) +### 5.4 Steady state and suppression (direct FULL) -For incoming Sync Data with inline `VectorType = FULL`, apply the +For incoming Sync Data with direct `VectorType = FULL`, apply the steady-state and suppression rules in §5.1–§5.4. ### 5.5 PARTIAL State Vector processing @@ -326,7 +327,7 @@ When `VectorType = PARTIAL`: (§5.6). PARTIAL processing is the only receive-side change relative to the -inline-FULL path. +direct-FULL path. ### 5.6 Full state recovery (publish + pull) @@ -335,7 +336,7 @@ inline-FULL path. | # | Trigger | Action | |---|---------|--------| | 1 | `mhash` in received `SvsData` ≠ locally computed `mhash` | Publish + pull | -| 2 | Inline `VectorType = FULL` is outdated per §6.2 | Merge inline if complete; otherwise publish + pull | +| 2 | Direct `VectorType = FULL` is outdated per §6.2 | Merge direct if complete; otherwise publish + pull | | 3 | Periodic sync while local FULL exceeds `SyncVectorThreshold` | Publish + pull (§5.8) | Recovery always fetches the complete State Vector from the referenced @@ -344,15 +345,15 @@ Recovery always fetches the complete State Vector from the referenced **Sender procedure** (on `mhash` mismatch or periodic large-group sync): 1. Produce full-vector Data at `////32=sv/` - with inline FULL `SvsData`. + with direct FULL `SvsData`. 2. Send Sync Interest with publish-only `SvsData` (`mhash` + `SvsDataRef`). **Receiver procedure:** 1. Identify the sender from the Sync Data signature, or — when the Sync - Data is PARTIAL — from PARTIAL entry `[0]`, which is the sender's own + Data is PARTIAL — from the first entry, which is the sender's own entry per §4.2. -2. If the Sync Data is inline FULL and complete: merge directly. +2. If the Sync Data is direct FULL and complete: merge directly. 3. If the Sync Data is publish-only: read `SvsDataRef`; express Interest for that name; validate; merge; update local `mhash`. 4. Continue application data fetch via SvsALO (`OnUpdate`) as today. @@ -364,17 +365,18 @@ Recovery always fetches the complete State Vector from the referenced > implementation detail and does not affect protocol correctness — a > debounced pull is equivalent to a slightly delayed pull. -Use ndnd segmentation when fetched Data content is large. +Fetched Data is segmented per the standard NDN convention when it does +not fit in a single packet. ### 5.7 New node join -1. Joining node **N** multicasts Sync Interest whose inline State Vector +1. Joining node **N** multicasts Sync Interest whose direct State Vector contains only itself: `(Name=N, SeqNo=0)`. The Sync Data's `mhash` is - the SHA-256 of N's single-member membership list. + the SHA-256 of N's membership set, which is a single member (N). 2. Existing members receive the announcement. 3. Suppression limits duplicate responses; typically one member **A** provides recovery state. -4. If FULL fits inline: **A** responds with inline `VectorType = FULL`. +4. If FULL fits in a direct packet: **A** responds with direct `VectorType = FULL`. 5. If FULL exceeds `SyncVectorThreshold`: **A** uses publish + pull (produce at `32=sv/`, then publish-only Sync Data). 6. Normal synchronization proceeds through SvsALO. @@ -383,17 +385,17 @@ Use ndnd segmentation when fetched Data content is large. | Local FULL size | Periodic Sync behavior | |-----------------|------------------------| -| `≤ SyncVectorThreshold` | Inline FULL | +| `≤ SyncVectorThreshold` | Direct FULL | | `> SyncVectorThreshold` | Publish + pull (produce full-vector Data, then publish-only Sync Data) | -Periodic sync does not send inline PARTIAL vectors. +Periodic sync does not send direct PARTIAL vectors. ### 5.9 Summary of sync triggers | Event | `size ≤ threshold` | `size > threshold` | |-------|--------------------|--------------------| -| **New publication** | Inline FULL | Inline PARTIAL (or publish + pull fallback) | -| **Periodic sync** | Inline FULL | Publish + pull | +| **New publication** | Direct FULL | Direct PARTIAL (or publish + pull fallback) | +| **Periodic sync** | Direct FULL | Publish + pull | | **`mhash` mismatch** | Publish + pull (if recovery needed) | Publish + pull | --- @@ -411,10 +413,11 @@ State Vector `A` is outdated to `B` if: - `A` is missing a name present in `B`, or - `A` has a strictly smaller `SeqNo` for any entry. -This rule applies to `VectorType = FULL`. For `VectorType = PARTIAL`, -omitted names are a subset by design (§4.2): the sender selected a -publication-time subset and `A`'s missing entries do not carry any -information about whether `A` is outdated relative to `B`. +This rule applies when `A` is a `VectorType = FULL` State Vector. When +`A` is `VectorType = PARTIAL`, `A`'s omitted names are a subset by +design (§4.2): the sender selected a publication-time subset and `A`'s +missing entries do not carry any information about whether `A` is +outdated relative to `B`. --- @@ -423,13 +426,13 @@ information about whether `A` is outdated relative to `B`. ### 7.1 Small group Three nodes `A`, `B`, `C`. Full State Vector fits. `A` publishes; sends -inline FULL Sync Interest `[A:11, B:15, C:25]`. Peers merge. +direct FULL Sync Interest `[A:11, B:15, C:25]`. Peers merge. ### 7.2 Large group Group exceeds `SyncVectorThreshold`. Producer `P` publishes: -- `P` sends inline PARTIAL `SvsData { mhash, VectorType=PARTIAL, +- `P` sends direct PARTIAL `SvsData { mhash, VectorType=PARTIAL, StateVector=[P:…, A:…, …] }`. - Receiver merges present entries only. - If `mhash` differs, `P` (or receiver per policy) triggers publish + pull @@ -444,8 +447,9 @@ Group exceeds `SyncVectorThreshold`. Producer `P` publishes: ### 7.4 New node join -- `N` sends self-only vector `[N:0]` with `mhash`. -- `A` responds with inline FULL or publish + pull. +- `N` sends a State Vector containing only itself (`[N:0]`) with `mhash` + computed over the single-member membership set `{N}`. +- `A` responds with direct FULL or publish + pull. - `N` merges and synchronizes via SvsALO. --- From 5f3a477ba08c32870890f8f0ecc24f8735d809a1 Mon Sep 17 00:00:00 2001 From: Taranum Wasu Date: Fri, 24 Jul 2026 20:58:48 +0530 Subject: [PATCH 13/17] sync: address v4 review comments on PARTIAL selection and SvsData flow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - encodePartialStateVector: return an empty StateVector (no entries) instead of nil when the sender-only baseline exceeds the size budget. The caller treats the empty vector as the publish+pull trigger. Doc updated; spec §4.2 explicitly allows the empty-PARTIAL signal. - buildSvsDataForSend: detect the empty result with len(...Entries) == 0. - partialCandidateNames -> priorityOrderedPeers, more descriptive for review and maintenance. - sortPartialTail: drop the redundant trailing sort in encodePartialStateVector — the inner-loop sort already produces a tail-sorted slice. - ComputeMembershipHash: rename the local `tuples` variable to `membershipTuples` to disambiguate from the spec tuple. - shouldUsePublishPull: now returns (usePublish, full, size). The build-the-full-SvsData-once pattern means the inline-full case no longer re-encodes for size check + send. - deriveFullVectorPrefix and resolveFullVectorPrefix: doc clarified to call out the helper as an implementation convention, not a spec constraint (the spec only requires a sender-controlled full-vector prefix; callers wire to a different location by setting opts.FullVectorPrefix). - buildPublishSvsData: doc added explaining what `ref` is (the sender's published full-vector Data name) and how it is consumed. Verified locally: all 3 e2e scenarios pass (NDNd + NFD + replay). Source: PR #190 review comments addressed: 3613120231, 3613133568, 3613158180, 3613167366, 3613250227, 3613262079, 3613275363 --- docs/svs-v4.md | 5 ++-- std/sync/svs.go | 10 ++++++-- std/sync/svs_encode.go | 41 +++++++++++++++++++++------------ std/sync/svs_membership_hash.go | 8 +++---- std/sync/svs_pull.go | 41 +++++++++++++++++++++++++-------- std/sync/svs_test.go | 18 ++++++++++----- 6 files changed, 84 insertions(+), 39 deletions(-) diff --git a/docs/svs-v4.md b/docs/svs-v4.md index 4d50f409..272b3516 100644 --- a/docs/svs-v4.md +++ b/docs/svs-v4.md @@ -252,8 +252,9 @@ Used on new publication when canonical name order. If the sender-only baseline already exceeds `SyncVectorThreshold`, the -sender falls back to publish + pull rather than emit a PARTIAL vector that -omits the sender's own entry. +sender falls back to publish + pull. The implementation MAY emit an empty +PARTIAL in this case as a signal to the caller; the caller MUST treat an +empty PARTIAL as the publish + pull trigger instead of forwarding it. An implementation MAY use the following selection priority: diff --git a/std/sync/svs.go b/std/sync/svs.go index 71c5d4f5..b3c6f3a3 100644 --- a/std/sync/svs.go +++ b/std/sync/svs.go @@ -518,14 +518,20 @@ func (s *SvSync) encodeSyncData(reason syncSendReason, sender enc.Name) enc.Wire s.mutex.Unlock() var svsData *spec_svs.SvsData - if shouldUsePublishPull(reason, syncVectorThreshold, stateSnap) { + usePublish, precomputedFull, _ := shouldUsePublishPull(reason, syncVectorThreshold, stateSnap) + switch { + case usePublish: ref, err := s.publishFullVectorData(stateSnap) if err != nil { log.Error(s, "publishFullVectorData failed", "err", err) return nil } svsData = buildPublishSvsData(stateSnap, ref) - } else { + case precomputedFull != nil: + // [Spec] Inline FULL fits the threshold; reuse the SvsData we + // already built during the size check instead of re-encoding. + svsData = precomputedFull + default: svsData = buildSvsDataForSend(svsSendInput{ State: stateSnap, Reason: reason, diff --git a/std/sync/svs_encode.go b/std/sync/svs_encode.go index 8c876796..835f0f8d 100644 --- a/std/sync/svs_encode.go +++ b/std/sync/svs_encode.go @@ -43,7 +43,8 @@ type svsSendInput struct { // buildSvsDataForSend picks embedded FULL or PARTIAL SvsData for an outgoing // Sync message. Returns nil when publication-triggered PARTIAL encoding cannot -// fit even the sender-only baseline: the caller MUST fall back to publish+pull +// fit even the sender-only baseline (signaled by an empty StateVector from +// encodePartialStateVector): the caller MUST fall back to publish+pull // (see shouldUsePublishPull). Other reasons always return a non-nil result. func buildSvsDataForSend(in svsSendInput) *spec_svs.SvsData { fullSv := in.State.Encode(func(seq uint64) uint64 { return seq }) @@ -64,7 +65,7 @@ func buildSvsDataForSend(in svsSendInput) *spec_svs.SvsData { Propagation: in.Propagation, Mtime: in.Mtime, }) - if partialSv == nil { + if len(partialSv.Entries) == 0 { // Baseline exceeded Threshold; caller must use publish+pull. return nil } @@ -78,9 +79,10 @@ func buildSvsDataForSend(in svsSendInput) *spec_svs.SvsData { // encodePartialStateVector builds a PARTIAL StateVector for new publication. // Entry [0] is the sender; entries [1..n] are in NDN canonical order. // -// Returns nil if the sender-only baseline itself exceeds Threshold: -// callers MUST fall back to publish+pull in that case, because including -// the sender entry is required by §4.2 of the v4 spec. +// Returns an empty StateVector (no entries) if the sender-only baseline +// itself exceeds Threshold: the caller MUST fall back to publish+pull in +// that case and the empty vector is the explicit signal that no PARTIAL +// body could be fit under the size budget. func encodePartialStateVector(state SvMap[uint64], opts PartialEncodeOpts) *spec_svs.StateVector { seq := func(v uint64) uint64 { return v } senderHash := opts.Sender.TlvStr() @@ -98,16 +100,17 @@ func encodePartialStateVector(state SvMap[uint64], opts PartialEncodeOpts) *spec StateVector: baseline, } if len(baselineData.Encode().Join()) > opts.Threshold { - // Caller falls back to publish+pull because we cannot satisfy - // the §4.2 "entry [0] is the sender" rule at this size budget. - return nil + // Baseline too large to fit even the sender entry: return an + // empty PARTIAL so the caller can detect the overflow and fall + // back to publish+pull. + return &spec_svs.StateVector{} } - candidates := partialCandidateNames(state, senderHash, opts) + peers := priorityOrderedPeers(state, senderHash, opts) included := map[string]bool{senderHash: true} entries := []*spec_svs.StateVectorEntry{senderEntry} - for _, name := range candidates { + for _, name := range peers { hash := name.TlvStr() if included[hash] { continue @@ -133,12 +136,17 @@ func encodePartialStateVector(state SvMap[uint64], opts PartialEncodeOpts) *spec included[hash] = true } - sortPartialTail(entries) + // entries is already sorted tail-wise: the loop's sortPartialTail(trial) + // runs on every accepted iteration, so the trailing sort is redundant. return &spec_svs.StateVector{Entries: entries} } -// partialCandidateNames returns the producer names considered for inclusion -// in a PARTIAL StateVector, in priority order: +// priorityOrderedPeers returns the producer names that consumers should +// consider for inclusion in a new-publication PARTIAL StateVector, ordered +// by inclusion priority. The caller iterates the returned slice and +// greedily adds entries until the size budget is reached. +// +// The priority order is: // // 1. Repair targets from the suppression-merge state (newest entries first). // 2. Propagation targets (the most recently updated producers). @@ -148,7 +156,7 @@ func encodePartialStateVector(state SvMap[uint64], opts PartialEncodeOpts) *spec // (b) canonical NDN name ascending. // // The sender is excluded — it is always included at entries[0]. -func partialCandidateNames(state SvMap[uint64], senderHash string, opts PartialEncodeOpts) []enc.Name { +func priorityOrderedPeers(state SvMap[uint64], senderHash string, opts PartialEncodeOpts) []enc.Name { seen := map[string]bool{senderHash: true} out := make([]enc.Name, 0, len(state)) @@ -222,7 +230,10 @@ func recencyScore(mtime map[string]time.Time, name enc.Name) int64 { return t.UnixNano() } -// sortPartialTail keeps entry [0] fixed and sorts [1..n] in canonical name order. +// sortPartialTail keeps the first entry fixed (the sender) and sorts the +// remaining entries in canonical NDN name order. PARTIAL vectors place the +// sender at entries[0] and the rest must be canonically ordered for byte- +// deterministic encoding. func sortPartialTail(entries []*spec_svs.StateVectorEntry) { if len(entries) <= 1 { return diff --git a/std/sync/svs_membership_hash.go b/std/sync/svs_membership_hash.go index 908729a9..86520e62 100644 --- a/std/sync/svs_membership_hash.go +++ b/std/sync/svs_membership_hash.go @@ -11,17 +11,17 @@ import ( // Each tuple is encoded as a TLV structure (Tuple-T 0xcc with Name and BootstrapTime // children) using the ndnd standard TLV codec. func ComputeMembershipHash(state SvMap[uint64]) []byte { - tuples := make([]*spec_svs.MembershipTuple, 0) + membershipTuples := make([]*spec_svs.MembershipTuple, 0) for name, vals := range state.Iter() { for _, val := range vals { - tuples = append(tuples, &spec_svs.MembershipTuple{ + membershipTuples = append(membershipTuples, &spec_svs.MembershipTuple{ Name: name, BootstrapTime: val.Boot, }) } } - slices.SortFunc(tuples, func(a, b *spec_svs.MembershipTuple) int { + slices.SortFunc(membershipTuples, func(a, b *spec_svs.MembershipTuple) int { if c := a.Name.Compare(b.Name); c != 0 { return c } @@ -35,7 +35,7 @@ func ComputeMembershipHash(state SvMap[uint64]) []byte { }) h := sha256.New() - for _, t := range tuples { + for _, t := range membershipTuples { h.Write(t.Encode().Join()) } return h.Sum(nil) diff --git a/std/sync/svs_pull.go b/std/sync/svs_pull.go index f4ad2538..69d717f3 100644 --- a/std/sync/svs_pull.go +++ b/std/sync/svs_pull.go @@ -18,7 +18,13 @@ const ( fullVectorKeyword = "sv" ) -// deriveFullVectorPrefix maps SyncDataName (.../32=svs) to the published full-vector prefix (.../32=sv). +// deriveFullVectorPrefix is an implementation convenience that maps a +// SyncDataName (.../32=svs) to the published full-vector prefix +// (.../32=sv) by replacing the trailing keyword. The spec only requires +// that a full-vector prefix exists at some sender-controlled location; this +// helper just provides a default when the caller does not set one. +// Callers that wire to a different prefix MUST supply FullVectorPrefix +// explicitly via NewSvSync opts. func deriveFullVectorPrefix(syncDataName enc.Name) enc.Name { if len(syncDataName) == 0 { return nil @@ -31,7 +37,9 @@ func deriveFullVectorPrefix(syncDataName enc.Name) enc.Name { } // resolveFullVectorPrefix returns the explicit FullVectorPrefix if set, -// otherwise derives it from SyncDataName. +// otherwise derives it from SyncDataName. Per the spec, the full-vector +// prefix is whatever location the producer publishes at; this helper picks +// the default location documented above. func resolveFullVectorPrefix(explicit, syncDataName enc.Name) enc.Name { if len(explicit) > 0 { return explicit.Clone() @@ -60,6 +68,13 @@ func pullRefFromSyncDataWire(dataWire enc.Wire) enc.Name { return deriveFullVectorPrefix(name) } +// buildPublishSvsData constructs the publish-only form of SvsData carried +// in a Sync message (per spec §3.1.2): the membership hash for the current +// state plus a reference (SvsDataRef) to the sender's retrievable full +// vector. The ref is the published full-vector Data name (e.g. +// ////32=sv/); receivers fetch it via +// pullFullVector, validate, and merge. The ref must sit below the +// sender's trust prefix (see isTrustedSvsDataRef). func buildPublishSvsData(state SvMap[uint64], ref enc.Name) *spec_svs.SvsData { return &spec_svs.SvsData{ MemberSetHash: ComputeMembershipHash(state), @@ -69,13 +84,15 @@ func buildPublishSvsData(state SvMap[uint64], ref enc.Name) *spec_svs.SvsData { // shouldUsePublishPull reports whether the sender should publish at .../32=sv // and emit publish-only Sync Data (mhash + SvsDataRef, no embedded vector) -// instead of an embedded FULL or PARTIAL StateVector. -func shouldUsePublishPull(reason syncSendReason, threshold int, state SvMap[uint64]) bool { - if reason == syncSendRecovery { - return true - } - if reason == syncSendPublication { - return false +// instead of an embedded FULL or PARTIAL StateVector. The full SvsData is +// built once (so the caller can reuse it if not publishing) and its size is +// returned alongside the decision so the caller does not have to re-encode. +func shouldUsePublishPull(reason syncSendReason, threshold int, state SvMap[uint64]) (usePublish bool, data *spec_svs.SvsData, size int) { + switch reason { + case syncSendRecovery: + return true, nil, 0 + case syncSendPublication: + return false, nil, 0 } sv := state.Encode(func(seq uint64) uint64 { return seq }) full := &spec_svs.SvsData{ @@ -83,7 +100,11 @@ func shouldUsePublishPull(reason syncSendReason, threshold int, state SvMap[uint VectorType: optional.Some(spec_svs.VectorTypeFull), StateVector: sv, } - return len(full.Encode().Join()) > threshold + wire := full.Encode().Join() + if len(wire) > threshold { + return true, nil, len(wire) + } + return false, full, len(wire) } // publishFullVectorData produces retrievable inline FULL SvsData at .../32=sv/. diff --git a/std/sync/svs_test.go b/std/sync/svs_test.go index 83862833..b33c40e4 100644 --- a/std/sync/svs_test.go +++ b/std/sync/svs_test.go @@ -373,11 +373,16 @@ func TestShouldUsePublishPull(t *testing.T) { StateVector: sv, }).Encode().Join()) - require.False(t, shouldUsePublishPull(syncSendPublication, fullSize-1, m)) - require.False(t, shouldUsePublishPull(syncSendPeriodic, fullSize+1, m)) - require.True(t, shouldUsePublishPull(syncSendPeriodic, fullSize-1, m)) - require.True(t, shouldUsePublishPull(syncSendOther, fullSize-1, m)) - require.True(t, shouldUsePublishPull(syncSendRecovery, fullSize-1, m)) + usePublish, _, _ := shouldUsePublishPull(syncSendPublication, fullSize-1, m) + require.False(t, usePublish) + usePublish, _, _ = shouldUsePublishPull(syncSendPeriodic, fullSize+1, m) + require.False(t, usePublish) + usePublish, _, _ = shouldUsePublishPull(syncSendPeriodic, fullSize-1, m) + require.True(t, usePublish) + usePublish, _, _ = shouldUsePublishPull(syncSendOther, fullSize-1, m) + require.True(t, usePublish) + usePublish, _, _ = shouldUsePublishPull(syncSendRecovery, fullSize-1, m) + require.True(t, usePublish) } func TestIsTrustedSvsDataRef(t *testing.T) { @@ -500,7 +505,8 @@ func TestEncodeSyncDataPublishMode(t *testing.T) { VectorType: optional.Some(spec_svs.VectorTypeFull), StateVector: m.Encode(func(s uint64) uint64 { return s }), }).Encode().Join()) - require.True(t, shouldUsePublishPull(syncSendPeriodic, fullSize-1, m)) + usePublish, _, _ := shouldUsePublishPull(syncSendPeriodic, fullSize-1, m) + require.True(t, usePublish) publish := buildPublishSvsData(m, tu.NoErr(enc.NameFromStr("/ndn/svs/alice/1/32=sv/2"))) require.Nil(t, publish.StateVector) From ea794832e58c559d7124904ed1869aeb61aee1fe Mon Sep 17 00:00:00 2001 From: Taranum Wasu Date: Fri, 24 Jul 2026 22:16:19 +0530 Subject: [PATCH 14/17] =?UTF-8?q?sync:=20replace=20VectorType=20with=20dis?= =?UTF-8?q?tinct=20FullStateVector/PartialStateVector=20TLVs=20Per=20Adam'?= =?UTF-8?q?s=20review=20note=20in=20the=20v4=20spec,=20the=20VectorType=20?= =?UTF-8?q?discriminator=20field=20is=20removed=20from=20SvsData.=20The=20?= =?UTF-8?q?wire=20TLV=20itself=20now=20disambiguates=20between=20direct=20?= =?UTF-8?q?FULL=20(0xCD),=20direct=20PARTIAL=20(0xCE),=20and=20publish-onl?= =?UTF-8?q?y=20SvsDataRef=20(0x07).=20MemberSetHash=20moves=20to=20the=20t?= =?UTF-8?q?op-level=20SvsData=20TLV=20so=20all=20three=20forms=20carry=20i?= =?UTF-8?q?t=20in=20the=20same=20place.=20The=20StateVector=20type=20itsel?= =?UTF-8?q?f=20is=20unchanged=20and=20remains=20used=20directly=20by=20DV'?= =?UTF-8?q?s=20advertisement=20(which=20has=20its=20own=20simpler=20wire?= =?UTF-8?q?=20shape).=20-=20definitions.go:=20introduce=20FullStateVector?= =?UTF-8?q?=20and=20PartialStateVector=20=20=20structs,=20drop=20VectorTyp?= =?UTF-8?q?e=20field=20and=20constants.=20-=20accessors.go=20(new):=20help?= =?UTF-8?q?ers=20Kind=20/=20IsFull=20/=20IsPartial=20/=20=20=20GetStateVec?= =?UTF-8?q?tor=20so=20callers=20don't=20have=20to=20switch=20on=20which=20?= =?UTF-8?q?struct=20=20=20was=20non-nil.=20-=20svs=5Fpull.go,=20svs=5Fenco?= =?UTF-8?q?de.go:=20build=20FULL/PARTIAL=20SvsData=20with=20the=20=20=20ne?= =?UTF-8?q?w=20struct=20types;=20mhash=20on=20SvsData,=20vector=20on=20inn?= =?UTF-8?q?er=20struct.=20-=20svs.go:=20svSyncRecvSvArgs=20now=20carries?= =?UTF-8?q?=20partial=20bool;=20onSyncData=20=20=20routes=20by=20IsFull/Is?= =?UTF-8?q?Partial=20and=20rejects=20malformed=20direct=20forms.=20-=20svs?= =?UTF-8?q?=5Ftest.go:=20all=20tests=20rewritten=20against=20the=20new=20c?= =?UTF-8?q?onstructors=20=20=20and=20accessors;=20TestSvsDataLegacyParse?= =?UTF-8?q?=20removed.=20-=20dv/dv/advert=5Fsync.go:=20use=20spec=5Fsvs.St?= =?UTF-8?q?ateVector=20directly=20(DV's=20=20=20wire=20shape=20was=20alway?= =?UTF-8?q?s=20a=20single-entry=20StateVector).=20-=20docs/svs-v4.md:=20co?= =?UTF-8?q?llapse=20=C2=A73.1,=20add=20new=20TLVs=20to=20the=20table=20in?= =?UTF-8?q?=20=C2=A73.2,=20=20=20replace=20=C2=A73.4=20VectorType=20sectio?= =?UTF-8?q?n=20with=20FullStateVector=20vs=20=20=20PartialStateVector,=20u?= =?UTF-8?q?pdate=20=C2=A74=20/=20=C2=A75=20/=20=C2=A76=20/=20=C2=A77=20/?= =?UTF-8?q?=20=C2=A78=20references.=20Tests:=20std/sync=20passes.=20ndnd?= =?UTF-8?q?=20build=20unchanged=20aside=20from=20new=20SVS=20v4=20wire=20f?= =?UTF-8?q?ormat.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/svs-v4.md | 161 +++++++------- dv/dv/advert_sync.go | 27 +-- std/ndn/svs/v3/accessors.go | 54 +++++ std/ndn/svs/v3/definitions.go | 35 ++- std/ndn/svs/v3/zz_generated.go | 380 +++++++++++++++++++++++++++------ std/sync/svs.go | 62 +++--- std/sync/svs_encode.go | 21 +- std/sync/svs_pull.go | 49 ++--- std/sync/svs_test.go | 151 +++++++------ 9 files changed, 629 insertions(+), 311 deletions(-) create mode 100644 std/ndn/svs/v3/accessors.go diff --git a/docs/svs-v4.md b/docs/svs-v4.md index 272b3516..c9942724 100644 --- a/docs/svs-v4.md +++ b/docs/svs-v4.md @@ -2,9 +2,10 @@ SVS v4 is a state-vector synchronization protocol for large sync groups. It introduces a membership hash (`mhash`), two direct state-vector -encodings (`FULL` and `PARTIAL`), and a third publish-only form that -references a retrievable full vector. Every Sync Data carries `mhash` and -a `VectorType`. +encodings (`FULL` and `PARTIAL`) carried as distinct TLVs, and a third +publish-only form that references a retrievable full vector. Every Sync +Data carries `mhash` and one of `FullStateVector`, `PartialStateVector`, +or `SvsDataRef`. --- @@ -24,9 +25,9 @@ modes: | Mode | Trigger | Wire shape | |------|---------|------------| -| **Direct FULL** | Encoded FULL fits in threshold | `mhash` + `VectorType=FULL` + complete `StateVector` in Sync Data | -| **Direct PARTIAL** | New publication and FULL exceeds threshold | `mhash` + `VectorType=PARTIAL` + subset `StateVector` in Sync Data | -| **Referenced FULL** | Periodic sync (large group), or `mhash` mismatch | Produce full vector Data at `32=sv/`; Sync Data carries `mhash` + reference Name only | +| **Direct FULL** | Encoded FULL fits in threshold | `FullStateVector` (`mhash` + complete `StateVector`) in Sync Data | +| **Direct PARTIAL** | New publication and FULL exceeds threshold | `PartialStateVector` (`mhash` + subset `StateVector`) in Sync Data | +| **Referenced FULL** | Periodic sync (large group), or `mhash` mismatch | Produce full vector Data at `32=sv/`; Sync Data carries only `SvsDataRef` | `mhash` is defined in §3.3. @@ -35,7 +36,7 @@ modes: 1. `mhash` differs from the local membership hash, or 2. Periodic sync runs while the local FULL encoding exceeds `SyncVectorThreshold`, or -3. A direct `VectorType = FULL` State Vector is outdated per §6.2. +3. A direct `FullStateVector` is outdated per §6.2. Retrievable full-vector Data uses the standard NDN segmentation convention when it exceeds a single packet. @@ -91,8 +92,8 @@ Retrievable full State Vector objects use a dedicated sync namespace: ////32=sv/ ``` -**Content:** signed `SvsData` in direct FULL form: `mhash` + -`VectorType = FULL` + complete `StateVector`. +**Content:** signed `SvsData` in direct FULL form: `FullStateVector` +containing `mhash` and complete `StateVector`. **Publish + pull procedure** (periodic sync, `mhash` recovery, join when FULL exceeds threshold): @@ -113,47 +114,38 @@ reference — not both. ### 3.1 `SvsData` -`SvsData` has two forms: direct (FULL or PARTIAL) and publish-only. The -`mhash` field is present in both forms. The direct form carries -`VectorType`; the publish-only form does not. +`SvsData` is a tagged union. The wire carries: -#### 3.1.1 Direct form (FULL or PARTIAL) - -Used when the State Vector (full or a publication-time PARTIAL subset) is -carried in the Sync Data packet itself, or in published full-vector Data -at `32=sv/`. +- `MemberSetHash` (`0xCB`): 32-byte `mhash`, present in all three forms. +- One of the following three top-level TLVs (the choice replaces the + previous `VectorType` discriminator): + - `FullStateVector` (`0xCD`): direct form with a complete State Vector. + - `PartialStateVector` (`0xCE`): direct form with a publication-time + subset. + - `SvsDataRef` (`0x07`): publish-only form pointing to a retrievable + full-vector Data. ``` SvsData = SVS-DATA-TYPE TLV-LENGTH - MemberSetHash - VectorType - StateVector -``` - -| Field | TLV type | Value | -|-------|----------|-------| -| `MemberSetHash` | `0xCB` | 32-byte SHA-256 digest (`mhash`) | -| `VectorType` | `0xCD` | `0` = FULL, `1` = PARTIAL | -| `StateVector` | `0xC9` | See §3.2 | - -#### 3.1.2 Publish-only form - -Used when Sync Data advertises a retrievable full-vector Data name (periodic -sync, `mhash` recovery). `VectorType` and `StateVector` are absent. + MemberSetHash ; always present + ( FullStateVector + | PartialStateVector + | SvsDataRef ) ; exactly one -``` -SvsData = SVS-DATA-TYPE TLV-LENGTH - MemberSetHash - SvsDataRef +FullStateVector = FULL-STATE-VECTOR-TYPE TLV-LENGTH StateVector +PartialStateVector = PARTIAL-STATE-VECTOR-TYPE TLV-LENGTH StateVector ``` | Field | TLV type | Value | |-------|----------|-------| | `MemberSetHash` | `0xCB` | 32-byte SHA-256 digest (`mhash`) | +| `FullStateVector` | `0xCD` | Complete `StateVector` (§3.2) | +| `PartialStateVector` | `0xCE` | Subset `StateVector` (§3.2) | | `SvsDataRef` | `0x07` (Name) | Name of the published full-vector Data. The receiver strips the trailing version component and uses the resulting `32=sv` prefix as the trust anchor for that sender's retrievable full vectors. | -The direct layout puts `MemberSetHash` and `VectorType` before -`StateVector` (`mhash` at `0xCB`, vector at `0xC9`/`0xCA`). +The receiver MUST reject Sync Data that carries both `FullStateVector` +and `PartialStateVector`, and Sync Data that carries none of the three. +`mhash` MUST be exactly 32 bytes when present. ### 3.2 `StateVector` @@ -172,6 +164,8 @@ SeqNoEntry = SEQ-NO-ENTRY-TYPE TLV-LENGTH | TLV | Type (decimal) | Type (hex) | |-----|----------------|------------| +| `FULL-STATE-VECTOR-TYPE` | 205 | `0xCD` | +| `PARTIAL-STATE-VECTOR-TYPE` | 206 | `0xCE` | | `STATE-VECTOR-TYPE` | 201 | `0xC9` | | `STATE-VECTOR-ENTRY-TYPE` | 202 | `0xCA` | | `SEQ-NO-ENTRY-TYPE` | 210 | `0xD2` | @@ -214,21 +208,26 @@ The full State Vector carries membership implicitly: every member's summarizes that membership for quick comparison without having to walk the full State Vector. -### 3.4 `VectorType` (direct form) +### 3.4 `FullStateVector` vs. `PartialStateVector` + +The wire TLV itself disambiguates the direct form: -| Value | Name | Meaning | -|-------|------|---------| -| `0` | **FULL** | `StateVector` contains the complete advertised state (§4.1 ordering). | -| `1` | **PARTIAL** | `StateVector` contains a subset (§4.2). Used for new publication only when FULL exceeds threshold. | +| TLV | Name | Meaning | +|-----|------|---------| +| `0xCD` | **FULL** | `StateVector` contains the complete advertised state (§4.1 ordering). | +| `0xCE` | **PARTIAL** | `StateVector` contains a subset (§4.2). Used for new publication only when FULL exceeds threshold. | -`VectorType` is required on the wire because it lets a receiver skip the -more expensive subset-evaluation code path when it sees `FULL`, and lets a -sender guarantee the receiver knows whether missing names imply partition -(FULL) or merely "not included in this subset" (PARTIAL). `mhash` alone -cannot convey this — two parties with identical membership but different -subscription views may legitimately disagree on what subset was sent. +Distinct TLVs (instead of a shared field with a discriminator) let a +receiver skip the more expensive subset-evaluation code path when it sees +`FullStateVector`, and let a sender guarantee the receiver knows whether +missing names imply partition (FULL) or merely "not included in this +subset" (PARTIAL). `mhash` alone cannot convey this — two parties with +identical membership but different subscription views may legitimately +disagree on what subset was sent. -`mhash` is present in both direct and publish-only `SvsData` messages. +The publish-only form (`SvsDataRef` only, no embedded `StateVector`) +carries neither TLV; the receiver treats it as a signal to fetch the +referenced full vector. --- @@ -238,14 +237,14 @@ subscription views may legitimately disagree on what subset was sent. - Include all known members and their latest sequence numbers per bootstrap. - Entries ordered in NDN canonical order of `Name`. -- `VectorType = FULL` (§3.4). +- Wire TLV is `FullStateVector` (`0xCD`, §3.4). ### 4.2 PARTIAL State Vector Used on new publication when `encoded_size(direct FULL SvsData) > SyncVectorThreshold`. -- `VectorType = PARTIAL` (§3.4). +- Wire TLV is `PartialStateVector` (`0xCE`, §3.4). - The first entry is the sender's own `StateVectorEntry`; the sender is always included. - The remaining entries are the sender's selected peers, ordered in NDN @@ -275,12 +274,14 @@ Stop adding entries when the estimated direct `SvsData` size approaches the size of a direct SvsData: - When `encoded_size(FULL) ≤ SyncVectorThreshold`, nodes use direct FULL - (with `mhash` and `VectorType=FULL`). + (`FullStateVector`). - When `encoded_size(FULL) > SyncVectorThreshold`, nodes switch to PARTIAL - (publication) or publish + pull (periodic sync and recovery). + (`PartialStateVector`, on publication) or publish + pull (`SvsDataRef`, + on periodic sync and recovery). -The wire format is independent of `SyncVectorThreshold`. All Sync messages -carry `mhash` and a `VectorType` (or `SvsDataRef` for publish-only). +The wire format is independent of `SyncVectorThreshold`. Every Sync +message carries exactly one of `FullStateVector`, `PartialStateVector`, +or `SvsDataRef`. --- @@ -302,8 +303,8 @@ Interest and resets the timer to `PeriodicTimeout`. | Trigger | Action | |---------|--------| -| `encoded_size(direct FULL) ≤ SyncVectorThreshold` | Send direct FULL (`mhash` + `VectorType=FULL` + `StateVector`) | -| `encoded_size(direct FULL) > SyncVectorThreshold` | Send direct PARTIAL (`mhash` + `VectorType=PARTIAL` + subset `StateVector`), or publish + pull if the sender-only baseline itself exceeds the threshold | +| `encoded_size(direct FULL) ≤ SyncVectorThreshold` | Send direct FULL (`FullStateVector` with `mhash` + `StateVector`) | +| `encoded_size(direct FULL) > SyncVectorThreshold` | Send direct PARTIAL (`PartialStateVector` with `mhash` + subset `StateVector`), or publish + pull if the sender-only baseline itself exceeds the threshold | ### 5.3 Sync Ack policy @@ -311,12 +312,12 @@ Sync Interests are unacknowledged. ### 5.4 Steady state and suppression (direct FULL) -For incoming Sync Data with direct `VectorType = FULL`, apply the +For incoming Sync Data carrying `FullStateVector`, apply the steady-state and suppression rules in §5.1–§5.4. ### 5.5 PARTIAL State Vector processing -When `VectorType = PARTIAL`: +When the wire carries `PartialStateVector`: 1. Parse `mhash` and `StateVector`. 2. Names omitted from the partial `StateVector` are interpreted as "not @@ -337,7 +338,7 @@ direct-FULL path. | # | Trigger | Action | |---|---------|--------| | 1 | `mhash` in received `SvsData` ≠ locally computed `mhash` | Publish + pull | -| 2 | Direct `VectorType = FULL` is outdated per §6.2 | Merge direct if complete; otherwise publish + pull | +| 2 | Direct `FullStateVector` is outdated per §6.2 | Merge direct if complete; otherwise publish + pull | | 3 | Periodic sync while local FULL exceeds `SyncVectorThreshold` | Publish + pull (§5.8) | Recovery always fetches the complete State Vector from the referenced @@ -346,8 +347,8 @@ Recovery always fetches the complete State Vector from the referenced **Sender procedure** (on `mhash` mismatch or periodic large-group sync): 1. Produce full-vector Data at `////32=sv/` - with direct FULL `SvsData`. -2. Send Sync Interest with publish-only `SvsData` (`mhash` + `SvsDataRef`). + with `FullStateVector` SvsData. +2. Send Sync Interest with publish-only `SvsData` (`SvsDataRef` only). **Receiver procedure:** @@ -373,11 +374,11 @@ not fit in a single packet. 1. Joining node **N** multicasts Sync Interest whose direct State Vector contains only itself: `(Name=N, SeqNo=0)`. The Sync Data's `mhash` is - the SHA-256 of N's membership set, which is a single member (N). + the SHA-256 of N's membership set, which is the single member `{N}`. 2. Existing members receive the announcement. 3. Suppression limits duplicate responses; typically one member **A** provides recovery state. -4. If FULL fits in a direct packet: **A** responds with direct `VectorType = FULL`. +4. If FULL fits in a direct packet: **A** responds with `FullStateVector`. 5. If FULL exceeds `SyncVectorThreshold`: **A** uses publish + pull (produce at `32=sv/`, then publish-only Sync Data). 6. Normal synchronization proceeds through SvsALO. @@ -386,7 +387,7 @@ not fit in a single packet. | Local FULL size | Periodic Sync behavior | |-----------------|------------------------| -| `≤ SyncVectorThreshold` | Direct FULL | +| `≤ SyncVectorThreshold` | Direct FULL (`FullStateVector`) | | `> SyncVectorThreshold` | Publish + pull (produce full-vector Data, then publish-only Sync Data) | Periodic sync does not send direct PARTIAL vectors. @@ -395,8 +396,8 @@ Periodic sync does not send direct PARTIAL vectors. | Event | `size ≤ threshold` | `size > threshold` | |-------|--------------------|--------------------| -| **New publication** | Direct FULL | Direct PARTIAL (or publish + pull fallback) | -| **Periodic sync** | Direct FULL | Publish + pull | +| **New publication** | Direct FULL (`FullStateVector`) | Direct PARTIAL (`PartialStateVector`, or publish + pull fallback) | +| **Periodic sync** | Direct FULL (`FullStateVector`) | Publish + pull | | **`mhash` mismatch** | Publish + pull (if recovery needed) | Publish + pull | --- @@ -414,11 +415,11 @@ State Vector `A` is outdated to `B` if: - `A` is missing a name present in `B`, or - `A` has a strictly smaller `SeqNo` for any entry. -This rule applies when `A` is a `VectorType = FULL` State Vector. When -`A` is `VectorType = PARTIAL`, `A`'s omitted names are a subset by -design (§4.2): the sender selected a publication-time subset and `A`'s -missing entries do not carry any information about whether `A` is -outdated relative to `B`. +This rule applies when `A` is a `FullStateVector`. When `A` is a +`PartialStateVector`, `A`'s omitted names are a subset by design (§4.2): +the sender selected a publication-time subset and `A`'s missing entries +do not carry any information about whether `A` is outdated relative to +`B`. --- @@ -433,7 +434,7 @@ direct FULL Sync Interest `[A:11, B:15, C:25]`. Peers merge. Group exceeds `SyncVectorThreshold`. Producer `P` publishes: -- `P` sends direct PARTIAL `SvsData { mhash, VectorType=PARTIAL, +- `P` sends direct PARTIAL `SvsData` carrying `PartialStateVector { mhash, StateVector=[P:…, A:…, …] }`. - Receiver merges present entries only. - If `mhash` differs, `P` (or receiver per policy) triggers publish + pull @@ -442,15 +443,14 @@ Group exceeds `SyncVectorThreshold`. Producer `P` publishes: ### 7.3 Large group - `A` produces full vector at `/group/A/boot/32=sv/`. -- `A` sends publish-only Sync Data `{ mhash, - SvsDataRef=/group/A/boot/32=sv/ }`. +- `A` sends publish-only Sync Data `{ SvsDataRef=/group/A/boot/32=sv/ }`. - Peers pull and merge. ### 7.4 New node join - `N` sends a State Vector containing only itself (`[N:0]`) with `mhash` computed over the single-member membership set `{N}`. -- `A` responds with direct FULL or publish + pull. +- `A` responds with `FullStateVector` or publish + pull. - `N` merges and synchronizes via SvsALO. --- @@ -458,6 +458,7 @@ Group exceeds `SyncVectorThreshold`. Producer `P` publishes: ## 8. Interoperability SVS v4 defines a single wire profile. Deployments upgrade all nodes in a -sync group at the same time. Every Sync Data carries `mhash` and a -`VectorType` (or `SvsDataRef` for publish-only). The implementation never -emits a `StateVector`-only `SvsData`, regardless of `SyncVectorThreshold`. \ No newline at end of file +sync group at the same time. Every Sync Data carries exactly one of +`FullStateVector`, `PartialStateVector`, or `SvsDataRef`. The +implementation never emits a bare `StateVector`-only `SvsData`, +regardless of `SyncVectorThreshold`. \ No newline at end of file diff --git a/dv/dv/advert_sync.go b/dv/dv/advert_sync.go index 5291a244..b65e50a0 100644 --- a/dv/dv/advert_sync.go +++ b/dv/dv/advert_sync.go @@ -50,17 +50,18 @@ func (a *advertModule) sendSyncInterest() (err error) { // (AI GENERATED DESCRIPTION): Sends a signed state‑vector Data packet as the payload of a sync Interest to the given `syncName`, expressing the Interest locally without expecting a reply. func (a *advertModule) sendSyncInterestImpl(syncName enc.Name) (err error) { - // State Vector for our group - sv := &spec_svs.SvsData{ - StateVector: &spec_svs.StateVector{ - Entries: []*spec_svs.StateVectorEntry{{ - Name: a.dv.config.RouterName(), - SeqNoEntries: []*spec_svs.SeqNoEntry{{ - BootstrapTime: a.bootTime, - SeqNo: a.seq, - }}, + // DV's advertisement Sync Data carries a single-entry StateVector + // directly. SVS v4's wire (FullStateVector/PartialStateVector/SvsDataRef + // tagged union) is a different protocol layer; DV keeps its own simpler + // shape to remain independent of SVS v4 changes. + sv := &spec_svs.StateVector{ + Entries: []*spec_svs.StateVectorEntry{{ + Name: a.dv.config.RouterName(), + SeqNoEntries: []*spec_svs.SeqNoEntry{{ + BootstrapTime: a.bootTime, + SeqNo: a.seq, }}, - }, + }}, } // Sign the Sync Data @@ -136,14 +137,14 @@ func (a *advertModule) OnSyncInterest(args ndn.InterestHandlerArgs, active bool) // Decode state vector svWire := data.Content() - params, err := spec_svs.ParseSvsData(enc.NewWireView(svWire), false) - if err != nil || params.StateVector == nil { + params, err := spec_svs.ParseStateVector(enc.NewWireView(svWire), false) + if err != nil { log.Warn(a, "Failed to parse StateVec", "err", err) return } // Process the state vector - go a.onStateVector(params.StateVector, args.IncomingFaceId.Unwrap(), active) + go a.onStateVector(params, args.IncomingFaceId.Unwrap(), active) }, }) } diff --git a/std/ndn/svs/v3/accessors.go b/std/ndn/svs/v3/accessors.go new file mode 100644 index 00000000..170136a5 --- /dev/null +++ b/std/ndn/svs/v3/accessors.go @@ -0,0 +1,54 @@ +package svs + +// VectorKind identifies which direct form an SvsData carries on the wire. +// Publish-only Sync Data carries neither FullStateVector nor +// PartialStateVector; instead it has SvsDataRef. The choice of wire TLV +// replaces the previous VectorType discriminator field. +type VectorKind int + +const ( + // VectorKindNone is the publish-only form: no embedded vector, only a + // retrievable SvsDataRef. + VectorKindNone VectorKind = iota + // VectorKindFull is a complete State Vector (FULL). + VectorKindFull + // VectorKindPartial is a sender-selected subset (PARTIAL). + VectorKindPartial +) + +// GetStateVector returns the embedded StateVector for direct forms (FULL +// or PARTIAL), or nil for the publish-only form. +func (d *SvsData) GetStateVector() *StateVector { + switch { + case d.FullStateVector != nil: + return d.FullStateVector.StateVector + case d.PartialStateVector != nil: + return d.PartialStateVector.StateVector + } + return nil +} + +// Kind reports which direct form (or none) the SvsData carries on the wire. +// An SvsData carrying both FullStateVector and PartialStateVector is +// reported as KindFull (the more specific case wins). +func (d *SvsData) Kind() VectorKind { + switch { + case d.FullStateVector != nil: + return VectorKindFull + case d.PartialStateVector != nil: + return VectorKindPartial + } + return VectorKindNone +} + +// IsPartial reports whether the embedded StateVector is a PARTIAL subset. +// Returns false for the publish-only form (callers that branch on this +// should also check SvsDataRef for the publish-only recovery path). +func (d *SvsData) IsPartial() bool { + return d.PartialStateVector != nil +} + +// IsFull reports whether the embedded StateVector is a FULL State Vector. +func (d *SvsData) IsFull() bool { + return d.FullStateVector != nil +} \ No newline at end of file diff --git a/std/ndn/svs/v3/definitions.go b/std/ndn/svs/v3/definitions.go index 4412d154..08d12832 100644 --- a/std/ndn/svs/v3/definitions.go +++ b/std/ndn/svs/v3/definitions.go @@ -3,24 +3,39 @@ package svs import ( enc "github.com/named-data/ndnd/std/encoding" - "github.com/named-data/ndnd/std/types/optional" ) -// VectorType values for inline SvsData (TLV 0xCD). -const ( - VectorTypeFull uint64 = 0 - VectorTypePartial uint64 = 1 -) +// FullStateVector is the wire form of a complete State Vector carried in a +// Sync message or published at .../32=sv/. The presence of this +// TLV (rather than PartialStateVector) tells the receiver that the embedded +// StateVector represents the sender's full membership view. +type FullStateVector struct { + //+field:struct:StateVector + StateVector *StateVector `tlv:"0xc9"` +} + +// PartialStateVector is the wire form of a publication-time subset State +// Vector. The presence of this TLV tells the receiver that omitted entries +// are a sender-selected subset (not a partition or out-of-date sender). +type PartialStateVector struct { + //+field:struct:StateVector + StateVector *StateVector `tlv:"0xc9"` +} +// SvsData is a tagged union: the wire carries exactly one of +// FullStateVector, PartialStateVector, or SvsDataRef. The choice of TLV +// type replaces the previous VectorType discriminator. MemberSetHash +// (`mhash`) is present on all three forms and lets receivers detect +// membership mismatches without walking a full StateVector. type SvsData struct { //+field:binary:optional MemberSetHash []byte `tlv:"0xcb"` - //+field:natural:optional - VectorType optional.Optional[uint64] `tlv:"0xcd"` + //+field:struct:FullStateVector + FullStateVector *FullStateVector `tlv:"0xcd"` + //+field:struct:PartialStateVector + PartialStateVector *PartialStateVector `tlv:"0xce"` //+field:name SvsDataRef enc.Name `tlv:"0x07"` - //+field:struct:StateVector - StateVector *StateVector `tlv:"0xc9"` } type StateVector struct { diff --git a/std/ndn/svs/v3/zz_generated.go b/std/ndn/svs/v3/zz_generated.go index 6d4590e7..230120be 100644 --- a/std/ndn/svs/v3/zz_generated.go +++ b/std/ndn/svs/v3/zz_generated.go @@ -8,28 +8,301 @@ import ( enc "github.com/named-data/ndnd/std/encoding" ) -type SvsDataEncoder struct { +type FullStateVectorEncoder struct { Length uint - SvsDataRef_length uint StateVector_encoder StateVectorEncoder } -type SvsDataParsingContext struct { +type FullStateVectorParsingContext struct { StateVector_context StateVectorParsingContext } +func (encoder *FullStateVectorEncoder) Init(value *FullStateVector) { + if value.StateVector != nil { + encoder.StateVector_encoder.Init(value.StateVector) + } + + l := uint(0) + if value.StateVector != nil { + l += 1 + l += uint(enc.TLNum(encoder.StateVector_encoder.Length).EncodingLength()) + l += encoder.StateVector_encoder.Length + } + encoder.Length = l + +} + +func (context *FullStateVectorParsingContext) Init() { + context.StateVector_context.Init() +} + +func (encoder *FullStateVectorEncoder) EncodeInto(value *FullStateVector, buf []byte) { + + pos := uint(0) + + if value.StateVector != nil { + buf[pos] = byte(201) + pos += 1 + pos += uint(enc.TLNum(encoder.StateVector_encoder.Length).EncodeInto(buf[pos:])) + if encoder.StateVector_encoder.Length > 0 { + encoder.StateVector_encoder.EncodeInto(value.StateVector, buf[pos:]) + pos += encoder.StateVector_encoder.Length + } + } +} + +func (encoder *FullStateVectorEncoder) Encode(value *FullStateVector) enc.Wire { + + wire := make(enc.Wire, 1) + wire[0] = make([]byte, encoder.Length) + buf := wire[0] + encoder.EncodeInto(value, buf) + + return wire +} + +func (context *FullStateVectorParsingContext) Parse(reader enc.WireView, ignoreCritical bool) (*FullStateVector, error) { + + var handled_StateVector bool = false + + progress := -1 + _ = progress + + value := &FullStateVector{} + var err error + var startPos int + for { + startPos = reader.Pos() + if startPos >= reader.Length() { + break + } + typ := enc.TLNum(0) + l := enc.TLNum(0) + typ, err = reader.ReadTLNum() + if err != nil { + return nil, enc.ErrFailToParse{TypeNum: 0, Err: err} + } + l, err = reader.ReadTLNum() + if err != nil { + return nil, enc.ErrFailToParse{TypeNum: 0, Err: err} + } + + err = nil + if handled := false; true { + switch typ { + case 201: + if true { + handled = true + handled_StateVector = true + value.StateVector, err = context.StateVector_context.Parse(reader.Delegate(int(l)), ignoreCritical) + } + default: + if !ignoreCritical && ((typ <= 31) || ((typ & 1) == 1)) { + return nil, enc.ErrUnrecognizedField{TypeNum: typ} + } + handled = true + err = reader.Skip(int(l)) + } + if err == nil && !handled { + } + if err != nil { + return nil, enc.ErrFailToParse{TypeNum: typ, Err: err} + } + } + } + + startPos = reader.Pos() + err = nil + + if !handled_StateVector && err == nil { + value.StateVector = nil + } + + if err != nil { + return nil, err + } + + return value, nil +} + +func (value *FullStateVector) Encode() enc.Wire { + encoder := FullStateVectorEncoder{} + encoder.Init(value) + return encoder.Encode(value) +} + +func (value *FullStateVector) Bytes() []byte { + return value.Encode().Join() +} + +func ParseFullStateVector(reader enc.WireView, ignoreCritical bool) (*FullStateVector, error) { + context := FullStateVectorParsingContext{} + context.Init() + return context.Parse(reader, ignoreCritical) +} + +type PartialStateVectorEncoder struct { + Length uint + + StateVector_encoder StateVectorEncoder +} + +type PartialStateVectorParsingContext struct { + StateVector_context StateVectorParsingContext +} + +func (encoder *PartialStateVectorEncoder) Init(value *PartialStateVector) { + if value.StateVector != nil { + encoder.StateVector_encoder.Init(value.StateVector) + } + + l := uint(0) + if value.StateVector != nil { + l += 1 + l += uint(enc.TLNum(encoder.StateVector_encoder.Length).EncodingLength()) + l += encoder.StateVector_encoder.Length + } + encoder.Length = l + +} + +func (context *PartialStateVectorParsingContext) Init() { + context.StateVector_context.Init() +} + +func (encoder *PartialStateVectorEncoder) EncodeInto(value *PartialStateVector, buf []byte) { + + pos := uint(0) + + if value.StateVector != nil { + buf[pos] = byte(201) + pos += 1 + pos += uint(enc.TLNum(encoder.StateVector_encoder.Length).EncodeInto(buf[pos:])) + if encoder.StateVector_encoder.Length > 0 { + encoder.StateVector_encoder.EncodeInto(value.StateVector, buf[pos:]) + pos += encoder.StateVector_encoder.Length + } + } +} + +func (encoder *PartialStateVectorEncoder) Encode(value *PartialStateVector) enc.Wire { + + wire := make(enc.Wire, 1) + wire[0] = make([]byte, encoder.Length) + buf := wire[0] + encoder.EncodeInto(value, buf) + + return wire +} + +func (context *PartialStateVectorParsingContext) Parse(reader enc.WireView, ignoreCritical bool) (*PartialStateVector, error) { + + var handled_StateVector bool = false + + progress := -1 + _ = progress + + value := &PartialStateVector{} + var err error + var startPos int + for { + startPos = reader.Pos() + if startPos >= reader.Length() { + break + } + typ := enc.TLNum(0) + l := enc.TLNum(0) + typ, err = reader.ReadTLNum() + if err != nil { + return nil, enc.ErrFailToParse{TypeNum: 0, Err: err} + } + l, err = reader.ReadTLNum() + if err != nil { + return nil, enc.ErrFailToParse{TypeNum: 0, Err: err} + } + + err = nil + if handled := false; true { + switch typ { + case 201: + if true { + handled = true + handled_StateVector = true + value.StateVector, err = context.StateVector_context.Parse(reader.Delegate(int(l)), ignoreCritical) + } + default: + if !ignoreCritical && ((typ <= 31) || ((typ & 1) == 1)) { + return nil, enc.ErrUnrecognizedField{TypeNum: typ} + } + handled = true + err = reader.Skip(int(l)) + } + if err == nil && !handled { + } + if err != nil { + return nil, enc.ErrFailToParse{TypeNum: typ, Err: err} + } + } + } + + startPos = reader.Pos() + err = nil + + if !handled_StateVector && err == nil { + value.StateVector = nil + } + + if err != nil { + return nil, err + } + + return value, nil +} + +func (value *PartialStateVector) Encode() enc.Wire { + encoder := PartialStateVectorEncoder{} + encoder.Init(value) + return encoder.Encode(value) +} + +func (value *PartialStateVector) Bytes() []byte { + return value.Encode().Join() +} + +func ParsePartialStateVector(reader enc.WireView, ignoreCritical bool) (*PartialStateVector, error) { + context := PartialStateVectorParsingContext{} + context.Init() + return context.Parse(reader, ignoreCritical) +} + +type SvsDataEncoder struct { + Length uint + + FullStateVector_encoder FullStateVectorEncoder + PartialStateVector_encoder PartialStateVectorEncoder + SvsDataRef_length uint +} + +type SvsDataParsingContext struct { + FullStateVector_context FullStateVectorParsingContext + PartialStateVector_context PartialStateVectorParsingContext +} + func (encoder *SvsDataEncoder) Init(value *SvsData) { + if value.FullStateVector != nil { + encoder.FullStateVector_encoder.Init(value.FullStateVector) + } + if value.PartialStateVector != nil { + encoder.PartialStateVector_encoder.Init(value.PartialStateVector) + } if value.SvsDataRef != nil { encoder.SvsDataRef_length = 0 for _, c := range value.SvsDataRef { encoder.SvsDataRef_length += uint(c.EncodingLength()) } } - if value.StateVector != nil { - encoder.StateVector_encoder.Init(value.StateVector) - } l := uint(0) if value.MemberSetHash != nil { @@ -37,27 +310,30 @@ func (encoder *SvsDataEncoder) Init(value *SvsData) { l += uint(enc.TLNum(len(value.MemberSetHash)).EncodingLength()) l += uint(len(value.MemberSetHash)) } - if optval, ok := value.VectorType.Get(); ok { + if value.FullStateVector != nil { + l += 1 + l += uint(enc.TLNum(encoder.FullStateVector_encoder.Length).EncodingLength()) + l += encoder.FullStateVector_encoder.Length + } + if value.PartialStateVector != nil { l += 1 - l += uint(1 + enc.Nat(optval).EncodingLength()) + l += uint(enc.TLNum(encoder.PartialStateVector_encoder.Length).EncodingLength()) + l += encoder.PartialStateVector_encoder.Length } if value.SvsDataRef != nil { l += 1 l += uint(enc.TLNum(encoder.SvsDataRef_length).EncodingLength()) l += encoder.SvsDataRef_length } - if value.StateVector != nil { - l += 1 - l += uint(enc.TLNum(encoder.StateVector_encoder.Length).EncodingLength()) - l += encoder.StateVector_encoder.Length - } encoder.Length = l } func (context *SvsDataParsingContext) Init() { - context.StateVector_context.Init() + context.FullStateVector_context.Init() + context.PartialStateVector_context.Init() + } func (encoder *SvsDataEncoder) EncodeInto(value *SvsData, buf []byte) { @@ -71,13 +347,23 @@ func (encoder *SvsDataEncoder) EncodeInto(value *SvsData, buf []byte) { copy(buf[pos:], value.MemberSetHash) pos += uint(len(value.MemberSetHash)) } - if optval, ok := value.VectorType.Get(); ok { + if value.FullStateVector != nil { buf[pos] = byte(205) pos += 1 - - buf[pos] = byte(enc.Nat(optval).EncodeInto(buf[pos+1:])) - pos += uint(1 + buf[pos]) - + pos += uint(enc.TLNum(encoder.FullStateVector_encoder.Length).EncodeInto(buf[pos:])) + if encoder.FullStateVector_encoder.Length > 0 { + encoder.FullStateVector_encoder.EncodeInto(value.FullStateVector, buf[pos:]) + pos += encoder.FullStateVector_encoder.Length + } + } + if value.PartialStateVector != nil { + buf[pos] = byte(206) + pos += 1 + pos += uint(enc.TLNum(encoder.PartialStateVector_encoder.Length).EncodeInto(buf[pos:])) + if encoder.PartialStateVector_encoder.Length > 0 { + encoder.PartialStateVector_encoder.EncodeInto(value.PartialStateVector, buf[pos:]) + pos += encoder.PartialStateVector_encoder.Length + } } if value.SvsDataRef != nil { buf[pos] = byte(7) @@ -87,15 +373,6 @@ func (encoder *SvsDataEncoder) EncodeInto(value *SvsData, buf []byte) { pos += uint(c.EncodeInto(buf[pos:])) } } - if value.StateVector != nil { - buf[pos] = byte(201) - pos += 1 - pos += uint(enc.TLNum(encoder.StateVector_encoder.Length).EncodeInto(buf[pos:])) - if encoder.StateVector_encoder.Length > 0 { - encoder.StateVector_encoder.EncodeInto(value.StateVector, buf[pos:]) - pos += encoder.StateVector_encoder.Length - } - } } func (encoder *SvsDataEncoder) Encode(value *SvsData) enc.Wire { @@ -111,9 +388,9 @@ func (encoder *SvsDataEncoder) Encode(value *SvsData) enc.Wire { func (context *SvsDataParsingContext) Parse(reader enc.WireView, ignoreCritical bool) (*SvsData, error) { var handled_MemberSetHash bool = false - var handled_VectorType bool = false + var handled_FullStateVector bool = false + var handled_PartialStateVector bool = false var handled_SvsDataRef bool = false - var handled_StateVector bool = false progress := -1 _ = progress @@ -150,25 +427,14 @@ func (context *SvsDataParsingContext) Parse(reader enc.WireView, ignoreCritical case 205: if true { handled = true - handled_VectorType = true - { - optval := uint64(0) - optval = uint64(0) - { - for i := 0; i < int(l); i++ { - x := byte(0) - x, err = reader.ReadByte() - if err != nil { - if err == io.EOF { - err = io.ErrUnexpectedEOF - } - break - } - optval = uint64(optval<<8) | uint64(x) - } - } - value.VectorType.Set(optval) - } + handled_FullStateVector = true + value.FullStateVector, err = context.FullStateVector_context.Parse(reader.Delegate(int(l)), ignoreCritical) + } + case 206: + if true { + handled = true + handled_PartialStateVector = true + value.PartialStateVector, err = context.PartialStateVector_context.Parse(reader.Delegate(int(l)), ignoreCritical) } case 7: if true { @@ -177,12 +443,6 @@ func (context *SvsDataParsingContext) Parse(reader enc.WireView, ignoreCritical delegate := reader.Delegate(int(l)) value.SvsDataRef, err = delegate.ReadName() } - case 201: - if true { - handled = true - handled_StateVector = true - value.StateVector, err = context.StateVector_context.Parse(reader.Delegate(int(l)), ignoreCritical) - } default: if !ignoreCritical && ((typ <= 31) || ((typ & 1) == 1)) { return nil, enc.ErrUnrecognizedField{TypeNum: typ} @@ -204,15 +464,15 @@ func (context *SvsDataParsingContext) Parse(reader enc.WireView, ignoreCritical if !handled_MemberSetHash && err == nil { value.MemberSetHash = nil } - if !handled_VectorType && err == nil { - value.VectorType.Unset() + if !handled_FullStateVector && err == nil { + value.FullStateVector = nil + } + if !handled_PartialStateVector && err == nil { + value.PartialStateVector = nil } if !handled_SvsDataRef && err == nil { value.SvsDataRef = nil } - if !handled_StateVector && err == nil { - value.StateVector = nil - } if err != nil { return nil, err diff --git a/std/sync/svs.go b/std/sync/svs.go index b3c6f3a3..aa0b7ea1 100644 --- a/std/sync/svs.go +++ b/std/sync/svs.go @@ -20,8 +20,8 @@ import ( // syncVectorThreshold is the max embedded SvsData size (bytes) above // which the sender switches to PARTIAL (on publication) or publish+pull -// (on periodic sync and recovery). SVS v4 always emits `mhash` and a -// `VectorType` on the wire. +// (on periodic sync and recovery). SVS v4 always emits `mhash` and one of +// FullStateVector/PartialStateVector on the wire (see std/ndn/svs/v3). const syncVectorThreshold = 1200 type SvSync struct { @@ -106,7 +106,7 @@ type SvSyncUpdate struct { type svSyncRecvSvArgs struct { sv *spec_svs.StateVector data enc.Wire - vectorType optional.Optional[uint64] + partial bool mhash []byte svsDataRef enc.Name } @@ -397,11 +397,11 @@ func (s *SvSync) onReceiveStateVector(args svSyncRecvSvArgs) { // The above checks each node in the incoming state vector, but // does not check if a node is missing from the incoming state vector. // - // [Spec] For embedded SvsData, VectorType is required by the protocol: - // publish-only Sync Data carries no StateVector and is filtered out - // earlier (see onSyncData). So args.vectorType is guaranteed present - // here; we default missing values to FULL rather than branch on `ok`. - isPartial := args.vectorType.GetOr(spec_svs.VectorTypeFull) == spec_svs.VectorTypePartial + // [Spec] The wire TLV (FullStateVector or PartialStateVector) replaces + // the previous VectorType discriminator. publish-only Sync Data carries + // no StateVector and is filtered out earlier (see onSyncData), so any + // args.sv reaching this function was either FULL or PARTIAL on the wire. + isPartial := args.partial // [Spec] Membership recovery is a FULL-boundary operation: only an embedded // FULL StateVector or a publish-only Sync Data (which carries no StateVector // and is filtered earlier in onSyncData) represents the sender's complete @@ -620,47 +620,47 @@ func (s *SvSync) onSyncData(dataWire enc.Wire) { return } - // [Spec] Every Sync Data carries a 32-byte mhash. Reject malformed - // packets that would misclassify PARTIAL-as-FULL or skip recovery. - if len(params.MemberSetHash) != 32 { - log.Warn(s, "onSyncInterest SvsData missing or invalid mhash", - "len", len(params.MemberSetHash)) - return - } + mhash := params.MemberSetHash + sv := params.GetStateVector() // Publish-only ref: advertise that the full vector is retrievable. - if params.StateVector == nil && len(params.SvsDataRef) > 0 { + if sv == nil && len(params.SvsDataRef) > 0 { + // [Spec] Every Sync Data carries a 32-byte mhash. Reject malformed + // packets that omit mhash on the publish-only form. + if len(mhash) != 32 { + log.Warn(s, "onSyncInterest publish-only SvsData missing or invalid mhash", + "len", len(mhash)) + return + } trustPrefix := pullRefFromSyncDataWire(dataWire) go s.pullFullVector(params.SvsDataRef, trustPrefix) return } - if params.StateVector == nil { + if sv == nil { log.Warn(s, "onSyncInterest SvsData has no StateVector") return } - // [Spec] Inline form must carry VectorType (FULL or PARTIAL). - vt, ok := params.VectorType.Get() - if !ok { - log.Warn(s, "onSyncInterest inline SvsData missing VectorType") + // [Spec] Direct form must carry exactly one of FullStateVector / + // PartialStateVector, and a 32-byte mhash. The wire TLV replaces + // the previous VectorType discriminator. + if !params.IsFull() && !params.IsPartial() { + log.Warn(s, "onSyncInterest inline SvsData missing direct form") return } - if vt != spec_svs.VectorTypeFull && vt != spec_svs.VectorTypePartial { - log.Warn(s, "onSyncInterest inline SvsData invalid VectorType", "vt", vt) + if len(mhash) != 32 { + log.Warn(s, "onSyncInterest inline SvsData missing or invalid mhash", + "len", len(mhash)) return } - args := svSyncRecvSvArgs{ - sv: params.StateVector, + s.recvSv <- svSyncRecvSvArgs{ + sv: sv, data: dataWire, - mhash: params.MemberSetHash, + partial: params.IsPartial(), + mhash: mhash, svsDataRef: params.SvsDataRef, } - if vt, ok := params.VectorType.Get(); ok { - args.vectorType = optional.Some(vt) - } - - s.recvSv <- args }, }) } diff --git a/std/sync/svs_encode.go b/std/sync/svs_encode.go index 835f0f8d..8047207b 100644 --- a/std/sync/svs_encode.go +++ b/std/sync/svs_encode.go @@ -8,7 +8,6 @@ import ( enc "github.com/named-data/ndnd/std/encoding" spec_svs "github.com/named-data/ndnd/std/ndn/svs/v3" - "github.com/named-data/ndnd/std/types/optional" ) // syncSendReason distinguishes why a Sync Interest is being sent. @@ -49,9 +48,8 @@ type svsSendInput struct { func buildSvsDataForSend(in svsSendInput) *spec_svs.SvsData { fullSv := in.State.Encode(func(seq uint64) uint64 { return seq }) fullData := &spec_svs.SvsData{ - MemberSetHash: ComputeMembershipHash(in.State), - VectorType: optional.Some(spec_svs.VectorTypeFull), - StateVector: fullSv, + MemberSetHash: ComputeMembershipHash(in.State), + FullStateVector: &spec_svs.FullStateVector{StateVector: fullSv}, } if in.Reason != syncSendPublication || len(fullData.Encode().Join()) <= in.Threshold { @@ -70,9 +68,8 @@ func buildSvsDataForSend(in svsSendInput) *spec_svs.SvsData { return nil } return &spec_svs.SvsData{ - MemberSetHash: ComputeMembershipHash(in.State), - VectorType: optional.Some(spec_svs.VectorTypePartial), - StateVector: partialSv, + MemberSetHash: ComputeMembershipHash(in.State), + PartialStateVector: &spec_svs.PartialStateVector{StateVector: partialSv}, } } @@ -95,9 +92,8 @@ func encodePartialStateVector(state SvMap[uint64], opts PartialEncodeOpts) *spec // Sender-only baseline must always fit when possible. baseline := &spec_svs.StateVector{Entries: []*spec_svs.StateVectorEntry{senderEntry}} baselineData := &spec_svs.SvsData{ - MemberSetHash: ComputeMembershipHash(state), - VectorType: optional.Some(spec_svs.VectorTypePartial), - StateVector: baseline, + MemberSetHash: ComputeMembershipHash(state), + PartialStateVector: &spec_svs.PartialStateVector{StateVector: baseline}, } if len(baselineData.Encode().Join()) > opts.Threshold { // Baseline too large to fit even the sender entry: return an @@ -124,9 +120,8 @@ func encodePartialStateVector(state SvMap[uint64], opts PartialEncodeOpts) *spec sortPartialTail(trial) trialSv := &spec_svs.StateVector{Entries: trial} trialData := &spec_svs.SvsData{ - MemberSetHash: ComputeMembershipHash(state), - VectorType: optional.Some(spec_svs.VectorTypePartial), - StateVector: trialSv, + MemberSetHash: ComputeMembershipHash(state), + PartialStateVector: &spec_svs.PartialStateVector{StateVector: trialSv}, } if len(trialData.Encode().Join()) > opts.Threshold { break diff --git a/std/sync/svs_pull.go b/std/sync/svs_pull.go index 69d717f3..e9ad0829 100644 --- a/std/sync/svs_pull.go +++ b/std/sync/svs_pull.go @@ -10,7 +10,6 @@ import ( "github.com/named-data/ndnd/std/ndn" spec "github.com/named-data/ndnd/std/ndn/spec_2022" spec_svs "github.com/named-data/ndnd/std/ndn/svs/v3" - "github.com/named-data/ndnd/std/types/optional" ) const ( @@ -83,9 +82,9 @@ func buildPublishSvsData(state SvMap[uint64], ref enc.Name) *spec_svs.SvsData { } // shouldUsePublishPull reports whether the sender should publish at .../32=sv -// and emit publish-only Sync Data (mhash + SvsDataRef, no embedded vector) -// instead of an embedded FULL or PARTIAL StateVector. The full SvsData is -// built once (so the caller can reuse it if not publishing) and its size is +// and emit publish-only Sync Data (SvsDataRef, no embedded vector) instead +// of an embedded FULL or PARTIAL StateVector. The full SvsData is built +// once (so the caller can reuse it if not publishing) and its size is // returned alongside the decision so the caller does not have to re-encode. func shouldUsePublishPull(reason syncSendReason, threshold int, state SvMap[uint64]) (usePublish bool, data *spec_svs.SvsData, size int) { switch reason { @@ -96,9 +95,8 @@ func shouldUsePublishPull(reason syncSendReason, threshold int, state SvMap[uint } sv := state.Encode(func(seq uint64) uint64 { return seq }) full := &spec_svs.SvsData{ - MemberSetHash: ComputeMembershipHash(state), - VectorType: optional.Some(spec_svs.VectorTypeFull), - StateVector: sv, + MemberSetHash: ComputeMembershipHash(state), + FullStateVector: &spec_svs.FullStateVector{StateVector: sv}, } wire := full.Encode().Join() if len(wire) > threshold { @@ -114,9 +112,8 @@ func (s *SvSync) publishFullVectorData(state SvMap[uint64]) (enc.Name, error) { } sv := state.Encode(func(seq uint64) uint64 { return seq }) content := (&spec_svs.SvsData{ - MemberSetHash: ComputeMembershipHash(state), - VectorType: optional.Some(spec_svs.VectorTypeFull), - StateVector: sv, + MemberSetHash: ComputeMembershipHash(state), + FullStateVector: &spec_svs.FullStateVector{StateVector: sv}, }).Encode() name := s.fullVectorPrefix.WithVersion(enc.VersionUnixMicro) return s.o.Client.Produce(ndn.ProduceArgs{ @@ -190,9 +187,8 @@ func (s *SvSync) onPulledFullVector(content []byte) { } s.recvSv <- svSyncRecvSvArgs{ - sv: params.StateVector, - vectorType: optional.Some(spec_svs.VectorTypeFull), - mhash: params.MemberSetHash, + sv: params.GetStateVector(), + mhash: params.MemberSetHash, } } @@ -201,23 +197,22 @@ func parseFullVectorContent(content []byte) (*spec_svs.SvsData, error) { if err != nil { return nil, err } - if params.StateVector == nil { - return nil, fmt.Errorf("full vector content has no StateVector") + // [Spec] Fetched full-vector Data is inline FULL: the wire TLV must be + // FullStateVector (0xCD), not PartialStateVector (0xCE), and the + // MemberSetHash must be present and valid. + if !params.IsFull() { + return nil, fmt.Errorf("full vector content is not FullStateVector") } - // [Spec] Fetched full-vector Data is inline FULL: VectorType must be FULL - // and MemberSetHash must be present and valid. - vt, ok := params.VectorType.Get() - if !ok { - return nil, fmt.Errorf("full vector content missing VectorType") - } - if vt != spec_svs.VectorTypeFull { - return nil, fmt.Errorf("full vector VectorType=%d, want FULL", vt) + sv := params.GetStateVector() + if sv == nil { + return nil, fmt.Errorf("full vector content has no StateVector") } - if len(params.MemberSetHash) != 32 { - return nil, fmt.Errorf("full vector content missing or invalid mhash (len=%d)", len(params.MemberSetHash)) + mhash := params.MemberSetHash + if len(mhash) != 32 { + return nil, fmt.Errorf("full vector content missing or invalid mhash (len=%d)", len(mhash)) } - computed := ComputeMembershipHash(stateVectorToMap(params.StateVector)) - if !bytes.Equal(params.MemberSetHash, computed) { + computed := ComputeMembershipHash(stateVectorToMap(sv)) + if !bytes.Equal(mhash, computed) { return nil, fmt.Errorf("full vector mhash mismatch") } return params, nil diff --git a/std/sync/svs_test.go b/std/sync/svs_test.go index b33c40e4..e27da35b 100644 --- a/std/sync/svs_test.go +++ b/std/sync/svs_test.go @@ -31,17 +31,16 @@ func TestBuildInlineFullSvsData(t *testing.T) { m := testSvMapAliceBob() sv := m.Encode(func(s uint64) uint64 { return s }) data := &spec_svs.SvsData{ - MemberSetHash: ComputeMembershipHash(m), - VectorType: optional.Some(spec_svs.VectorTypeFull), - StateVector: sv, + MemberSetHash: ComputeMembershipHash(m), + FullStateVector: &spec_svs.FullStateVector{StateVector: sv}, } + require.True(t, data.IsFull()) + require.False(t, data.IsPartial()) require.Equal(t, ComputeMembershipHash(m), data.MemberSetHash) - vt, ok := data.VectorType.Get() - require.True(t, ok) - require.Equal(t, spec_svs.VectorTypeFull, vt) - require.NotNil(t, data.StateVector) - require.Len(t, data.StateVector.Entries, 2) + svOut := data.GetStateVector() + require.NotNil(t, svOut) + require.Len(t, svOut.Entries, 2) } func TestOnReceivePartialSkipsMissingNameOutdated(t *testing.T) { @@ -65,9 +64,9 @@ func TestOnReceivePartialSkipsMissingNameOutdated(t *testing.T) { partialSv := bobOnly.Encode(func(s uint64) uint64 { return s }) s.onReceiveStateVector(svSyncRecvSvArgs{ - sv: partialSv, - vectorType: optional.Some(spec_svs.VectorTypePartial), - mhash: ComputeMembershipHash(bobOnly), + sv: partialSv, + partial: true, + mhash: ComputeMembershipHash(bobOnly), }) require.False(t, s.suppress) @@ -93,9 +92,8 @@ func TestOnReceiveFullTreatsMissingNameOutdated(t *testing.T) { fullSv := bobOnly.Encode(func(s uint64) uint64 { return s }) s.onReceiveStateVector(svSyncRecvSvArgs{ - sv: fullSv, - vectorType: optional.Some(spec_svs.VectorTypeFull), - mhash: ComputeMembershipHash(bobOnly), + sv: fullSv, + mhash: ComputeMembershipHash(bobOnly), }) require.True(t, s.suppress) @@ -115,9 +113,8 @@ func TestEncodePartialSenderFirst(t *testing.T) { // Threshold large enough for sender + one peer. full := &spec_svs.SvsData{ - MemberSetHash: ComputeMembershipHash(m), - VectorType: optional.Some(spec_svs.VectorTypeFull), - StateVector: m.Encode(func(s uint64) uint64 { return s }), + MemberSetHash: ComputeMembershipHash(m), + FullStateVector: &spec_svs.FullStateVector{StateVector: m.Encode(func(s uint64) uint64 { return s })}, } threshold := len(full.Encode().Join()) - 1 @@ -149,26 +146,25 @@ func TestBuildSvsDataForSendPublicationPartial(t *testing.T) { } full := &spec_svs.SvsData{ - MemberSetHash: ComputeMembershipHash(m), - VectorType: optional.Some(spec_svs.VectorTypeFull), - StateVector: m.Encode(func(s uint64) uint64 { return s }), + MemberSetHash: ComputeMembershipHash(m), + FullStateVector: &spec_svs.FullStateVector{StateVector: m.Encode(func(s uint64) uint64 { return s })}, } threshold := len(full.Encode().Join()) / 2 pub := buildSvsDataForSend(svsSendInput{ State: m, Reason: syncSendPublication, Threshold: threshold, Sender: alice, }) - vt, ok := pub.VectorType.Get() - require.True(t, ok) - require.Equal(t, spec_svs.VectorTypePartial, vt) - require.Less(t, len(pub.StateVector.Entries), len(full.StateVector.Entries)) + require.True(t, pub.IsPartial()) + require.False(t, pub.IsFull()) + pubSv := pub.GetStateVector() + fullSv := full.GetStateVector() + require.Less(t, len(pubSv.Entries), len(fullSv.Entries)) periodic := buildSvsDataForSend(svsSendInput{ State: m, Reason: syncSendPeriodic, Threshold: threshold, Sender: alice, }) - vt, ok = periodic.VectorType.Get() - require.True(t, ok) - require.Equal(t, spec_svs.VectorTypeFull, vt) + require.True(t, periodic.IsFull()) + require.False(t, periodic.IsPartial()) } func TestOnReceivePartialMergesPresentEntriesOnly(t *testing.T) { @@ -195,9 +191,9 @@ func TestOnReceivePartialMergesPresentEntriesOnly(t *testing.T) { partialSv := bobOnly.Encode(func(s uint64) uint64 { return s }) s.onReceiveStateVector(svSyncRecvSvArgs{ - sv: partialSv, - vectorType: optional.Some(spec_svs.VectorTypePartial), - mhash: ComputeMembershipHash(bobOnly), + sv: partialSv, + partial: true, + mhash: ComputeMembershipHash(bobOnly), }) require.Len(t, updates, 1) @@ -268,17 +264,19 @@ func TestSvsDataInlineTLV(t *testing.T) { sv := m.Encode(func(s uint64) uint64 { return s }) original := &spec_svs.SvsData{ - MemberSetHash: ComputeMembershipHash(m), - VectorType: optional.Some(spec_svs.VectorTypeFull), - StateVector: sv, + MemberSetHash: ComputeMembershipHash(m), + FullStateVector: &spec_svs.FullStateVector{StateVector: sv}, } wire := original.Encode().Join() parsed, err := spec_svs.ParseSvsData(enc.NewBufferView(wire), false) require.NoError(t, err) + require.True(t, parsed.IsFull()) + require.False(t, parsed.IsPartial()) require.Equal(t, original.MemberSetHash, parsed.MemberSetHash) - require.Equal(t, original.VectorType, parsed.VectorType) - require.Equal(t, original.StateVector.Entries[0].Name.String(), parsed.StateVector.Entries[0].Name.String()) + origSv := original.GetStateVector() + parsedSv := parsed.GetStateVector() + require.Equal(t, origSv.Entries[0].Name.String(), parsedSv.Entries[0].Name.String()) } func TestSvsDataPublishTLV(t *testing.T) { @@ -287,32 +285,37 @@ func TestSvsDataPublishTLV(t *testing.T) { ref := tu.NoErr(enc.NameFromStr("/ndn/svs/alice/100/32=sv/1")) mhash := make([]byte, 32) - original := &spec_svs.SvsData{ - MemberSetHash: mhash, - SvsDataRef: ref, - } + original := &spec_svs.SvsData{MemberSetHash: mhash, SvsDataRef: ref} wire := original.Encode().Join() parsed, err := spec_svs.ParseSvsData(enc.NewBufferView(wire), false) require.NoError(t, err) - require.Equal(t, mhash, parsed.MemberSetHash) + require.Equal(t, spec_svs.VectorKindNone, parsed.Kind()) require.Equal(t, ref.String(), parsed.SvsDataRef.String()) - require.Nil(t, parsed.StateVector) - require.False(t, parsed.VectorType.IsSet()) + require.Nil(t, parsed.GetStateVector()) + require.Equal(t, mhash, parsed.MemberSetHash) } -func TestSvsDataLegacyParse(t *testing.T) { +func TestSvsDataPartialTLV(t *testing.T) { tu.SetT(t) m := NewSvMap[uint64](0) m.Set(tu.NoErr(enc.NameFromStr("/ndn/alice")).TlvStr(), 100, 1) - legacy := &spec_svs.SvsData{StateVector: m.Encode(func(s uint64) uint64 { return s })} - wire := legacy.Encode().Join() + sv := m.Encode(func(s uint64) uint64 { return s }) + + original := &spec_svs.SvsData{ + MemberSetHash: ComputeMembershipHash(m), + PartialStateVector: &spec_svs.PartialStateVector{StateVector: sv}, + } + wire := original.Encode().Join() parsed, err := spec_svs.ParseSvsData(enc.NewBufferView(wire), false) require.NoError(t, err) - require.Nil(t, parsed.MemberSetHash) - require.NotNil(t, parsed.StateVector) + require.True(t, parsed.IsPartial()) + require.False(t, parsed.IsFull()) + require.Equal(t, spec_svs.VectorKindPartial, parsed.Kind()) + require.NotNil(t, parsed.GetStateVector()) + require.Equal(t, ComputeMembershipHash(m), parsed.MemberSetHash) } // --- pull / recovery tests --- @@ -349,17 +352,18 @@ func TestBuildPublishSvsData(t *testing.T) { ref := tu.NoErr(enc.NameFromStr("/ndn/svs/alice/1700000000/32=sv/999")) data := buildPublishSvsData(m, ref) + require.Equal(t, spec_svs.VectorKindNone, data.Kind()) require.Equal(t, ComputeMembershipHash(m), data.MemberSetHash) require.True(t, ref.Equal(data.SvsDataRef)) - require.Nil(t, data.StateVector) - require.False(t, data.VectorType.IsSet()) + require.Nil(t, data.GetStateVector()) wire := data.Encode().Join() parsed, err := spec_svs.ParseSvsData(enc.NewBufferView(wire), false) require.NoError(t, err) - require.Equal(t, data.MemberSetHash, parsed.MemberSetHash) + require.Equal(t, spec_svs.VectorKindNone, parsed.Kind()) + require.Equal(t, ComputeMembershipHash(m), parsed.MemberSetHash) require.True(t, ref.Equal(parsed.SvsDataRef)) - require.Nil(t, parsed.StateVector) + require.Nil(t, parsed.GetStateVector()) } func TestShouldUsePublishPull(t *testing.T) { @@ -368,9 +372,8 @@ func TestShouldUsePublishPull(t *testing.T) { m := testSvMapAliceBob() sv := m.Encode(func(s uint64) uint64 { return s }) fullSize := len((&spec_svs.SvsData{ - MemberSetHash: ComputeMembershipHash(m), - VectorType: optional.Some(spec_svs.VectorTypeFull), - StateVector: sv, + MemberSetHash: ComputeMembershipHash(m), + FullStateVector: &spec_svs.FullStateVector{StateVector: sv}, }).Encode().Join()) usePublish, _, _ := shouldUsePublishPull(syncSendPublication, fullSize-1, m) @@ -403,9 +406,8 @@ func TestParseFullVectorContentRejectsBadMhash(t *testing.T) { m := testSvMapAliceBob() inline := &spec_svs.SvsData{ - MemberSetHash: []byte("not-a-valid-mhash-padding-000000"), - VectorType: optional.Some(spec_svs.VectorTypeFull), - StateVector: m.Encode(func(s uint64) uint64 { return s }), + MemberSetHash: []byte("not-a-valid-mhash-padding-000000"), + FullStateVector: &spec_svs.FullStateVector{StateVector: m.Encode(func(s uint64) uint64 { return s })}, } wire := inline.Encode().Join() @@ -413,14 +415,14 @@ func TestParseFullVectorContentRejectsBadMhash(t *testing.T) { require.Error(t, err) } -func TestParseFullVectorContentRejectsMissingVectorType(t *testing.T) { +func TestParseFullVectorContentRejectsPartial(t *testing.T) { tu.SetT(t) m := testSvMapAliceBob() + // Wrong wire TLV: PartialStateVector instead of FullStateVector. inline := &spec_svs.SvsData{ - MemberSetHash: ComputeMembershipHash(m), - VectorType: optional.None[uint64](), - StateVector: m.Encode(func(s uint64) uint64 { return s }), + MemberSetHash: ComputeMembershipHash(m), + PartialStateVector: &spec_svs.PartialStateVector{StateVector: m.Encode(func(s uint64) uint64 { return s })}, } wire := inline.Encode().Join() @@ -433,9 +435,8 @@ func TestParseFullVectorContentRejectsMissingMhash(t *testing.T) { m := testSvMapAliceBob() inline := &spec_svs.SvsData{ - MemberSetHash: nil, - VectorType: optional.Some(spec_svs.VectorTypeFull), - StateVector: m.Encode(func(s uint64) uint64 { return s }), + MemberSetHash: nil, + FullStateVector: &spec_svs.FullStateVector{StateVector: m.Encode(func(s uint64) uint64 { return s })}, } wire := inline.Encode().Join() @@ -448,16 +449,15 @@ func TestParseFullVectorContent(t *testing.T) { m := testSvMapAliceBob() inline := &spec_svs.SvsData{ - MemberSetHash: ComputeMembershipHash(m), - VectorType: optional.Some(spec_svs.VectorTypeFull), - StateVector: m.Encode(func(s uint64) uint64 { return s }), + MemberSetHash: ComputeMembershipHash(m), + FullStateVector: &spec_svs.FullStateVector{StateVector: m.Encode(func(s uint64) uint64 { return s })}, } wire := inline.Encode().Join() parsed, err := parseFullVectorContent(wire) require.NoError(t, err) require.Equal(t, inline.MemberSetHash, parsed.MemberSetHash) - require.Len(t, parsed.StateVector.Entries, 2) + require.Len(t, parsed.GetStateVector().Entries, 2) } func TestOnPulledFullVectorMergesState(t *testing.T) { @@ -480,9 +480,8 @@ func TestOnPulledFullVectorMergesState(t *testing.T) { remote := testSvMapAliceBob() content := (&spec_svs.SvsData{ - MemberSetHash: ComputeMembershipHash(remote), - VectorType: optional.Some(spec_svs.VectorTypeFull), - StateVector: remote.Encode(func(s uint64) uint64 { return s }), + MemberSetHash: ComputeMembershipHash(remote), + FullStateVector: &spec_svs.FullStateVector{StateVector: remote.Encode(func(s uint64) uint64 { return s })}, }).Encode().Join() go func() { @@ -501,15 +500,13 @@ func TestEncodeSyncDataPublishMode(t *testing.T) { m := testSvMapAliceBob() fullSize := len((&spec_svs.SvsData{ - MemberSetHash: ComputeMembershipHash(m), - VectorType: optional.Some(spec_svs.VectorTypeFull), - StateVector: m.Encode(func(s uint64) uint64 { return s }), + MemberSetHash: ComputeMembershipHash(m), + FullStateVector: &spec_svs.FullStateVector{StateVector: m.Encode(func(s uint64) uint64 { return s })}, }).Encode().Join()) usePublish, _, _ := shouldUsePublishPull(syncSendPeriodic, fullSize-1, m) require.True(t, usePublish) publish := buildPublishSvsData(m, tu.NoErr(enc.NameFromStr("/ndn/svs/alice/1/32=sv/2"))) - require.Nil(t, publish.StateVector) - vt, ok := publish.VectorType.Get() - require.False(t, ok || vt == spec_svs.VectorTypePartial) + require.Equal(t, spec_svs.VectorKindNone, publish.Kind()) + require.Nil(t, publish.GetStateVector()) } From e6ce78e5f4ab89955976718a99e86866e8139b63 Mon Sep 17 00:00:00 2001 From: Taranum Wasu Date: Fri, 24 Jul 2026 22:34:30 +0530 Subject: [PATCH 15/17] lint+refactor: fix goimports and rename svs v3 package to v4 goimports had two complaints on the prior commit: * std/ndn/svs/v3/accessors.go was missing a trailing newline. * std/sync/svs_pull.go had a misaligned struct literal in shouldUsePublishPull (MemberSetHash not column-aligned with FullStateVector after the SvsData tagged-union reshuffle). Rename the wire-spec Go package std/ndn/svs/v3 -> std/ndn/svs/v4 to match the spec filename (svs-v4.md) and the new wire behaviour (mhash, distinct FULL / PARTIAL / SvsDataRef TLVs). The folder previously advertised itself as the v3 package, but the wire format it now produces is v4. Per Adam's review note: keeping "v3" in the import path would be misleading for any downstream consumer pulling this ndnd build. * git mv std/ndn/svs/v3 std/ndn/svs/v4 * update import path in 9 caller files (std/sync/*, dv/dv/*, std/ndn/svs_ps/*) and the one in-tree comment reference. * struct names, TLV type numbers, and the spec text are unchanged - only the import path moves. Tests: std/sync, std/object/storage, std/security pass. goimports -l clean on all touched files. --- dv/dv/advert_sync.go | 2 +- std/ndn/svs/{v3 => v4}/accessors.go | 2 +- std/ndn/svs/{v3 => v4}/definitions.go | 0 std/ndn/svs/{v3 => v4}/zz_generated.go | 0 std/ndn/svs_ps/definitions.go | 2 +- std/ndn/svs_ps/zz_generated.go | 2 +- std/sync/svs.go | 4 ++-- std/sync/svs_encode.go | 2 +- std/sync/svs_map.go | 2 +- std/sync/svs_membership_hash.go | 2 +- std/sync/svs_pull.go | 4 ++-- std/sync/svs_test.go | 2 +- 12 files changed, 12 insertions(+), 12 deletions(-) rename std/ndn/svs/{v3 => v4}/accessors.go (99%) rename std/ndn/svs/{v3 => v4}/definitions.go (100%) rename std/ndn/svs/{v3 => v4}/zz_generated.go (100%) diff --git a/dv/dv/advert_sync.go b/dv/dv/advert_sync.go index b65e50a0..8fc43360 100644 --- a/dv/dv/advert_sync.go +++ b/dv/dv/advert_sync.go @@ -9,7 +9,7 @@ import ( "github.com/named-data/ndnd/std/log" "github.com/named-data/ndnd/std/ndn" spec "github.com/named-data/ndnd/std/ndn/spec_2022" - spec_svs "github.com/named-data/ndnd/std/ndn/svs/v3" + spec_svs "github.com/named-data/ndnd/std/ndn/svs/v4" "github.com/named-data/ndnd/std/object/storage" "github.com/named-data/ndnd/std/types/optional" "github.com/named-data/ndnd/std/utils" diff --git a/std/ndn/svs/v3/accessors.go b/std/ndn/svs/v4/accessors.go similarity index 99% rename from std/ndn/svs/v3/accessors.go rename to std/ndn/svs/v4/accessors.go index 170136a5..06cdef63 100644 --- a/std/ndn/svs/v3/accessors.go +++ b/std/ndn/svs/v4/accessors.go @@ -51,4 +51,4 @@ func (d *SvsData) IsPartial() bool { // IsFull reports whether the embedded StateVector is a FULL State Vector. func (d *SvsData) IsFull() bool { return d.FullStateVector != nil -} \ No newline at end of file +} diff --git a/std/ndn/svs/v3/definitions.go b/std/ndn/svs/v4/definitions.go similarity index 100% rename from std/ndn/svs/v3/definitions.go rename to std/ndn/svs/v4/definitions.go diff --git a/std/ndn/svs/v3/zz_generated.go b/std/ndn/svs/v4/zz_generated.go similarity index 100% rename from std/ndn/svs/v3/zz_generated.go rename to std/ndn/svs/v4/zz_generated.go diff --git a/std/ndn/svs_ps/definitions.go b/std/ndn/svs_ps/definitions.go index 2e3f879c..900ed9d4 100644 --- a/std/ndn/svs_ps/definitions.go +++ b/std/ndn/svs_ps/definitions.go @@ -3,7 +3,7 @@ package svs_ps import ( enc "github.com/named-data/ndnd/std/encoding" - "github.com/named-data/ndnd/std/ndn/svs/v3" + "github.com/named-data/ndnd/std/ndn/svs/v4" ) type InstanceState struct { diff --git a/std/ndn/svs_ps/zz_generated.go b/std/ndn/svs_ps/zz_generated.go index c16c1abc..2fdaae5a 100644 --- a/std/ndn/svs_ps/zz_generated.go +++ b/std/ndn/svs_ps/zz_generated.go @@ -5,7 +5,7 @@ import ( "io" enc "github.com/named-data/ndnd/std/encoding" - "github.com/named-data/ndnd/std/ndn/svs/v3" + "github.com/named-data/ndnd/std/ndn/svs/v4" ) type InstanceStateEncoder struct { diff --git a/std/sync/svs.go b/std/sync/svs.go index aa0b7ea1..98ffbe05 100644 --- a/std/sync/svs.go +++ b/std/sync/svs.go @@ -13,7 +13,7 @@ import ( "github.com/named-data/ndnd/std/log" "github.com/named-data/ndnd/std/ndn" spec "github.com/named-data/ndnd/std/ndn/spec_2022" - spec_svs "github.com/named-data/ndnd/std/ndn/svs/v3" + spec_svs "github.com/named-data/ndnd/std/ndn/svs/v4" "github.com/named-data/ndnd/std/types/optional" "github.com/named-data/ndnd/std/utils" ) @@ -21,7 +21,7 @@ import ( // syncVectorThreshold is the max embedded SvsData size (bytes) above // which the sender switches to PARTIAL (on publication) or publish+pull // (on periodic sync and recovery). SVS v4 always emits `mhash` and one of -// FullStateVector/PartialStateVector on the wire (see std/ndn/svs/v3). +// FullStateVector/PartialStateVector on the wire (see std/ndn/svs/v4). const syncVectorThreshold = 1200 type SvSync struct { diff --git a/std/sync/svs_encode.go b/std/sync/svs_encode.go index 8047207b..0c8d0c48 100644 --- a/std/sync/svs_encode.go +++ b/std/sync/svs_encode.go @@ -7,7 +7,7 @@ import ( "time" enc "github.com/named-data/ndnd/std/encoding" - spec_svs "github.com/named-data/ndnd/std/ndn/svs/v3" + spec_svs "github.com/named-data/ndnd/std/ndn/svs/v4" ) // syncSendReason distinguishes why a Sync Interest is being sent. diff --git a/std/sync/svs_map.go b/std/sync/svs_map.go index 3be8e1eb..f19be637 100644 --- a/std/sync/svs_map.go +++ b/std/sync/svs_map.go @@ -7,7 +7,7 @@ import ( enc "github.com/named-data/ndnd/std/encoding" "github.com/named-data/ndnd/std/log" - spec_svs "github.com/named-data/ndnd/std/ndn/svs/v3" + spec_svs "github.com/named-data/ndnd/std/ndn/svs/v4" ) // Map representation of the state vector. diff --git a/std/sync/svs_membership_hash.go b/std/sync/svs_membership_hash.go index 86520e62..d0aae4d3 100644 --- a/std/sync/svs_membership_hash.go +++ b/std/sync/svs_membership_hash.go @@ -4,7 +4,7 @@ import ( "crypto/sha256" "slices" - spec_svs "github.com/named-data/ndnd/std/ndn/svs/v3" + spec_svs "github.com/named-data/ndnd/std/ndn/svs/v4" ) // ComputeMembershipHash returns the membership hash over all (Name, BootstrapTime) pairs in state. diff --git a/std/sync/svs_pull.go b/std/sync/svs_pull.go index e9ad0829..dd37a2df 100644 --- a/std/sync/svs_pull.go +++ b/std/sync/svs_pull.go @@ -9,7 +9,7 @@ import ( "github.com/named-data/ndnd/std/log" "github.com/named-data/ndnd/std/ndn" spec "github.com/named-data/ndnd/std/ndn/spec_2022" - spec_svs "github.com/named-data/ndnd/std/ndn/svs/v3" + spec_svs "github.com/named-data/ndnd/std/ndn/svs/v4" ) const ( @@ -95,7 +95,7 @@ func shouldUsePublishPull(reason syncSendReason, threshold int, state SvMap[uint } sv := state.Encode(func(seq uint64) uint64 { return seq }) full := &spec_svs.SvsData{ - MemberSetHash: ComputeMembershipHash(state), + MemberSetHash: ComputeMembershipHash(state), FullStateVector: &spec_svs.FullStateVector{StateVector: sv}, } wire := full.Encode().Join() diff --git a/std/sync/svs_test.go b/std/sync/svs_test.go index e27da35b..26ea675f 100644 --- a/std/sync/svs_test.go +++ b/std/sync/svs_test.go @@ -10,7 +10,7 @@ import ( enc "github.com/named-data/ndnd/std/encoding" "github.com/named-data/ndnd/std/ndn" spec "github.com/named-data/ndnd/std/ndn/spec_2022" - spec_svs "github.com/named-data/ndnd/std/ndn/svs/v3" + spec_svs "github.com/named-data/ndnd/std/ndn/svs/v4" sig "github.com/named-data/ndnd/std/security/signer" "github.com/named-data/ndnd/std/types/optional" tu "github.com/named-data/ndnd/std/utils/testutils" From bbce41018f35894a3ea695e9d7da5497ee789e0b Mon Sep 17 00:00:00 2001 From: Taranum Wasu Date: Sat, 25 Jul 2026 00:08:09 +0530 Subject: [PATCH 16/17] =?UTF-8?q?e2e:=20bump=20DV=20convergence=20+=20put-?= =?UTF-8?q?then-cat=20windows;=20revert=20client=20retry=20bump=20Per=20Ad?= =?UTF-8?q?am's=20review=20(PR=20#190,=20comment=203641275220):=20the=20DV?= =?UTF-8?q?=20convergence=20flake=20on=20the=2052-node=20sprint=20topology?= =?UTF-8?q?=20should=20be=20fixed=20by=20giving=20the=20test=20more=20time?= =?UTF-8?q?=20to=20converge,=20not=20by=20relaxing=20the=20client's=20meta?= =?UTF-8?q?data/prefix=20fetch=20retry=20budget.=20Reverts=20std/object/cl?= =?UTF-8?q?ient=5Fconsume.go=20to=20Retries:3=20/=20Lifetime:1s=20on=20bot?= =?UTF-8?q?h=20fetchMetadata=20and=20fetchDataByPrefix=20(the=20values=20u?= =?UTF-8?q?sed=20before=20commit=209340c00).=20The=20library=20default=20i?= =?UTF-8?q?s=20now=20consistent=20across=20all=20consumers,=20not=20just?= =?UTF-8?q?=20the=20sprint=20e2e.=20Bumps=20the=20e2e=20wait=20times=20ins?= =?UTF-8?q?tead:=20=20=20*=20dv=5Futil.converge=20deadline:=2030s=20->=209?= =?UTF-8?q?0s=20=20=20=20=20Worst=20observed=20post-DV-startup=20propagati?= =?UTF-8?q?on=20in=20CI=20is=20~20s=20=20=20=20=20(9340c00=20commit=20mess?= =?UTF-8?q?age);=2090s=20gives=20a=204x=20safety=20margin.=20=20=20*=20tes?= =?UTF-8?q?t=5F001=20scenario=5Fndnd=5Ffw=20post-put=20sleep:=2030s=20->?= =?UTF-8?q?=2090s=20=20=20=20=20The=20cat-phase=20failure=20mode=20in=20CI?= =?UTF-8?q?=20was=20a=20metadata=20Timeout=20=20=20=20=20~13s=20after=20th?= =?UTF-8?q?e=20put=20started=20(Time=3Dcat=20=E2=89=88=20Put+30s=20in=20fa?= =?UTF-8?q?iling=20=20=20=20=20runs).=20After=208=20simultaneous=20puts,?= =?UTF-8?q?=20DV=20has=20to=20re-converge=20for=20=20=20=20=20the=20new=20?= =?UTF-8?q?--expose=20prefixes;=2090s=20of=20post-put=20quiescence=20is=20?= =?UTF-8?q?=20=20=20=20enough=20for=20the=20late=20"Reset"=20storm=20to=20?= =?UTF-8?q?fully=20drain.=20Test=20runtime=20impact:=20scenario=5Fndnd=5Ff?= =?UTF-8?q?w=20goes=20from=20~5min=20to=20~7min.=20Tests:=20std/object,=20?= =?UTF-8?q?std/sync,=20std/security=20pass=20locally.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- e2e/dv_util.py | 2 +- e2e/test_001.py | 2 +- std/object/client_consume.go | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/e2e/dv_util.py b/e2e/dv_util.py index c3c020f1..86e6eed7 100644 --- a/e2e/dv_util.py +++ b/e2e/dv_util.py @@ -15,7 +15,7 @@ def setup(ndn: Minindn, network=DEFAULT_NETWORK) -> None: info('Starting ndn-dv on nodes\n') AppManager(ndn, ndn.net.hosts, NDNd_DV, network=network) -def converge(nodes: list[Node], deadline=30, network=DEFAULT_NETWORK, use_nfdc=False) -> int: +def converge(nodes: list[Node], deadline=90, network=DEFAULT_NETWORK, use_nfdc=False) -> int: info('Waiting for routing to converge\n') start = time.time() while time.time() - start < deadline: diff --git a/e2e/test_001.py b/e2e/test_001.py index b933ca9d..6123da34 100644 --- a/e2e/test_001.py +++ b/e2e/test_001.py @@ -43,7 +43,7 @@ def scenario(ndn: Minindn, fw=None, network='/minindn'): node.cmd(cmd) info('Waiting for put to complete\n') - time.sleep(30) + time.sleep(90) for node in cat_nodes: put_node = random.choice(put_nodes) diff --git a/std/object/client_consume.go b/std/object/client_consume.go index cf597715..78262002 100644 --- a/std/object/client_consume.go +++ b/std/object/client_consume.go @@ -117,9 +117,9 @@ func (c *Client) fetchMetadata( Config: &ndn.InterestConfig{ CanBePrefix: true, MustBeFresh: true, - Lifetime: optional.Some(time.Millisecond * 2000), + Lifetime: optional.Some(time.Millisecond * 1000), }, - Retries: 5, // TODO: configurable (sprint e2e needs ~10s budget for DV startup reset storm) + Retries: 3, TryStore: utils.If(tryStore, c.store, nil), Callback: func(args ndn.ExpressCallbackArgs) { if args.Result == ndn.InterestResultError { @@ -174,9 +174,9 @@ func (c *Client) fetchDataByPrefix( Config: &ndn.InterestConfig{ CanBePrefix: true, MustBeFresh: true, - Lifetime: optional.Some(time.Millisecond * 2000), + Lifetime: optional.Some(time.Millisecond * 1000), }, - Retries: 5, // TODO: configurable (sprint e2e needs ~10s budget for DV startup reset storm) + Retries: 3, TryStore: utils.If(tryStore, c.store, nil), Callback: func(args ndn.ExpressCallbackArgs) { if args.Result == ndn.InterestResultError { From a9a0de7a002ae82a90a6cd31249e914f2e7d9fc3 Mon Sep 17 00:00:00 2001 From: Taranum Wasu Date: Sat, 25 Jul 2026 04:53:15 +0530 Subject: [PATCH 17/17] sync: tighten direct-form exclusivity and hoist mhash in PARTIAL encode Two Copilot review nits on PR #190, both approved by Adam. * svs.go onSyncData (the direct-form gate): the previous check `!IsFull() && !IsPartial()` only rejected packets that had neither form. A malformed packet carrying both FullStateVector and PartialStateVector, or carrying SvsDataRef alongside an embedded vector, was being processed. Tighten to `IsFull() == IsPartial() || len(SvsDataRef) > 0` so only the exactly-one-of-{FULL, PARTIAL}-and-no-ref shape is accepted. * svs_encode.go encodePartialStateVector: ComputeMembershipHash(state) is constant for a given state but was being recomputed for the sender-only baseline check and again for every trial entry. Hoist to a single `mhash` local at the top of the function. PR description also updated to drop the now-stale references to the previous VectorType discriminator, the v3 package path, the "Retries 3 -> 5" / "Lifetime 1s -> 2s" claim, and the legacy `Threshold <= 0` mode. The new description matches the wire, the package path, and the e2e wait windows that are actually shipped. Tests: std/sync passes. --- std/sync/svs.go | 4 ++-- std/sync/svs_encode.go | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/std/sync/svs.go b/std/sync/svs.go index 98ffbe05..24940873 100644 --- a/std/sync/svs.go +++ b/std/sync/svs.go @@ -644,8 +644,8 @@ func (s *SvSync) onSyncData(dataWire enc.Wire) { // [Spec] Direct form must carry exactly one of FullStateVector / // PartialStateVector, and a 32-byte mhash. The wire TLV replaces // the previous VectorType discriminator. - if !params.IsFull() && !params.IsPartial() { - log.Warn(s, "onSyncInterest inline SvsData missing direct form") + if params.IsFull() == params.IsPartial() || len(params.SvsDataRef) > 0 { + log.Warn(s, "onSyncInterest inline SvsData has invalid direct form") return } if len(mhash) != 32 { diff --git a/std/sync/svs_encode.go b/std/sync/svs_encode.go index 0c8d0c48..2db361a5 100644 --- a/std/sync/svs_encode.go +++ b/std/sync/svs_encode.go @@ -83,6 +83,7 @@ func buildSvsDataForSend(in svsSendInput) *spec_svs.SvsData { func encodePartialStateVector(state SvMap[uint64], opts PartialEncodeOpts) *spec_svs.StateVector { seq := func(v uint64) uint64 { return v } senderHash := opts.Sender.TlvStr() + mhash := ComputeMembershipHash(state) senderEntry := state.encodeNameEntry(opts.Sender, seq) if senderEntry == nil { @@ -92,7 +93,7 @@ func encodePartialStateVector(state SvMap[uint64], opts PartialEncodeOpts) *spec // Sender-only baseline must always fit when possible. baseline := &spec_svs.StateVector{Entries: []*spec_svs.StateVectorEntry{senderEntry}} baselineData := &spec_svs.SvsData{ - MemberSetHash: ComputeMembershipHash(state), + MemberSetHash: mhash, PartialStateVector: &spec_svs.PartialStateVector{StateVector: baseline}, } if len(baselineData.Encode().Join()) > opts.Threshold { @@ -120,7 +121,7 @@ func encodePartialStateVector(state SvMap[uint64], opts PartialEncodeOpts) *spec sortPartialTail(trial) trialSv := &spec_svs.StateVector{Entries: trial} trialData := &spec_svs.SvsData{ - MemberSetHash: ComputeMembershipHash(state), + MemberSetHash: mhash, PartialStateVector: &spec_svs.PartialStateVector{StateVector: trialSv}, } if len(trialData.Encode().Join()) > opts.Threshold {