diff --git a/lib/replicator.js b/lib/replicator.js index bf1eb9e7..af3ed1b4 100644 --- a/lib/replicator.js +++ b/lib/replicator.js @@ -285,6 +285,10 @@ class RangeRequest extends Attachable { this.ranges[rangeIndex] = h } + if (!this.resolved && this.replicator._updatesRunning) { + this.replicator._updatesQueued = true + } + if (this.end === -1) { this.replicator._alwaysLatestBlock-- } @@ -1541,8 +1545,8 @@ class Peer { } _requestSeek(s) { - // if replicator is updating the seeks etc, bail and wait for it to drain - if (this.replicator._updatesPending > 0) return false + // if replicator is updating the seeks, bail and wait for it to drain + if (this.replicator._updatingSeeks) return false if (this.replicator.pushOnly) return false const { length, fork } = this.core.state @@ -2028,7 +2032,9 @@ module.exports = class Replicator { this._hadPeers = false this._active = 0 this._ifAvailable = 0 - this._updatesPending = 0 + this._updatingSeeks = false + this._updatesRunning = false + this._updatesQueued = false this._applyingReorg = null this._manifestPeer = null this._notDownloadingLinger = notDownloadingLinger @@ -2614,55 +2620,118 @@ module.exports = class Replicator { } } + _updateRanges(index, limit) { + index = Math.min(index, this._ranges.length - 1) + + let checked = 0 + let resolved = 0 + let updateAll = false + let remaining = Math.min(limit, this._ranges.length) + + while (remaining-- > 0 && this._ranges.length > 0) { + if (index >= this._ranges.length) index = 0 + + const r = this._ranges[index] + + clampRange(this.core, r) + checked++ + + if (r.end !== -1 && r.start >= r.end) { + this._resolveRangeRequest(r) + resolved++ + // Crossing into the capped window exposes another range to peer scheduling. + if (this._ranges.length === MAX_RANGES) updateAll = true + } else { + index++ + } + } + + index = this._ranges.length === 0 ? 0 : index % this._ranges.length + + return { checked, resolved, index, updateAll } + } + // "slow" updates here - async but not allowed to ever throw async _updateNonPrimary(updateAll) { - // Check if running, if so skip it and the running one will issue another update for us (debounce) - while (++this._updatesPending === 1) { - let len = Math.min(MAX_RANGES, this._ranges.length) + this._updatesQueued = true + if (this._updatesRunning) { + if (this._inflight.idle || updateAll) this.queueUpdateAll() + return + } - for (let i = 0; i < len; i++) { - const r = this._ranges[i] + this._updatesRunning = true - clampRange(this.core, r) + while (this._updatesQueued) { + this._updatesQueued = false - if (r.end !== -1 && r.start >= r.end) { - this._resolveRangeRequest(r) - i-- - if (len > this._ranges.length) len-- - if (this._ranges.length === MAX_RANGES) updateAll = true - } - } + let checkedSinceResolve = 0 + let rangeIndex = 0 + const drain = this._inflight.idle || updateAll + const checkedSeeks = new Set() - for (let i = 0; i < this._seeks.length; i++) { - const s = this._seeks[i] + while (true) { + const limit = Math.min(MAX_RANGES, this._ranges.length - checkedSinceResolve) + const { + checked, + resolved, + index, + updateAll: resultUpdateAll + } = this._updateRanges(rangeIndex, limit) + rangeIndex = index - let err = null - let res = null + if (resultUpdateAll) updateAll = true - try { - res = await s.seeker.update() - } catch (error) { - err = error - } + if (resolved > 0) checkedSinceResolve = 0 + else checkedSinceResolve += checked + + const continueRanges = checked > 0 && checkedSinceResolve < this._ranges.length && drain + + this._updatingSeeks = true + + let checkedSeek = false + for (const s of this._seeks.slice()) { + if (checkedSeeks.has(s) || !this._seeks.includes(s)) continue + checkedSeeks.add(s) + checkedSeek = true + + let err = null + let res = null + + try { + res = await s.seeker.update() + } catch (error) { + err = error + } - if (!res && !err) continue + const seekIndex = this._seeks.indexOf(s) + if (seekIndex === -1 || (!res && !err)) continue - if (i < this._seeks.length - 1) this._seeks[i] = this._seeks.pop() - else this._seeks.pop() + const h = this._seeks.pop() + if (h !== s) this._seeks[seekIndex] = h - i-- + if (err) s.reject(err) + else s.resolve(res) + } + + this._updatingSeeks = false + + if ( + (!continueRanges || (checkedSeek && this._seeks.length > 0)) && + (this._inflight.idle || updateAll) + ) { + this.queueUpdateAll() + } + if (!continueRanges) break - if (err) s.reject(err) - else s.resolve(res) + await yieldToLoop() + if (this.destroyed) break } - // No additional updates scheduled - break - if (--this._updatesPending === 0) break - // Debounce the additional updates - continue - this._updatesPending = 0 + if (this.destroyed) break } - if (this._inflight.idle || updateAll) this.queueUpdateAll() + this._updatingSeeks = false + this._updatesRunning = false } _clearRequest(peer, req) { @@ -3398,6 +3467,10 @@ function incrementRx(stats1, stats2) { function noop() {} +function yieldToLoop() { + return new Promise((resolve) => setImmediate(resolve)) +} + function backoff(times) { const sleep = times < 2 ? 200 : times < 5 ? 500 : times < 40 ? 1000 : 5000 return new Promise((resolve) => setTimeout(resolve, sleep)) diff --git a/test/replicate.js b/test/replicate.js index da5b4572..0370bd03 100644 --- a/test/replicate.js +++ b/test/replicate.js @@ -2937,6 +2937,348 @@ test('delayed updateAll timer doesnt keep event loop alive', async function (t) t.is(r._updateAllBump, null, 'timer reset to null') }) +test('completed range exposes capped range to existing peer', async function (t) { + const writer = await create(t) + const totalLength = 65 + + await writer.append(new Array(totalLength).fill('a')) + + const firstPeer = await create(t, writer.key) + let streams = replicate(writer, firstPeer, t) + await firstPeer.get(0) + await unreplicate(streams) + + const cappedPeer = await create(t, writer.key) + streams = replicate(writer, cappedPeer, t) + await cappedPeer.get(totalLength - 1) + await unreplicate(streams) + + const downloader = await create(t, writer.key) + + t.absent(await firstPeer.has(totalLength - 1), 'first peer does not have capped block') + + const random = Math.random + Math.random = () => 0.999 + t.teardown(() => { + Math.random = random + }) + + const cappedDone = downloader + .download({ start: totalLength - 1, end: totalLength }) + .done() + .then(() => true, noop) + const firstDone = downloader.download({ start: 0, end: 1 }).done() + + for (let i = 1; i < totalLength - 1; i++) { + downloader.download({ start: i, end: i + 1 }) + } + + const replicator = downloader.core.replicator + t.is(replicator._ranges.length, totalLength, 'one range is outside the scan window') + + // Model unrelated inflight work so the idle fallback cannot trigger the peer update. + replicator._inflight._active++ + t.teardown(() => { + replicator._inflight._active-- + }) + + const cappedPeerAdded = new Promise((resolve) => downloader.once('peer-add', resolve)) + replicate(cappedPeer, downloader, t) + + const remoteCappedPeer = await cappedPeerAdded + while (!remoteCappedPeer.remoteBitfield.get(totalLength - 1)) await eventFlush() + + t.absent(await downloader.has(totalLength - 1), 'capped range was skipped initially') + + replicate(firstPeer, downloader, t) + await firstDone + + let timer = null + const downloaded = await Promise.race([ + cappedDone, + new Promise((resolve) => { + timer = setTimeout(resolve, 1000, false) + }) + ]) + clearTimeout(timer) + + t.ok(downloaded, 'existing peer is revisited for the newly exposed range') +}) + +test('processing ranges does not block seeks', async function (t) { + const core = await create(t) + const totalLength = 129 + + await core.append(new Array(totalLength).fill('a')) + + const clone = await create(t, core.key, { eagerUpgrade: false }) + const peerAdded = new Promise((resolve) => clone.once('peer-add', resolve)) + replicate(core, clone, t) + + const peer = await peerAdded + peer.paused = true + while (!peer.remoteSynced) await eventFlush() + + for (let i = 0; i < totalLength; i++) { + clone + .download({ start: i, end: i + 1 }) + .done() + .catch(noop) + } + + const replicator = clone.core.replicator + t.is(replicator._ranges.length, totalLength, 'range backlog spans multiple chunks') + + const updateRanges = replicator._updateRanges + const requestSeek = peer._requestSeek + let updates = 0 + let checked = 0 + let checkedAtRequest = totalLength + let seek = null + + replicator._updateRanges = function (index, limit) { + const result = updateRanges.call(this, index, limit) + checked += result.checked + + if (++updates === 1) { + setImmediate(() => { + peer.paused = false + seek = clone.seek(Math.floor(core.byteLength / 2)) + }) + } + + return result + } + + peer._requestSeek = function (s) { + const sent = requestSeek.call(this, s) + if (sent && checkedAtRequest === totalLength) checkedAtRequest = checked + return sent + } + + t.teardown(() => { + replicator._updateRanges = updateRanges + peer._requestSeek = requestSeek + }) + + await replicator._updateNonPrimary(true) + await seek + + t.ok(checkedAtRequest < totalLength, 'seek was requested before the range scan completed') +}) + +test('range drain checks unresolved seeks once', async function (t) { + const core = await create(t) + const totalRanges = 129 + + for (let i = 0; i < totalRanges; i++) { + core.download({ start: i, end: i + 1 }) + } + + const replicator = core.core.replicator + let seekUpdates = 0 + const seek = replicator.addSeek([], { + update() { + seekUpdates++ + return null + } + }) + seek.promise.catch(noop) + + const queueUpdateAll = replicator.queueUpdateAll + let peerUpdates = 0 + replicator.queueUpdateAll = function () { + peerUpdates++ + } + + t.teardown(() => { + replicator.queueUpdateAll = queueUpdateAll + if (seek.context) replicator.cancel(seek) + }) + + await replicator._updateNonPrimary(true) + + t.is(seekUpdates, 1, 'seek was checked once across all range chunks') + t.is(peerUpdates, 2, 'peers were updated before and after the range drain') +}) + +test('cancelling seek during local update preserves other seeks', async function (t) { + const core = await create(t) + const replicator = core.core.replicator + + let release = null + let startedResolve = null + const started = new Promise((resolve) => { + startedResolve = resolve + }) + + const first = replicator.addSeek([], { + update() { + startedResolve() + return new Promise((resolve) => { + release = resolve + }) + } + }) + first.promise.catch(noop) + + const second = replicator.addSeek([], { + update() { + return [1, 0] + } + }) + + t.teardown(() => { + if (first.context) replicator.cancel(first) + if (second.context) replicator.cancel(second) + }) + + const updating = replicator._updateNonPrimary(true) + await started + + replicator.cancel(first) + release([0, 0]) + await updating + + const result = await Promise.race([second.promise, eventFlush()]) + t.alike(result, [1, 0], 'remaining seek resolved') +}) + +test('idle range completion drains past max range window', async function (t) { + const core = await create(t) + const clone = await create(t, core.key) + + const complete = [] + const totalLength = 150 + const availableStart = 100 + + await core.append(new Array(totalLength).fill('a')) + await core.clear(0, availableStart) + + const random = Math.random + Math.random = () => 0.999 + t.teardown(() => { + Math.random = random + }) + + replicate(core, clone, t) + + for (let i = 0; i < totalLength; i++) { + const done = clone.download({ start: i, end: i + 1 }).done() + if (i >= availableStart) complete.push(done) + else done.catch(noop) + } + + t.is(clone.core.replicator._ranges.length, totalLength, 'all ranges are pending') + + let timer = null + const resolved = await Promise.race([ + Promise.all(complete).then(() => true), + new Promise((resolve) => { + timer = setTimeout(resolve, 1000, false) + }) + ]) + clearTimeout(timer) + + t.ok(resolved, 'all locally complete ranges resolved') +}) + +test('idle range completion keeps draining if update queues during yield', async function (t) { + const core = await create(t) + + const totalLength = 150 + const availableStart = 100 + const replicator = core.core.replicator + let completed = 0 + + for (let i = 0; i < totalLength; i++) { + const done = core.download({ start: i, end: i + 1 }).done() + if (i >= availableStart) done.then(() => completed++, noop) + else done.catch(noop) + } + + t.is(replicator._ranges.length, totalLength, 'all ranges are pending') + + core.core._setBitfieldRanges(availableStart, totalLength, true) + + const updateRanges = replicator._updateRanges + let updates = 0 + + replicator._updateRanges = function (index, limit) { + const result = updateRanges.call(this, index, limit) + + if (++updates === 1) { + setImmediate(() => { + this._inflight._active++ + this._updateNonPrimary(false).catch(noop) + }) + } + + return result + } + + t.teardown(() => { + replicator._updateRanges = updateRanges + replicator._inflight._active = 0 + }) + + await replicator._updateNonPrimary(false) + replicator._inflight._active = 0 + await eventFlush() + + t.is(completed, totalLength - availableStart, 'all locally complete ranges resolved') +}) + +test('idle range completion restarts if ranges cancel during yield', async function (t) { + const core = await create(t) + const downloads = [] + + const totalLength = 100 + const availableStart = 80 + const cancelLength = totalLength - availableStart + let completed = 0 + + for (let i = 0; i < totalLength; i++) { + const download = core.download({ start: i, end: i + 1 }) + const done = download.done() + + if (i >= availableStart) done.then(() => completed++, noop) + else done.catch(noop) + + if (i < cancelLength) downloads.push(download) + } + + const replicator = core.core.replicator + t.is(replicator._ranges.length, totalLength, 'range backlog spans multiple chunks') + + core.core._setBitfieldRanges(availableStart, totalLength, true) + + const updateRanges = replicator._updateRanges + let updates = 0 + + replicator._updateRanges = function (index, limit) { + const result = updateRanges.call(this, index, limit) + + if (++updates === 1) { + setImmediate(() => { + // Swap complete ranges behind the saved cursor while the drain is suspended. + for (const download of downloads) download.destroy() + }) + } + + return result + } + + t.teardown(() => { + replicator._updateRanges = updateRanges + }) + + await replicator._updateNonPrimary(false) + await eventFlush() + + t.is(completed, cancelLength, 'all complete ranges resolved') +}) + async function createAndDownload(t, core) { const b = await create(t, core.key) replicate(core, b, t, { teardown: false })