diff --git a/core/libs/prx/libSceAgcDriver/CMakeLists.txt b/core/libs/prx/libSceAgcDriver/CMakeLists.txt index 6a76bea34fa..0994ddf0aac 100644 --- a/core/libs/prx/libSceAgcDriver/CMakeLists.txt +++ b/core/libs/prx/libSceAgcDriver/CMakeLists.txt @@ -410,6 +410,17 @@ if(BUILD_TESTING) set_tests_properties(agc_handoff_spin PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + + add_test_executable(agc_range_index_tests tests/RangeIndex.cpp) + target_include_directories(agc_range_index_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(agc_range_index_tests PRIVATE libc) + configure_windows_unwind(agc_range_index_tests) + add_test(NAME agc_range_index COMMAND agc_range_index_tests) + set_tests_properties(agc_range_index PROPERTIES TIMEOUT 30) + if(WIN32) + set_tests_properties(agc_range_index PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() endif() if(WIN32) diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp index 9622fb93a1b..9039274c786 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp @@ -72,6 +72,14 @@ void SetImportWatch(const Context& context, ImportWatch watch); // address)` in the import's buffer. const HostImport* HostImportFor(const Context& context, std::uint64_t address, std::size_t bytes); const HostImport* ExistingHostImportFor(const Context& context, std::uint64_t address, std::size_t bytes); +struct HostImportLookupCache { + const HostImport* import = nullptr; + std::uint64_t address = 0; + std::uint64_t bytes = 0; + std::uint64_t epoch = 0; + std::uint64_t inserts = 0; +}; +const HostImport* HostImportForCached(const Context& context, std::uint64_t address, std::size_t bytes, HostImportLookupCache& cache); void SetHostImportWindow(std::uint64_t bytes); // Tests: the window of one large registered range (64 MiB) and the import budget (0: the configured one). void SetHostImportRangeWindow(std::uint64_t bytes); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/RangeIndex.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/RangeIndex.hpp new file mode 100644 index 00000000000..5ed0cc7ac2d --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/RangeIndex.hpp @@ -0,0 +1,93 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace AgcDriver { + +template +class RangeIndex { +public: + void Insert(Item item, std::uint64_t begin, std::uint64_t end) { + const auto at = std::upper_bound(entries.begin(), entries.end(), begin, [](std::uint64_t value, const Entry& entry) { return value < entry.begin; }); + entries.insert(at, Entry{begin, end, item}); + stale = true; + } + + std::size_t Remove(Item item, std::uint64_t begin, bool all = false) { + auto first = std::lower_bound(entries.begin(), entries.end(), begin, [](const Entry& entry, std::uint64_t value) { return entry.begin < value; }); + std::size_t removed = 0; + while (first != entries.end() && first->begin == begin) { + if (first->item == item) { + first = entries.erase(first); + ++removed; + if (!all) break; + } else { + ++first; + } + } + if (removed != 0) stale = true; + return removed; + } + + template + bool Overlapping(std::uint64_t address, std::uint64_t bytes, Visit&& visit) { + if (entries.empty() || bytes == 0) return false; + if (stale) rebuild(); + const auto end = bytes > std::numeric_limits::max() - address ? std::numeric_limits::max() : address + bytes; + return walk(0, entries.size(), address, end, visit); + } + + template + bool StartingAt(std::uint64_t begin, Visit&& visit) const { + auto it = std::lower_bound(entries.begin(), entries.end(), begin, [](const Entry& entry, std::uint64_t value) { return entry.begin < value; }); + for (; it != entries.end() && it->begin == begin; ++it) { + if (visit(it->item)) return true; + } + return false; + } + + std::size_t Size() const { return entries.size(); } + +private: + struct Entry { + std::uint64_t begin; + std::uint64_t end; + Item item; + }; + + std::uint64_t build(std::size_t from, std::size_t to) { + if (from >= to) return 0; + const auto mid = from + (to - from) / 2; + const auto left = build(from, mid); + const auto right = build(mid + 1, to); + reach[mid] = std::max({entries[mid].end, left, right}); + return reach[mid]; + } + + void rebuild() { + reach.assign(entries.size(), 0); + build(0, entries.size()); + stale = false; + } + + template + bool walk(std::size_t from, std::size_t to, std::uint64_t address, std::uint64_t end, Visit& visit) const { + if (from >= to) return false; + const auto mid = from + (to - from) / 2; + if (reach[mid] <= address) return false; + if (walk(from, mid, address, end, visit)) return true; + if (entries[mid].begin >= end) return false; + if (address < entries[mid].end && visit(entries[mid].item)) return true; + return walk(mid + 1, to, address, end, visit); + } + + std::vector entries; + std::vector reach; + bool stale = false; +}; + +} diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp index c69c85b6629..dc6f65994a9 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp @@ -3,6 +3,7 @@ #include "prx/libSceAgcDriver/Graphics/include/Context.hpp" #include "prx/libSceAgcDriver/Graphics/include/DccMetadata.hpp" +#include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp" #include "prx/libSceAgcDriver/Graphics/include/GuestTextureResource.hpp" #include "prx/libSceAgcDriver/Graphics/include/TextureDetiler.hpp" #include "prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp" @@ -571,6 +572,7 @@ class StorageTexture : public std::enable_shared_from_this { std::weak_ptr borrowedFrom; std::uint64_t borrowedVersion = 0; std::vector borrowedUnits; + HostImportLookupCache importLookup; // Units of this image were borrowed since it was last written: the borrower's proofs // (the resource cache's serial memo) hold only while this image's version does, so the next // write here bumps the pending serial although the image was pending already. diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp index 6f6b03fb6e4..8d4aab31766 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp @@ -218,6 +218,7 @@ struct HostImports { // Bumped whenever an import is dropped, so HostImport pointers taken under the lock earlier can be // reused while it is unchanged. std::uint64_t epoch = 1; + std::uint64_t inserts = 0; VkDevice watchDevice = VK_NULL_HANDLE; bool unwatchImports = false; bool unwatchDmaBufImports = false; @@ -1616,6 +1617,7 @@ const HostImport* importRunAt(const Context& context, HostImports& state, Import } } #endif + ++state.inserts; const auto inserted = state.imports.emplace(base, entry); owner.armed = false; return &inserted.first->second; @@ -2999,14 +3001,7 @@ std::uint64_t HostImportsSettled() { return settledImports.load(std::memory_order_relaxed); } -const HostImport* HostImportFor(const Context& context, std::uint64_t address, std::size_t bytes) { - if (context.hostImportAlignment == 0 || bytes == 0 || bytes > std::numeric_limits::max() - address) return nullptr; - ensurePinWaiter(); - auto& state = Imports(); - std::lock_guard lock(state.mutex); -#ifdef _WIN32 - settleUnwatchedImports(context, state); -#endif +static const HostImport* hostImportSettled(const Context& context, HostImports& state, std::uint64_t address, std::size_t bytes) { // A hit is only valid while the registry has not changed since the imports were reconciled. if (!importsStale(context, state)) { if (const auto* entry = findImport(state, address, address + bytes)) { @@ -3019,6 +3014,36 @@ const HostImport* HostImportFor(const Context& context, std::uint64_t address, s return importAccess(context, state, address, address + bytes, lease); } +const HostImport* HostImportFor(const Context& context, std::uint64_t address, std::size_t bytes) { + if (context.hostImportAlignment == 0 || bytes == 0 || bytes > std::numeric_limits::max() - address) return nullptr; + ensurePinWaiter(); + auto& state = Imports(); + std::lock_guard lock(state.mutex); +#ifdef _WIN32 + settleUnwatchedImports(context, state); +#endif + return hostImportSettled(context, state, address, bytes); +} + +const HostImport* HostImportForCached(const Context& context, std::uint64_t address, std::size_t bytes, HostImportLookupCache& cache) { + if (context.hostImportAlignment == 0 || bytes == 0 || bytes > std::numeric_limits::max() - address) return nullptr; + ensurePinWaiter(); + auto& state = Imports(); + std::lock_guard lock(state.mutex); +#ifdef _WIN32 + settleUnwatchedImports(context, state); +#endif + if (cache.import != nullptr && cache.address == address && cache.bytes == bytes && cache.epoch == state.epoch && cache.inserts == state.inserts && !importsStale(context, state)) { + cache.import->lastUse = ++state.useCounter; + cache.import->lastUse = ++state.useCounter; + return cache.import; + } + const auto* entry = hostImportSettled(context, state, address, bytes); + if (entry != nullptr && !importsStale(context, state)) cache = {entry, address, bytes, state.epoch, state.inserts}; + else cache = {}; + return entry; +} + const HostImport* ExistingHostImportFor(const Context& context, std::uint64_t address, std::size_t bytes) { if (context.hostImportAlignment == 0 || bytes == 0 || bytes > std::numeric_limits::max() - address) return nullptr; auto& state = Imports(); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp index fdd30339371..cbbf3ea947a 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp @@ -7,6 +7,7 @@ #include "prx/libSceAgcDriver/Graphics/include/BufferPool.hpp" #include "prx/libSceAgcDriver/Graphics/include/Resources.hpp" #include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" +#include "prx/libSceAgcDriver/Graphics/include/RangeIndex.hpp" #include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp" // For the declaration of PendingStorageOverlaps, defined below. #include "prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp" @@ -1249,6 +1250,11 @@ StorageTexture::StorageTexture(const Context& context, TextureDetiler& detiler, namespace { +bool RefreshLookupsEnabled() { + static const bool enabled = std::getenv("APS5_NO_REFRESH_LOOKUP_CACHE") == nullptr; + return enabled; +} + class PendingList { public: using iterator = std::vector::iterator; @@ -1257,14 +1263,23 @@ class PendingList { void push_back(StorageTexture* texture) { textures.push_back(texture); ++version; + if (RefreshLookupsEnabled()) index.Insert(texture, texture->Descriptor().baseAddress, texture->Descriptor().baseAddress + texture->GuestBytes()); } iterator erase(iterator it) { ++version; + if (RefreshLookupsEnabled()) index.Remove(*it, (*it)->Descriptor().baseAddress); return textures.erase(it); } void remove(const StorageTexture* texture) { - if (std::erase(textures, texture) != 0) ++version; + if (std::erase(textures, texture) != 0) { + ++version; + if (RefreshLookupsEnabled()) index.Remove(const_cast(texture), texture->Descriptor().baseAddress, true); + } } + template + bool Overlapping(std::uint64_t address, std::uint64_t bytes, Visit&& visit) { return index.Overlapping(address, bytes, visit); } + template + bool StartingAt(std::uint64_t address, Visit&& visit) const { return index.StartingAt(address, visit); } bool MayOverlap(std::uint64_t address, std::size_t bytes) { if (textures.empty() || bytes == 0) return false; if (indexed != version) rebuild(); @@ -1287,6 +1302,7 @@ class PendingList { indexed = version; } std::vector textures; + RangeIndex index; std::vector> ranges; std::uint64_t version = 0; std::uint64_t indexed = ~std::uint64_t{0}; @@ -1866,7 +1882,7 @@ bool StorageTexture::Refresh() { // since `generation` need no comparison. const auto current = GuestMemory::CollectWrites(descriptor.baseAddress, static_cast(guestBytes)); // Per tracked layer: changed since its generation (a keys change makes every layer stale). - std::vector changed(trackedLayers, false); + std::vector changed; // Per 64 KiB block of the surface (blockGenerations): stamped since its layer's generation, and // by a CPU store; scanned once when anything changed. std::vector generations; @@ -1906,6 +1922,9 @@ bool StorageTexture::Refresh() { const auto keysStart = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; keys = ProvedKeys(); if (profile && descriptor.dccAddress != 0) LookupOutcomes::Add(LookupOutcomes::DccScan, keysStart); + const bool quickCandidate = RefreshLookupsEnabled() && alias == nullptr && keys == uploadedKeys && blockUnits; + const bool quick = quickCandidate && GuestMemory::UnchangedSince(descriptor.baseAddress, static_cast(guestBytes), generation); + if (!quick) changed.assign(trackedLayers, false); if (keys != uploadedKeys) { changed.assign(trackedLayers, true); unchanged = false; @@ -1913,7 +1932,7 @@ bool StorageTexture::Refresh() { // Which units the CPU stamped still decides what the drop rule below may drop. if (blockUnits) scanBlocks(); } else if (blockUnits) { - if (!GuestMemory::UnchangedSince(descriptor.baseAddress, static_cast(guestBytes), generation)) { + if (quickCandidate ? !quick : !GuestMemory::UnchangedSince(descriptor.baseAddress, static_cast(guestBytes), generation)) { // One stamp scan for every unit (each is one stamp block); the unchanged units move // to the current generation, so the whole-surface check answers next time. scanBlocks(); @@ -1952,7 +1971,7 @@ bool StorageTexture::Refresh() { // the same bytes read differently, and a changed unit holding pending results must take // the drop rule below (the image holds the results, `original` what the CPU wrote back). bool pendingChanged = false; - for (std::uint32_t layer = 0; layer < trackedLayers; ++layer) { + for (std::uint32_t layer = 0; !quick && layer < trackedLayers; ++layer) { if (changed[layer] && layerPending[layer]) pendingChanged = true; } if (!unchanged && !keysChanged && !pendingChanged && originalHolds(descriptor.baseAddress, static_cast(guestBytes)) && equalsOriginal()) { @@ -1961,7 +1980,7 @@ bool StorageTexture::Refresh() { } // Only the direct path uploads the selected layers alone (see upload); the others replace // the whole image, so every pending layer's results are stored first. - direct = keys == DccKeys::Uncompressed && HostImportFor(context, descriptor.baseAddress, static_cast(guestBytes)) != nullptr; + direct = keys == DccKeys::Uncompressed && (RefreshLookupsEnabled() ? HostImportForCached(context, descriptor.baseAddress, static_cast(guestBytes), importLookup) : HostImportFor(context, descriptor.baseAddress, static_cast(guestBytes))) != nullptr; if (alias != nullptr && direct) { // The alias's pending units that nothing stamped since its generation (a CPU store or a // driver store there wins over its results, as at its own write-back) are taken from @@ -2109,7 +2128,8 @@ bool StorageTexture::otherPendingOverlaps() const { auto& pending = Pending(); std::lock_guard lock(pending.mutex); const auto other = [&](const StorageTexture* texture) { return texture != this && texture->overlaps(descriptor.baseAddress, static_cast(guestBytes)); }; - return std::any_of(pending.textures.begin(), pending.textures.end(), other) || std::any_of(pending.flushing.begin(), pending.flushing.end(), other); + const bool pendingOverlap = RefreshLookupsEnabled() ? pending.textures.Overlapping(descriptor.baseAddress, guestBytes, [&](StorageTexture* texture) { return texture != this; }) : std::any_of(pending.textures.begin(), pending.textures.end(), other); + return pendingOverlap || std::any_of(pending.flushing.begin(), pending.flushing.end(), other); } bool StorageTexture::refreshProved() { @@ -4486,9 +4506,17 @@ std::shared_ptr StorageTexture::pendingAlias() const { { auto& pending = Pending(); std::lock_guard lock(pending.mutex); - for (auto* texture : pending.textures) { - if (texture == this || texture->descriptor.baseAddress != descriptor.baseAddress || texture->guestBytes != guestBytes || !texture->blockUnits) continue; + const auto consider = [&](StorageTexture* texture) { + if (texture == this || texture->descriptor.baseAddress != descriptor.baseAddress || texture->guestBytes != guestBytes || !texture->blockUnits) return; if (auto alive = texture->weak_from_this().lock()) candidates.push_back(std::move(alive)); + }; + if (RefreshLookupsEnabled()) { + pending.textures.StartingAt(descriptor.baseAddress, [&](StorageTexture* texture) { + consider(texture); + return false; + }); + } else { + for (auto* texture : pending.textures) consider(texture); } } for (const auto& candidate : candidates) { diff --git a/core/libs/prx/libSceAgcDriver/tests/HostImportRetireHold.cpp b/core/libs/prx/libSceAgcDriver/tests/HostImportRetireHold.cpp index 92c6fe639a0..37d34b683c8 100644 --- a/core/libs/prx/libSceAgcDriver/tests/HostImportRetireHold.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/HostImportRetireHold.cpp @@ -359,6 +359,54 @@ void BusyRetireOutlivesALaterBatch() { Expect(!Live(retired), "the dropped import outlived every batch of its hold"); } +void CachedLookupMatchesTheFullLookup() { + Heap heap; + ActiveRecorder active; + const auto context = mockContext(); + std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex()); + HostImportLookupCache cache; + const auto* first = HostImportForCached(context, heap.Address(), 4096, cache); + Expect(first != nullptr && cache.import == first, "the first cached lookup did not import and remember the range"); + if (first == nullptr) return; + Expect(HostImportFor(context, heap.Address(), 4096) == first, "the full lookup answered another import than the cached one"); + const auto before = first->lastUse; + Expect(HostImportForCached(context, heap.Address(), 4096, cache) == first, "a repeat lookup did not answer from the cache"); + const auto cachedUse = first->lastUse; + static_cast(HostImportFor(context, heap.Address(), 4096)); + Expect(cachedUse - before == first->lastUse - cachedUse, "a cached lookup and a full lookup stamp the last use differently"); + Expect(HostImportForCached(context, heap.Address(), 8192, cache) == HostImportFor(context, heap.Address(), 8192) && cache.bytes == 8192, "another range was answered from the cache"); + Expect(HostImportForCached(context, 0, 0, cache) == nullptr && HostImportForCached(context, heap.Address(), 0, cache) == nullptr, "an empty range was answered"); +} + +void CachedLookupFollowsTheImportTable() { + Heap heap; + ActiveRecorder active; + const auto context = mockContext(); + std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex()); + HostImportLookupCache cache; + const auto* first = HostImportForCached(context, heap.Address(), 4096, cache); + Expect(first != nullptr, "the first range was not imported"); + if (first == nullptr) return; + const auto firstBuffer = first->buffer; + { + Heap other; + const auto* added = HostImportFor(context, other.Address(), 4096); + Expect(added != nullptr && added != first, "an import of another range was not made"); + Expect(HostImportForCached(context, heap.Address(), 4096, cache) == HostImportFor(context, heap.Address(), 4096), "an import added between two lookups changed the cached answer"); + Expect(HostImportForCached(context, other.Address(), 4096, cache) == added, "the cache answered the range of another import"); + Expect(HostImportForCached(context, heap.Address(), 4096, cache) == first, "the cache lost the import that stayed"); + } + HostImportLookupCache removed; + static_cast(HostImportForCached(context, heap.Address(), 4096, removed)); + heap.Grow(); + const auto* grown = HostImportForCached(context, heap.Address(), 4096, removed); + Expect(grown != nullptr && grown == HostImportFor(context, heap.Address(), 4096) && grown->buffer != firstBuffer && grown->bytes == 2 * RangeBytes, "a dropped import was still answered from the cache"); + Expect(removed.import == grown && HostImportForCached(context, heap.Address(), 4096, removed) == grown, "the cache did not refill after the import was dropped"); + HostImportLookupCache gone; + Expect(HostImportForCached(context, heap.Address(), 4096, gone) == grown, "a cache filled after the drop answered another import"); + Expect(HostImportForCached(context, heap.Address(2 * RangeBytes), 4096, gone) == nullptr && gone.import == nullptr, "a range outside every registered range was answered or remembered"); +} + } int main() { @@ -372,6 +420,8 @@ int main() { WindowPromotionOutlivesTheHold(); IdleRetireOutlivesTheHold(); BusyRetireOutlivesALaterBatch(); + CachedLookupMatchesTheFullLookup(); + CachedLookupFollowsTheImportTable(); } catch (const std::exception& error) { std::fprintf(stderr, "%s\n", error.what()); return 1; diff --git a/core/libs/prx/libSceAgcDriver/tests/RangeIndex.cpp b/core/libs/prx/libSceAgcDriver/tests/RangeIndex.cpp new file mode 100644 index 00000000000..15700e64b22 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/RangeIndex.cpp @@ -0,0 +1,182 @@ +#include "prx/libSceAgcDriver/Graphics/include/RangeIndex.hpp" +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::RangeIndex; + +void Require(bool condition, const std::string& message) { + if (!condition) throw std::runtime_error(message); +} + +struct Entry { + std::uint64_t begin; + std::uint64_t end; + int id; + std::uint64_t sequence; +}; + +class BruteForce { +public: + void Insert(int id, std::uint64_t begin, std::uint64_t end) { entries.push_back({begin, end, id, ++sequence}); } + std::size_t Remove(int id, bool all) { + std::size_t removed = 0; + for (auto it = entries.begin(); it != entries.end();) { + if (it->id == id && (all || removed == 0)) { + it = entries.erase(it); + ++removed; + } else { + ++it; + } + } + return removed; + } + std::vector Overlapping(std::uint64_t address, std::uint64_t bytes) const { + std::vector hits; + for (const auto& entry : entries) { + if (bytes != 0 && address < entry.end && entry.begin < address + bytes) hits.push_back(entry); + } + return order(std::move(hits)); + } + std::vector StartingAt(std::uint64_t begin) const { + std::vector hits; + for (const auto& entry : entries) { + if (entry.begin == begin) hits.push_back(entry); + } + return order(std::move(hits)); + } + std::size_t Size() const { return entries.size(); } + const std::vector& All() const { return entries; } + +private: + static std::vector order(std::vector hits) { + std::sort(hits.begin(), hits.end(), [](const Entry& a, const Entry& b) { return a.begin != b.begin ? a.begin < b.begin : a.sequence < b.sequence; }); + std::vector ids; + for (const auto& hit : hits) ids.push_back(hit.id); + return ids; + } + std::vector entries; + std::uint64_t sequence = 0; +}; + +std::vector Collect(RangeIndex& index, std::uint64_t address, std::uint64_t bytes) { + std::vector ids; + index.Overlapping(address, bytes, [&](int id) { + ids.push_back(id); + return false; + }); + return ids; +} + +std::vector CollectStarting(const RangeIndex& index, std::uint64_t begin) { + std::vector ids; + index.StartingAt(begin, [&](int id) { + ids.push_back(id); + return false; + }); + return ids; +} + +void CheckSmall() { + RangeIndex index; + Require(Collect(index, 0, 100).empty(), "an empty index returned an entry"); + index.Insert(1, 100, 200); + index.Insert(2, 150, 160); + index.Insert(3, 100, 120); + index.Insert(4, 300, 300); + Require((Collect(index, 0, 100) == std::vector{}), "a range ending where the first entry starts overlapped it"); + Require((Collect(index, 199, 1) == std::vector{1}), "the last byte of an entry was missed"); + Require((Collect(index, 200, 1) == std::vector{}), "the byte after an entry overlapped it"); + Require((Collect(index, 110, 100) == std::vector{1, 3, 2}), "overlap order is not by start, then registration"); + Require((Collect(index, 100, 0).empty()), "an empty query range returned entries"); + Require((Collect(index, 250, 100) == std::vector{4}) && Collect(index, 300, 10).empty(), "a zero-length entry does not follow the strict overlap rule of the scan"); + Require((CollectStarting(index, 100) == std::vector{1, 3}), "entries at one start are not in registration order"); + int seen = 0; + const bool stopped = index.Overlapping(0, 1000, [&](int) { return ++seen == 2; }); + Require(stopped && seen == 2, "the walk did not stop when asked"); + Require(index.Remove(1, 100) == 1, "a present entry was not removed"); + Require(index.Remove(1, 100) == 0, "an absent entry was removed"); + Require((CollectStarting(index, 100) == std::vector{3}), "a removal changed the order of its neighbours"); + index.Insert(1, 100, 200); + Require((CollectStarting(index, 100) == std::vector{3, 1}), "a re-registered entry did not go behind the others"); + const std::uint64_t top = ~std::uint64_t{0}; + index.Insert(5, top - 10, top); + Require((Collect(index, top - 5, 100) == std::vector{5}), "a query running past the address space missed an entry"); +} + +void CheckRandom(std::uint32_t seed, std::uint64_t space, std::uint64_t maxLength, int rounds) { + std::mt19937_64 random(seed); + RangeIndex index; + BruteForce expected; + int nextId = 0; + std::vector live; + for (int round = 0; round < rounds; ++round) { + const auto action = random() % 10; + if (action < 5 || live.empty()) { + const auto begin = random() % space; + const auto end = begin + random() % (maxLength + 1); + const int id = nextId++; + index.Insert(id, begin, end); + expected.Insert(id, begin, end); + live.push_back(id); + } else if (action < 8) { + const auto at = random() % live.size(); + const int id = live[at]; + std::uint64_t begin = 0; + for (const auto& entry : expected.All()) { + if (entry.id == id) begin = entry.begin; + } + Require(index.Remove(id, begin) == expected.Remove(id, false), "a removal went differently from the list"); + live.erase(live.begin() + static_cast(at)); + } else { + index.Insert(nextId, 7, 7 + maxLength); + expected.Insert(nextId, 7, 7 + maxLength); + live.push_back(nextId++); + } + Require(index.Size() == expected.Size(), "the index and the list differ in size"); + for (int query = 0; query < 4; ++query) { + const auto address = random() % (space + maxLength); + const auto bytes = random() % (maxLength + 2); + Require(Collect(index, address, bytes) == expected.Overlapping(address, bytes), "overlap query " + std::to_string(address) + "+" + std::to_string(bytes) + " differs from the scan at round " + std::to_string(round)); + } + const auto start = random() % space; + Require(CollectStarting(index, start) == expected.StartingAt(start), "the entries at one start differ from the scan"); + if (!expected.All().empty()) { + const auto& pick = expected.All()[random() % expected.All().size()]; + Require(CollectStarting(index, pick.begin) == expected.StartingAt(pick.begin), "the entries at an existing start differ from the scan"); + } + } + while (!live.empty()) { + const int id = live.back(); + live.pop_back(); + std::uint64_t begin = 0; + for (const auto& entry : expected.All()) { + if (entry.id == id) begin = entry.begin; + } + Require(index.Remove(id, begin, true) == expected.Remove(id, true), "the final removal differs from the list"); + } + Require(index.Size() == 0 && Collect(index, 0, space + maxLength).empty(), "entries are left after removing every one"); +} + +} + +int main() { + try { + CheckSmall(); + CheckRandom(1, 64, 8, 4000); + CheckRandom(2, 4096, 512, 4000); + CheckRandom(3, 1ull << 40, 1ull << 30, 3000); + CheckRandom(4, 16, 64, 3000); + std::cout << "range index tests passed\n"; + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +}