From 18aaa32533220a909f2fe28c8937535da750e920 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bart=C5=82omiej=20Dach?= Date: Tue, 28 Jul 2026 11:19:16 +0200 Subject: [PATCH 1/7] Add online caching asset store --- osu.Game/Database/RealmAccess.cs | 9 +- osu.Game/Database/RealmObjectExtensions.cs | 1 + osu.Game/Graphics/OnlineAssetCachingStore.cs | 87 ++++++++++++++++++++ osu.Game/Models/RealmOnlineAsset.cs | 41 +++++++++ osu.Game/OsuGameBase.cs | 2 + 5 files changed, 139 insertions(+), 1 deletion(-) create mode 100644 osu.Game/Graphics/OnlineAssetCachingStore.cs create mode 100644 osu.Game/Models/RealmOnlineAsset.cs diff --git a/osu.Game/Database/RealmAccess.cs b/osu.Game/Database/RealmAccess.cs index 1eca90826a1b..d449cb217f01 100644 --- a/osu.Game/Database/RealmAccess.cs +++ b/osu.Game/Database/RealmAccess.cs @@ -101,8 +101,9 @@ public class RealmAccess : IDisposable /// 49 2025-06-10 Reset the LegacyOnlineID to -1 for all scores that have it set to 0 (which is semantically the same) for consistency of handling with OnlineID. /// 50 2025-07-11 Add UserTags to BeatmapMetadata. /// 51 2025-07-22 Add ScoreInfo.Pauses. + /// 52 2026-07-28 Add RealmOnlineAsset. /// - private const int schema_version = 51; + private const int schema_version = 52; /// /// Lock object which is held during sections, blocking realm retrieval during blocking periods. @@ -413,6 +414,12 @@ private void cleanupPendingDeletions(Realm realm) foreach (var s in pendingDeletePresets) realm.Remove(s); + var onlineAssetAccessCutoff = DateTimeOffset.Now.AddMonths(-1); + var pendingDeleteOnlineAssets = realm.All().Where(a => a.LastAccessed < onlineAssetAccessCutoff); + + foreach (var a in pendingDeleteOnlineAssets) + realm.Remove(a); + transaction.Commit(); } diff --git a/osu.Game/Database/RealmObjectExtensions.cs b/osu.Game/Database/RealmObjectExtensions.cs index c334f1152dfc..5d4c1ff37805 100644 --- a/osu.Game/Database/RealmObjectExtensions.cs +++ b/osu.Game/Database/RealmObjectExtensions.cs @@ -180,6 +180,7 @@ private static void applyCommonConfiguration(IMapperConfigurationExpression c) c.CreateMap(); c.CreateMap(); c.CreateMap(); + c.CreateMap(); c.CreateMap(); } diff --git a/osu.Game/Graphics/OnlineAssetCachingStore.cs b/osu.Game/Graphics/OnlineAssetCachingStore.cs new file mode 100644 index 000000000000..5b1a9df532e7 --- /dev/null +++ b/osu.Game/Graphics/OnlineAssetCachingStore.cs @@ -0,0 +1,87 @@ +// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. +// See the LICENCE file in the repository root for full licence text. + +using System; +using System.Linq; +using osu.Framework.Graphics.Textures; +using osu.Framework.IO.Stores; +using osu.Framework.Logging; +using osu.Framework.Platform; +using osu.Game.Database; +using osu.Game.Extensions; +using osu.Game.Models; +using osu.Game.Online; +using Realms; + +namespace osu.Game.Graphics +{ + /// + /// + /// Store for retrieval and caching of assets (background, avatars, covers) retrieved from the web to disk. + /// + /// + /// This store assumes relies on the uniqueness of the URL of retrieved assets to determine identity. + /// Therefore, this store MUST only be used with URLs that are content-addressed in some way + /// (by containing a content-based hash in the filename, or a cache-busting query string based on time of last update). + /// + /// + /// This store MUST NOT be used with URLs containing naive cache-busting strings (e.g. test.jpg?TIMESTAMP) + /// as it both makes the caching ineffective AND trashes the cache with entries that will never be used again. + /// + /// + public class OnlineAssetCachingStore + { + private readonly RealmAccess realmAccess; + private readonly OnlineStore onlineStore; + private readonly RealmFileStore fileStore; + private readonly LargeTextureStore largeTextureStore; + + public OnlineAssetCachingStore(GameHost host, RealmAccess realmAccess) + { + this.realmAccess = realmAccess; + onlineStore = new TrustedDomainOnlineStore(); + fileStore = new RealmFileStore(realmAccess, host.Storage); + largeTextureStore = new LargeTextureStore(host.Renderer, host.CreateTextureLoaderStore(new StorageBackedResourceStore(fileStore.Storage))); + } + + public Texture? Get(string url) + { + var existingAsset = realmAccess.Write(r => + { + var a = r.All().Filter($@"{nameof(RealmOnlineAsset.File)}.{nameof(RealmNamedFileUsage.Filename)} == $0", url).FirstOrDefault(); + if (a != null) + a.LastAccessed = DateTimeOffset.Now; + return a?.Detach(); + }); + + if (existingAsset == null) + { + var onlineStream = onlineStore.GetStream(url); + + if (onlineStream == null) + return null; + + existingAsset = realmAccess.Write(r => + { + var file = fileStore.Add(onlineStream, r); + var newAsset = new RealmOnlineAsset(file, url); + r.Add(newAsset); + return newAsset.Detach(); + }); + } + else + { + Logger.Log($"Online asset {url} retrieved from {nameof(OnlineAssetCachingStore)}.", LoggingTarget.Network); + } + + string path = existingAsset.File.File.GetStoragePath(); + return largeTextureStore.Get(path); + } + + public void Dispose() + { + onlineStore.Dispose(); + largeTextureStore.Dispose(); + } + } +} diff --git a/osu.Game/Models/RealmOnlineAsset.cs b/osu.Game/Models/RealmOnlineAsset.cs new file mode 100644 index 000000000000..e8a5d1a6eaa5 --- /dev/null +++ b/osu.Game/Models/RealmOnlineAsset.cs @@ -0,0 +1,41 @@ +// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. +// See the LICENCE file in the repository root for full licence text. + +using System; +using JetBrains.Annotations; +using osu.Game.Database; +using Realms; + +namespace osu.Game.Models +{ + /// + /// Describes an online asset (background image, user avatar, cover...) which is persisted to disk + /// to reduce the number of online requests and shorten retrieval time. + /// + public class RealmOnlineAsset : RealmObject + { + /// + /// Contains information about the original URL of the file and its location on disk. + /// + public RealmNamedFileUsage File { get; set; } = null!; + + /// + /// Contains the last time of access of this asset. + /// + /// + /// Assets that have not been accessed for over a month are purged + /// (). + /// + public DateTimeOffset LastAccessed { get; set; } = DateTimeOffset.Now; + + [UsedImplicitly] + private RealmOnlineAsset() + { + } + + public RealmOnlineAsset(RealmFile localFile, string remoteUrl) + { + File = new RealmNamedFileUsage(localFile, remoteUrl); + } + } +} diff --git a/osu.Game/OsuGameBase.cs b/osu.Game/OsuGameBase.cs index cc4fa5b618ca..0228af495465 100644 --- a/osu.Game/OsuGameBase.cs +++ b/osu.Game/OsuGameBase.cs @@ -291,6 +291,8 @@ private void load(ReadableKeyCombinationProvider keyCombinationProvider, Framewo largeStore.AddTextureSource(Host.CreateTextureLoaderStore(CreateOnlineStore())); dependencies.Cache(largeStore); + dependencies.Cache(new OnlineAssetCachingStore(Host, realm)); + dependencies.CacheAs(LocalConfig); dependencies.CacheAs(LocalConfig); From c44d33d3e3e0ca0b8ca4b3aefc766ea294f4b8a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bart=C5=82omiej=20Dach?= Date: Tue, 28 Jul 2026 11:23:37 +0200 Subject: [PATCH 2/7] Use online asset caching store in a few places which can benefit from it --- osu.Game/Users/Drawables/DrawableAvatar.cs | 5 +++-- osu.Game/Users/Drawables/DrawableTeamFlag.cs | 6 +++--- osu.Game/Users/UserCoverBackground.cs | 4 ++-- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/osu.Game/Users/Drawables/DrawableAvatar.cs b/osu.Game/Users/Drawables/DrawableAvatar.cs index bd09b9516482..137bff6af242 100644 --- a/osu.Game/Users/Drawables/DrawableAvatar.cs +++ b/osu.Game/Users/Drawables/DrawableAvatar.cs @@ -7,6 +7,7 @@ using osu.Framework.Graphics; using osu.Framework.Graphics.Sprites; using osu.Framework.Graphics.Textures; +using osu.Game.Graphics; using osu.Game.Online.API.Requests.Responses; namespace osu.Game.Users.Drawables @@ -31,12 +32,12 @@ public DrawableAvatar(IUser user = null) } [BackgroundDependencyLoader] - private void load(LargeTextureStore textures) + private void load(LargeTextureStore textures, OnlineAssetCachingStore onlineTextures) { if (user != null && user.OnlineID > 1) // TODO: The fallback here should not need to exist. Users should be looked up and populated via UserLookupCache or otherwise // in remaining cases where this is required (chat tabs, local leaderboard), at which point this should be removed. - Texture = textures.Get((user as APIUser)?.AvatarUrl ?? $@"https://a.ppy.sh/{user.OnlineID}"); + Texture = onlineTextures.Get((user as APIUser)?.AvatarUrl ?? $@"https://a.ppy.sh/{user.OnlineID}"); Texture ??= textures.Get(@"Online/avatar-guest"); } diff --git a/osu.Game/Users/Drawables/DrawableTeamFlag.cs b/osu.Game/Users/Drawables/DrawableTeamFlag.cs index 27b5f447a5ac..5db46da7bc67 100644 --- a/osu.Game/Users/Drawables/DrawableTeamFlag.cs +++ b/osu.Game/Users/Drawables/DrawableTeamFlag.cs @@ -6,7 +6,7 @@ using osu.Framework.Graphics.Containers; using osu.Framework.Graphics.Shapes; using osu.Framework.Graphics.Sprites; -using osu.Framework.Graphics.Textures; +using osu.Game.Graphics; using osu.Game.Online.API.Requests.Responses; namespace osu.Game.Users.Drawables @@ -44,9 +44,9 @@ public DrawableTeamFlag(APITeam? team) } [BackgroundDependencyLoader] - private void load(LargeTextureStore textures) + private void load(OnlineAssetCachingStore textures) { - if (team != null) + if (team?.FlagUrl != null) sprite.Texture = textures.Get(team.FlagUrl); } } diff --git a/osu.Game/Users/UserCoverBackground.cs b/osu.Game/Users/UserCoverBackground.cs index 4d248d450b81..7b7dc9306ad3 100644 --- a/osu.Game/Users/UserCoverBackground.cs +++ b/osu.Game/Users/UserCoverBackground.cs @@ -9,7 +9,7 @@ using osu.Framework.Graphics.Containers; using osu.Framework.Graphics.Shapes; using osu.Framework.Graphics.Sprites; -using osu.Framework.Graphics.Textures; +using osu.Game.Graphics; using osu.Game.Online.API.Requests.Responses; using osuTK.Graphics; @@ -51,7 +51,7 @@ public Cover(APIUser? user) } [BackgroundDependencyLoader] - private void load(LargeTextureStore textures) + private void load(OnlineAssetCachingStore textures) { if (user == null) { From ad4e6eda4adb081cbfe66cdb749d1130be125100 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bart=C5=82omiej=20Dach?= Date: Wed, 29 Jul 2026 12:39:19 +0200 Subject: [PATCH 3/7] Possibly fix test failures This fixes one deadlock that I can observe locally. The cause for said deadlock was `RealmAccess.getRealmInstance()` spinning forever on a disposed semaphore. Remains to be seen whether this is all there is to the failures. --- osu.Game/Graphics/OnlineAssetCachingStore.cs | 7 ++++++- osu.Game/OsuGameBase.cs | 5 ++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/osu.Game/Graphics/OnlineAssetCachingStore.cs b/osu.Game/Graphics/OnlineAssetCachingStore.cs index 5b1a9df532e7..5d7bc5dae5d4 100644 --- a/osu.Game/Graphics/OnlineAssetCachingStore.cs +++ b/osu.Game/Graphics/OnlineAssetCachingStore.cs @@ -29,13 +29,15 @@ namespace osu.Game.Graphics /// as it both makes the caching ineffective AND trashes the cache with entries that will never be used again. /// /// - public class OnlineAssetCachingStore + public sealed class OnlineAssetCachingStore { private readonly RealmAccess realmAccess; private readonly OnlineStore onlineStore; private readonly RealmFileStore fileStore; private readonly LargeTextureStore largeTextureStore; + private bool disposed; + public OnlineAssetCachingStore(GameHost host, RealmAccess realmAccess) { this.realmAccess = realmAccess; @@ -46,6 +48,8 @@ public OnlineAssetCachingStore(GameHost host, RealmAccess realmAccess) public Texture? Get(string url) { + ObjectDisposedException.ThrowIf(disposed, this); + var existingAsset = realmAccess.Write(r => { var a = r.All().Filter($@"{nameof(RealmOnlineAsset.File)}.{nameof(RealmNamedFileUsage.Filename)} == $0", url).FirstOrDefault(); @@ -82,6 +86,7 @@ public void Dispose() { onlineStore.Dispose(); largeTextureStore.Dispose(); + disposed = true; } } } diff --git a/osu.Game/OsuGameBase.cs b/osu.Game/OsuGameBase.cs index 0228af495465..48e2eede0a17 100644 --- a/osu.Game/OsuGameBase.cs +++ b/osu.Game/OsuGameBase.cs @@ -208,6 +208,8 @@ public virtual string Version private BeatmapDifficultyCache difficultyCache; private IBeatmapUpdater beatmapUpdater; + private OnlineAssetCachingStore onlineAssetCache; + private UserLookupCache userCache; private BeatmapLookupCache beatmapCache; protected LeaderboardManager LeaderboardManager { get; private set; } @@ -291,7 +293,7 @@ private void load(ReadableKeyCombinationProvider keyCombinationProvider, Framewo largeStore.AddTextureSource(Host.CreateTextureLoaderStore(CreateOnlineStore())); dependencies.Cache(largeStore); - dependencies.Cache(new OnlineAssetCachingStore(Host, realm)); + dependencies.Cache(onlineAssetCache = new OnlineAssetCachingStore(Host, realm)); dependencies.CacheAs(LocalConfig); dependencies.CacheAs(LocalConfig); @@ -790,6 +792,7 @@ protected override void Dispose(bool isDisposing) LocalConfig?.Dispose(); beatmapUpdater?.Dispose(); + onlineAssetCache?.Dispose(); realm?.Dispose(); From 57d5ee8f9571fa4495e49d2f4e5a3198088eaec2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bart=C5=82omiej=20Dach?= Date: Thu, 30 Jul 2026 08:40:14 +0200 Subject: [PATCH 4/7] DEBUG: add timeout to realm retrieval This is a TEMPORARY measure to hopefully help diagnose deadlocks of `OnlineCachingAssetStore` on single-thread CI runs. I am unable to reproduce those deadlocks locally despite almost an hour of debug test runs in various configurations. I can reproduce *other* tests deadlocking, but on nothing seemingly related to this one new class. --- osu.Game/Database/RealmAccess.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osu.Game/Database/RealmAccess.cs b/osu.Game/Database/RealmAccess.cs index d449cb217f01..67fb5a7b2555 100644 --- a/osu.Game/Database/RealmAccess.cs +++ b/osu.Game/Database/RealmAccess.cs @@ -778,7 +778,7 @@ private Realm getRealmInstance() // Ensure that the thread that currently has the `realmRetrievalLock` can retrieve nested contexts and not deadlock on itself. if (!currentThreadHasRealmRetrievalLock.Value) { - realmRetrievalLock.Wait(); + realmRetrievalLock.Wait(10000); currentThreadHasRealmRetrievalLock.Value = true; tookSemaphoreLock = true; } From 8c1a46f6b37e23e5c53d0e07f605a5fc6b0feca1 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Fri, 31 Jul 2026 16:39:13 +0900 Subject: [PATCH 5/7] Revert "Possibly fix test failures" This reverts commit ad4e6eda4adb081cbfe66cdb749d1130be125100. --- osu.Game/Graphics/OnlineAssetCachingStore.cs | 7 +------ osu.Game/OsuGameBase.cs | 5 +---- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/osu.Game/Graphics/OnlineAssetCachingStore.cs b/osu.Game/Graphics/OnlineAssetCachingStore.cs index 5d7bc5dae5d4..5b1a9df532e7 100644 --- a/osu.Game/Graphics/OnlineAssetCachingStore.cs +++ b/osu.Game/Graphics/OnlineAssetCachingStore.cs @@ -29,15 +29,13 @@ namespace osu.Game.Graphics /// as it both makes the caching ineffective AND trashes the cache with entries that will never be used again. /// /// - public sealed class OnlineAssetCachingStore + public class OnlineAssetCachingStore { private readonly RealmAccess realmAccess; private readonly OnlineStore onlineStore; private readonly RealmFileStore fileStore; private readonly LargeTextureStore largeTextureStore; - private bool disposed; - public OnlineAssetCachingStore(GameHost host, RealmAccess realmAccess) { this.realmAccess = realmAccess; @@ -48,8 +46,6 @@ public OnlineAssetCachingStore(GameHost host, RealmAccess realmAccess) public Texture? Get(string url) { - ObjectDisposedException.ThrowIf(disposed, this); - var existingAsset = realmAccess.Write(r => { var a = r.All().Filter($@"{nameof(RealmOnlineAsset.File)}.{nameof(RealmNamedFileUsage.Filename)} == $0", url).FirstOrDefault(); @@ -86,7 +82,6 @@ public void Dispose() { onlineStore.Dispose(); largeTextureStore.Dispose(); - disposed = true; } } } diff --git a/osu.Game/OsuGameBase.cs b/osu.Game/OsuGameBase.cs index 48e2eede0a17..0228af495465 100644 --- a/osu.Game/OsuGameBase.cs +++ b/osu.Game/OsuGameBase.cs @@ -208,8 +208,6 @@ public virtual string Version private BeatmapDifficultyCache difficultyCache; private IBeatmapUpdater beatmapUpdater; - private OnlineAssetCachingStore onlineAssetCache; - private UserLookupCache userCache; private BeatmapLookupCache beatmapCache; protected LeaderboardManager LeaderboardManager { get; private set; } @@ -293,7 +291,7 @@ private void load(ReadableKeyCombinationProvider keyCombinationProvider, Framewo largeStore.AddTextureSource(Host.CreateTextureLoaderStore(CreateOnlineStore())); dependencies.Cache(largeStore); - dependencies.Cache(onlineAssetCache = new OnlineAssetCachingStore(Host, realm)); + dependencies.Cache(new OnlineAssetCachingStore(Host, realm)); dependencies.CacheAs(LocalConfig); dependencies.CacheAs(LocalConfig); @@ -792,7 +790,6 @@ protected override void Dispose(bool isDisposing) LocalConfig?.Dispose(); beatmapUpdater?.Dispose(); - onlineAssetCache?.Dispose(); realm?.Dispose(); From 2065cb813ed9b7d0726192d050804ca6f7954a5b Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Fri, 31 Jul 2026 16:39:14 +0900 Subject: [PATCH 6/7] Revert "DEBUG: add timeout to realm retrieval" This reverts commit 57d5ee8f9571fa4495e49d2f4e5a3198088eaec2. --- osu.Game/Database/RealmAccess.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osu.Game/Database/RealmAccess.cs b/osu.Game/Database/RealmAccess.cs index 67fb5a7b2555..d449cb217f01 100644 --- a/osu.Game/Database/RealmAccess.cs +++ b/osu.Game/Database/RealmAccess.cs @@ -778,7 +778,7 @@ private Realm getRealmInstance() // Ensure that the thread that currently has the `realmRetrievalLock` can retrieve nested contexts and not deadlock on itself. if (!currentThreadHasRealmRetrievalLock.Value) { - realmRetrievalLock.Wait(10000); + realmRetrievalLock.Wait(); currentThreadHasRealmRetrievalLock.Value = true; tookSemaphoreLock = true; } From 17a62807aef2f3fd45e5416d6f62ba1859609f5f Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Fri, 31 Jul 2026 16:39:37 +0900 Subject: [PATCH 7/7] Attempt different maybe realm fix --- osu.Game/Database/RealmAccess.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/osu.Game/Database/RealmAccess.cs b/osu.Game/Database/RealmAccess.cs index d449cb217f01..6bcf3b4c6653 100644 --- a/osu.Game/Database/RealmAccess.cs +++ b/osu.Game/Database/RealmAccess.cs @@ -781,6 +781,8 @@ private Realm getRealmInstance() realmRetrievalLock.Wait(); currentThreadHasRealmRetrievalLock.Value = true; tookSemaphoreLock = true; + + ObjectDisposedException.ThrowIf(isDisposed, this); } else {