From a6d43bd8b3b462a69a0fb4f03f3401e86e512819 Mon Sep 17 00:00:00 2001 From: Marino Faggiana Date: Sun, 23 Aug 2026 09:09:59 +0200 Subject: [PATCH 1/4] fix: limit media verification request size Signed-off-by: Marino Faggiana --- iOSClient/Media/NCMediaDataSource.swift | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/iOSClient/Media/NCMediaDataSource.swift b/iOSClient/Media/NCMediaDataSource.swift index 52e66f13b5..b804e67c59 100644 --- a/iOSClient/Media/NCMediaDataSource.swift +++ b/iOSClient/Media/NCMediaDataSource.swift @@ -231,8 +231,9 @@ extension NCMedia { var firstDateNew = Date.distantFuture var lastDateNew = Date.distantPast - var firstDate: Date? - var lastDate: Date? + var firstVisibleCellDate: Date? + var lastVisibleCellDate: Date? + var visibleCellCount = 0 var visibleCells: [NCMediaCell] = [] await MainActor.run { @@ -270,8 +271,9 @@ extension NCMedia { return date1 > date2 } - firstDate = visibleCells.first?.date - lastDate = visibleCells.last?.date + firstVisibleCellDate = visibleCells.first?.date + lastVisibleCellDate = visibleCells.last?.date + visibleCellCount = visibleCells.count if !visibleCells.isEmpty, !distant { let firstCellDate = visibleCells.first?.date @@ -324,17 +326,19 @@ extension NCMedia { guard !Task.isCancelled, self.isViewActived, self.session.account == account, - let firstDate, - let lastDate else { + let firstVisibleCellDate, + let lastVisibleCellDate else { return } // VERIFY MEDIA // - await self.verifyNetworkMedia(firstDate: firstDate, - lastDate: lastDate, + let verificationLimit = max(visibleCellCount * 3, 300) + await self.verifyNetworkMedia(firstDate: firstVisibleCellDate, + lastDate: lastVisibleCellDate, mediaPath: tblAccount.mediaPath, - account: account) { + account: account, + limit: verificationLimit) { Task { [weak self] in guard let self else { return @@ -409,6 +413,7 @@ extension NCMedia { lastDate: Date, mediaPath: String, account: String, + limit: Int, update: @escaping () -> Void, finish: @escaping () -> Void) async { await NCMediaNetwork().searchMediaPage( @@ -417,7 +422,7 @@ extension NCMedia { lastDate: lastDate, account: account, paginate: true, - limit: 1000000) { task in + limit: limit) { task in Task { let identifier = await NCNetworking.shared.networkingTasks.createIdentifier( account: account, From 945712a0867b6c81153bb7a23a57f06fe29d0169 Mon Sep 17 00:00:00 2001 From: Marino Faggiana Date: Sun, 23 Aug 2026 10:05:40 +0200 Subject: [PATCH 2/4] fix: resume media metadata backfill in bounded batches Signed-off-by: Marino Faggiana --- Brand/Database.swift | 2 +- ...ManageDatabase+MediaMetadataBackfill.swift | 8 +- .../NCMediaMetadataBackfillProcessor.swift | 172 +++++++++++++++--- 3 files changed, 156 insertions(+), 26 deletions(-) diff --git a/Brand/Database.swift b/Brand/Database.swift index 3e692dd6b7..6dc29a2020 100644 --- a/Brand/Database.swift +++ b/Brand/Database.swift @@ -8,4 +8,4 @@ import Foundation // let databaseName = "nextcloud.realm" let tableAccountBackup = "tableAccountBackup.json" -let databaseSchemaVersion: UInt64 = 413 +let databaseSchemaVersion: UInt64 = 414 diff --git a/iOSClient/Data/NCManageDatabase+MediaMetadataBackfill.swift b/iOSClient/Data/NCManageDatabase+MediaMetadataBackfill.swift index 5627471f9c..4bc7921a46 100644 --- a/iOSClient/Data/NCManageDatabase+MediaMetadataBackfill.swift +++ b/iOSClient/Data/NCManageDatabase+MediaMetadataBackfill.swift @@ -10,6 +10,7 @@ import NextcloudKit class tableMediaMetadataBackfill: Object { @Persisted(primaryKey: true) var account = "" @Persisted var offset = 0 + @Persisted var cursorDate: Date? @Persisted var lastRunDate: Date? @Persisted var lastCompletedCycleDate: Date? @@ -36,15 +37,18 @@ extension NCManageDatabase { // MARK: - Realm write func updateMediaMetadataBackfillAsync(account: String, - offset: Int) async { + offset: Int, + cursorDate: Date) async { await core.performRealmWriteAsync { realm in if let backfill = realm.object(ofType: tableMediaMetadataBackfill.self, forPrimaryKey: account) { backfill.offset = offset + backfill.cursorDate = cursorDate backfill.lastRunDate = Date() } else { let backfill = tableMediaMetadataBackfill(account: account) backfill.offset = offset + backfill.cursorDate = cursorDate backfill.lastRunDate = Date() realm.add(backfill) } @@ -55,11 +59,13 @@ extension NCManageDatabase { await core.performRealmWriteAsync { realm in if let backfill = realm.object(ofType: tableMediaMetadataBackfill.self, forPrimaryKey: account) { backfill.offset = 0 + backfill.cursorDate = nil backfill.lastRunDate = Date() backfill.lastCompletedCycleDate = Date() } else { let backfill = tableMediaMetadataBackfill(account: account) backfill.offset = 0 + backfill.cursorDate = nil backfill.lastRunDate = Date() backfill.lastCompletedCycleDate = Date() realm.add(backfill) diff --git a/iOSClient/Processor/NCMediaMetadataBackfillProcessor.swift b/iOSClient/Processor/NCMediaMetadataBackfillProcessor.swift index 0bfe833ad5..786ec3bbe4 100644 --- a/iOSClient/Processor/NCMediaMetadataBackfillProcessor.swift +++ b/iOSClient/Processor/NCMediaMetadataBackfillProcessor.swift @@ -4,15 +4,19 @@ import Foundation import NextcloudKit +import os /// Incrementally scans the remote media archive and creates missing local metadata placeholders. /// -/// The current offset is persisted so interrupted executions can resume later. +/// The oldest date processed in a bounded result batch is persisted so later executions can resume. /// Once the archive has been fully processed, subsequent executions are skipped. final class NCMediaMetadataBackfillProcessor { + private let pagesPerBatch = 4 + /// Represents the result of a media metadata backfill execution. enum BackfillStatus { case skippedAlreadyCompleted(account: String) + case batchCompleted(account: String, processed: Int, inserted: Int, updated: Int, cursorDate: Date) case completed(account: String, processed: Int, inserted: Int, updated: Int) case failed(account: String, processed: Int, inserted: Int, updated: Int, errorCode: Int, errorDescription: String) case cancelled(account: String, processed: Int, inserted: Int, updated: Int) @@ -20,7 +24,7 @@ final class NCMediaMetadataBackfillProcessor { /// Returns whether the backfill completed successfully or was already completed. var isSuccessful: Bool { switch self { - case .skippedAlreadyCompleted, .completed: + case .skippedAlreadyCompleted, .batchCompleted, .completed: return true case .failed, .cancelled: return false @@ -33,6 +37,9 @@ final class NCMediaMetadataBackfillProcessor { case .skippedAlreadyCompleted(let account): return "Media metadata backfill skipped for account \(account): cycle already completed" + case .batchCompleted(let account, let processed, let inserted, let updated, let cursorDate): + return "Media metadata backfill batch completed for account \(account): processed \(processed) - inserted \(inserted) - updated \(updated) - cursor date \(cursorDate)" + case .completed(let account, let processed, let inserted, let updated): return "Media metadata backfill completed for account \(account): processed \(processed) - inserted \(inserted) - updated \(updated)" @@ -45,9 +52,11 @@ final class NCMediaMetadataBackfillProcessor { } } - /// Processes the remote media archive page by page and creates missing metadata placeholders. + /// Processes one bounded batch of the remote media archive and creates missing metadata placeholders. /// - /// An interrupted cycle resumes immediately from the stored offset. + /// Each completed page checkpoints its oldest date after placeholder synchronization. + /// An interrupted page is safely retried because placeholder synchronization is idempotent. + /// A full batch persists its oldest date so the next execution can continue from that boundary. /// A completed cycle starts again after the configured interval. func runBackfill( account: tableAccount, @@ -57,25 +66,49 @@ final class NCMediaMetadataBackfillProcessor { let database = NCManageDatabase.shared let state = await database.getMediaMetadataBackfillAsync(account: account.account) let cycleInterval: TimeInterval = 7 * 24 * 60 * 60 // week - var offset = state?.offset ?? 0 + let previousCursorDate = state?.cursorDate + let firstDate = previousCursorDate ?? .distantFuture + let previouslyProcessed = previousCursorDate == nil ? 0 : state?.offset ?? 0 + var pageOffset = 0 var token: String? var processed = 0 var inserted = 0 var updated = 0 + var oldestProcessedDate: Date? + + guard limit > 0 else { + return .failed( + account: account.account, + processed: 0, + inserted: 0, + updated: 0, + errorCode: NCGlobal.shared.errorPreconditionFailed, + errorDescription: "Invalid media metadata backfill page size: \(limit)" + ) + } + + let searchResultLimit = limit * pagesPerBatch if state?.offset == 0, + state?.cursorDate == nil, let lastCompletedCycleDate = state?.lastCompletedCycleDate, Date().timeIntervalSince(lastCompletedCycleDate) < cycleInterval { return .skippedAlreadyCompleted(account: account.account) } - while !Task.isCancelled { + for _ in 0..