diff --git a/Brand/Database.swift b/Brand/Database.swift index 3e692dd6b7..6dc29a2020 100644 --- a/Brand/Database.swift +++ b/Brand/Database.swift @@ -8,4 +8,4 @@ import Foundation // let databaseName = "nextcloud.realm" let tableAccountBackup = "tableAccountBackup.json" -let databaseSchemaVersion: UInt64 = 413 +let databaseSchemaVersion: UInt64 = 414 diff --git a/Nextcloud.xcodeproj/project.pbxproj b/Nextcloud.xcodeproj/project.pbxproj index 91d1ff2fe4..fdaf4e233b 100644 --- a/Nextcloud.xcodeproj/project.pbxproj +++ b/Nextcloud.xcodeproj/project.pbxproj @@ -12,6 +12,7 @@ 2C33C48223E2C475005F963B /* NotificationService.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2C33C48123E2C475005F963B /* NotificationService.swift */; }; 2C33C48623E2C475005F963B /* Notification Service Extension.appex in Embed Foundation Extensions */ = {isa = PBXBuildFile; fileRef = 2C33C47F23E2C475005F963B /* Notification Service Extension.appex */; settings = {ATTRIBUTES = (RemoveHeadersOnCopy, ); }; }; 370D26AF248A3D7A00121797 /* NCCellMain.swift in Sources */ = {isa = PBXBuildFile; fileRef = 370D26AE248A3D7A00121797 /* NCCellMain.swift */; }; + A1B2C3D430B0000100ABCDEF /* NCMediaViewerLoadingPolicy.swift in Sources */ = {isa = PBXBuildFile; fileRef = A1B2C3D330B0000100ABCDEF /* NCMediaViewerLoadingPolicy.swift */; }; A5A87F9E4B0E4441A6A4BC20 /* NCContextMenuProfile.swift in Sources */ = {isa = PBXBuildFile; fileRef = BB7697C94BA14450A0867940 /* NCContextMenuProfile.swift */; }; AA3C85E82D36B08C00F74F12 /* UITestBackend.swift in Sources */ = {isa = PBXBuildFile; fileRef = AA3C85E72D36B08C00F74F12 /* UITestBackend.swift */; }; AA3C85EB2D36BBFB00F74F12 /* OCSResponse.swift in Sources */ = {isa = PBXBuildFile; fileRef = AA3C85EA2D36BBF400F74F12 /* OCSResponse.swift */; }; @@ -850,7 +851,6 @@ F7CDB5C32FA33CA300F72306 /* NCMediaViewerPageView.swift in Sources */ = {isa = PBXBuildFile; fileRef = F7CDB5B92FA33CA300F72306 /* NCMediaViewerPageView.swift */; }; F7CDB5C42FA33CA300F72306 /* NCImageViewerContentView.swift in Sources */ = {isa = PBXBuildFile; fileRef = F7CDB5B62FA33CA300F72306 /* NCImageViewerContentView.swift */; }; F7CDB5C52FA33CA300F72306 /* NCMediaViewerModel.swift in Sources */ = {isa = PBXBuildFile; fileRef = F7CDB5B82FA33CA300F72306 /* NCMediaViewerModel.swift */; }; - A1B2C3D430B0000100ABCDEF /* NCMediaViewerLoadingPolicy.swift in Sources */ = {isa = PBXBuildFile; fileRef = A1B2C3D330B0000100ABCDEF /* NCMediaViewerLoadingPolicy.swift */; }; F7CDB5C62FA33CA300F72306 /* NCMediaViewerView.swift in Sources */ = {isa = PBXBuildFile; fileRef = F7CDB5BB2FA33CA300F72306 /* NCMediaViewerView.swift */; }; F7CDB5CC2FA33CA300F72306 /* NCNextcloudMediaViewerLoader.swift in Sources */ = {isa = PBXBuildFile; fileRef = F7CDB5BD2FA33CA300F72306 /* NCNextcloudMediaViewerLoader.swift */; }; F7CDB5D32FA3448B00F72306 /* NCAudioViewerContentView.swift in Sources */ = {isa = PBXBuildFile; fileRef = F7CDB5D22FA3448A00F72306 /* NCAudioViewerContentView.swift */; }; @@ -1217,6 +1217,7 @@ 2C33C48A23E2CC26005F963B /* Notification_Service_Extension-Bridging-Header.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = "Notification_Service_Extension-Bridging-Header.h"; sourceTree = ""; }; 370D26AE248A3D7A00121797 /* NCCellMain.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NCCellMain.swift; sourceTree = ""; }; 8932E90EC4278026D86CCCC9 /* NCContextMenuComment.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NCContextMenuComment.swift; sourceTree = ""; }; + A1B2C3D330B0000100ABCDEF /* NCMediaViewerLoadingPolicy.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NCMediaViewerLoadingPolicy.swift; sourceTree = ""; }; AA3C85E72D36B08C00F74F12 /* UITestBackend.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = UITestBackend.swift; sourceTree = ""; }; AA3C85EA2D36BBF400F74F12 /* OCSResponse.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = OCSResponse.swift; sourceTree = ""; }; AA3C85ED2D36BCCB00F74F12 /* SharesResponse.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SharesResponse.swift; sourceTree = ""; }; @@ -1873,7 +1874,6 @@ F7CCAB502ECF315F00F8E68B /* NCCollectionViewCommon+SyncMetadata.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "NCCollectionViewCommon+SyncMetadata.swift"; sourceTree = ""; }; F7CDB5B62FA33CA300F72306 /* NCImageViewerContentView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NCImageViewerContentView.swift; sourceTree = ""; }; F7CDB5B82FA33CA300F72306 /* NCMediaViewerModel.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NCMediaViewerModel.swift; sourceTree = ""; }; - A1B2C3D330B0000100ABCDEF /* NCMediaViewerLoadingPolicy.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NCMediaViewerLoadingPolicy.swift; sourceTree = ""; }; F7CDB5B92FA33CA300F72306 /* NCMediaViewerPageView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NCMediaViewerPageView.swift; sourceTree = ""; }; F7CDB5BB2FA33CA300F72306 /* NCMediaViewerView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NCMediaViewerView.swift; sourceTree = ""; }; F7CDB5BD2FA33CA300F72306 /* NCNextcloudMediaViewerLoader.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NCNextcloudMediaViewerLoader.swift; sourceTree = ""; }; @@ -6347,7 +6347,7 @@ CLANG_WARN_UNREACHABLE_CODE = YES; CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; COPY_PHASE_STRIP = NO; - CURRENT_PROJECT_VERSION = 0; + CURRENT_PROJECT_VERSION = 1; DEAD_CODE_STRIPPING = YES; DEBUG_INFORMATION_FORMAT = dwarf; DEVELOPMENT_TEAM = NKUJUXUJ3B; @@ -6415,7 +6415,7 @@ CLANG_WARN_UNREACHABLE_CODE = YES; CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; COPY_PHASE_STRIP = NO; - CURRENT_PROJECT_VERSION = 0; + CURRENT_PROJECT_VERSION = 1; DEAD_CODE_STRIPPING = YES; DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; DEVELOPMENT_TEAM = NKUJUXUJ3B; diff --git a/iOSClient/Data/NCManageDatabase+MediaMetadataBackfill.swift b/iOSClient/Data/NCManageDatabase+MediaMetadataBackfill.swift index 5627471f9c..4bc7921a46 100644 --- a/iOSClient/Data/NCManageDatabase+MediaMetadataBackfill.swift +++ b/iOSClient/Data/NCManageDatabase+MediaMetadataBackfill.swift @@ -10,6 +10,7 @@ import NextcloudKit class tableMediaMetadataBackfill: Object { @Persisted(primaryKey: true) var account = "" @Persisted var offset = 0 + @Persisted var cursorDate: Date? @Persisted var lastRunDate: Date? @Persisted var lastCompletedCycleDate: Date? @@ -36,15 +37,18 @@ extension NCManageDatabase { // MARK: - Realm write func updateMediaMetadataBackfillAsync(account: String, - offset: Int) async { + offset: Int, + cursorDate: Date) async { await core.performRealmWriteAsync { realm in if let backfill = realm.object(ofType: tableMediaMetadataBackfill.self, forPrimaryKey: account) { backfill.offset = offset + backfill.cursorDate = cursorDate backfill.lastRunDate = Date() } else { let backfill = tableMediaMetadataBackfill(account: account) backfill.offset = offset + backfill.cursorDate = cursorDate backfill.lastRunDate = Date() realm.add(backfill) } @@ -55,11 +59,13 @@ extension NCManageDatabase { await core.performRealmWriteAsync { realm in if let backfill = realm.object(ofType: tableMediaMetadataBackfill.self, forPrimaryKey: account) { backfill.offset = 0 + backfill.cursorDate = nil backfill.lastRunDate = Date() backfill.lastCompletedCycleDate = Date() } else { let backfill = tableMediaMetadataBackfill(account: account) backfill.offset = 0 + backfill.cursorDate = nil backfill.lastRunDate = Date() backfill.lastCompletedCycleDate = Date() realm.add(backfill) diff --git a/iOSClient/Media/NCMediaDataSource.swift b/iOSClient/Media/NCMediaDataSource.swift index 52e66f13b5..213655d874 100644 --- a/iOSClient/Media/NCMediaDataSource.swift +++ b/iOSClient/Media/NCMediaDataSource.swift @@ -231,8 +231,8 @@ extension NCMedia { var firstDateNew = Date.distantFuture var lastDateNew = Date.distantPast - var firstDate: Date? - var lastDate: Date? + var firstVisibleCellDate: Date? + var lastVisibleCellDate: Date? var visibleCells: [NCMediaCell] = [] await MainActor.run { @@ -270,8 +270,8 @@ extension NCMedia { return date1 > date2 } - firstDate = visibleCells.first?.date - lastDate = visibleCells.last?.date + firstVisibleCellDate = visibleCells.first?.date + lastVisibleCellDate = visibleCells.last?.date if !visibleCells.isEmpty, !distant { let firstCellDate = visibleCells.first?.date @@ -324,17 +324,20 @@ extension NCMedia { guard !Task.isCancelled, self.isViewActived, self.session.account == account, - let firstDate, - let lastDate else { + let firstVisibleCellDate, + let lastVisibleCellDate else { return } // VERIFY MEDIA // - await self.verifyNetworkMedia(firstDate: firstDate, - lastDate: lastDate, + // Nextcloud cannot range-filter displayname, so same-date results are bounded by this cap. + let verificationLimit = 1000 + await self.verifyNetworkMedia(firstDate: firstVisibleCellDate, + lastDate: lastVisibleCellDate, mediaPath: tblAccount.mediaPath, - account: account) { + account: account, + limit: verificationLimit) { Task { [weak self] in guard let self else { return @@ -409,6 +412,7 @@ extension NCMedia { lastDate: Date, mediaPath: String, account: String, + limit: Int, update: @escaping () -> Void, finish: @escaping () -> Void) async { await NCMediaNetwork().searchMediaPage( @@ -417,7 +421,7 @@ extension NCMedia { lastDate: lastDate, account: account, paginate: true, - limit: 1000000) { task in + limit: limit) { task in Task { let identifier = await NCNetworking.shared.networkingTasks.createIdentifier( account: account, diff --git a/iOSClient/Processor/NCMediaMetadataBackfillProcessor.swift b/iOSClient/Processor/NCMediaMetadataBackfillProcessor.swift index 0bfe833ad5..786ec3bbe4 100644 --- a/iOSClient/Processor/NCMediaMetadataBackfillProcessor.swift +++ b/iOSClient/Processor/NCMediaMetadataBackfillProcessor.swift @@ -4,15 +4,19 @@ import Foundation import NextcloudKit +import os /// Incrementally scans the remote media archive and creates missing local metadata placeholders. /// -/// The current offset is persisted so interrupted executions can resume later. +/// The oldest date processed in a bounded result batch is persisted so later executions can resume. /// Once the archive has been fully processed, subsequent executions are skipped. final class NCMediaMetadataBackfillProcessor { + private let pagesPerBatch = 4 + /// Represents the result of a media metadata backfill execution. enum BackfillStatus { case skippedAlreadyCompleted(account: String) + case batchCompleted(account: String, processed: Int, inserted: Int, updated: Int, cursorDate: Date) case completed(account: String, processed: Int, inserted: Int, updated: Int) case failed(account: String, processed: Int, inserted: Int, updated: Int, errorCode: Int, errorDescription: String) case cancelled(account: String, processed: Int, inserted: Int, updated: Int) @@ -20,7 +24,7 @@ final class NCMediaMetadataBackfillProcessor { /// Returns whether the backfill completed successfully or was already completed. var isSuccessful: Bool { switch self { - case .skippedAlreadyCompleted, .completed: + case .skippedAlreadyCompleted, .batchCompleted, .completed: return true case .failed, .cancelled: return false @@ -33,6 +37,9 @@ final class NCMediaMetadataBackfillProcessor { case .skippedAlreadyCompleted(let account): return "Media metadata backfill skipped for account \(account): cycle already completed" + case .batchCompleted(let account, let processed, let inserted, let updated, let cursorDate): + return "Media metadata backfill batch completed for account \(account): processed \(processed) - inserted \(inserted) - updated \(updated) - cursor date \(cursorDate)" + case .completed(let account, let processed, let inserted, let updated): return "Media metadata backfill completed for account \(account): processed \(processed) - inserted \(inserted) - updated \(updated)" @@ -45,9 +52,11 @@ final class NCMediaMetadataBackfillProcessor { } } - /// Processes the remote media archive page by page and creates missing metadata placeholders. + /// Processes one bounded batch of the remote media archive and creates missing metadata placeholders. /// - /// An interrupted cycle resumes immediately from the stored offset. + /// Each completed page checkpoints its oldest date after placeholder synchronization. + /// An interrupted page is safely retried because placeholder synchronization is idempotent. + /// A full batch persists its oldest date so the next execution can continue from that boundary. /// A completed cycle starts again after the configured interval. func runBackfill( account: tableAccount, @@ -57,25 +66,49 @@ final class NCMediaMetadataBackfillProcessor { let database = NCManageDatabase.shared let state = await database.getMediaMetadataBackfillAsync(account: account.account) let cycleInterval: TimeInterval = 7 * 24 * 60 * 60 // week - var offset = state?.offset ?? 0 + let previousCursorDate = state?.cursorDate + let firstDate = previousCursorDate ?? .distantFuture + let previouslyProcessed = previousCursorDate == nil ? 0 : state?.offset ?? 0 + var pageOffset = 0 var token: String? var processed = 0 var inserted = 0 var updated = 0 + var oldestProcessedDate: Date? + + guard limit > 0 else { + return .failed( + account: account.account, + processed: 0, + inserted: 0, + updated: 0, + errorCode: NCGlobal.shared.errorPreconditionFailed, + errorDescription: "Invalid media metadata backfill page size: \(limit)" + ) + } + + let searchResultLimit = limit * pagesPerBatch if state?.offset == 0, + state?.cursorDate == nil, let lastCompletedCycleDate = state?.lastCompletedCycleDate, Date().timeIntervalSince(lastCompletedCycleDate) < cycleInterval { return .skippedAlreadyCompleted(account: account.account) } - while !Task.isCancelled { + for _ in 0..