Merge pull request #6461 from nextcloud/feature/file-provider-fast-in…

…it-sync

Add option to perform fast synchronisation runs in File Provider sync engine

resources.qrc

@@ -67,5 +67,6 @@
         <file>src/gui/macOS/ui/FileProviderEvictionDialog.qml</file>
         <file>src/gui/macOS/ui/FileProviderSyncStatus.qml</file>
         <file>src/gui/macOS/ui/FileProviderStorageInfo.qml</file>
+        <file>src/gui/macOS/ui/FileProviderFastEnumerationSettings.qml</file>
     </qresource>
 </RCC>

shell_integration/MacOSX/NextcloudIntegration/FileProviderExt/FileProviderConfig.swift

@@ -0,0 +1,40 @@
+//
+//  FileProviderConfig.swift
+//  FileProviderExt
+//
+//  Created by Claudio Cambra on 5/2/24.
+//
+
+import FileProvider
+import Foundation
+
+struct FileProviderConfig {
+    private enum ConfigKey: String {
+        case fastEnumerationEnabled = "fastEnumerationEnabled"
+    }
+
+    let domainIdentifier: NSFileProviderDomainIdentifier
+
+    private var internalConfig: [String: Any] {
+        get {
+            let defaults = UserDefaults.standard
+            if let settings = defaults.dictionary(forKey: domainIdentifier.rawValue) {
+                return settings
+            }
+            let dictionary: [String: Any] = [:]
+            defaults.setValue(dictionary, forKey: domainIdentifier.rawValue)
+            return dictionary
+        }
+        set {
+            let defaults = UserDefaults.standard
+            defaults.setValue(newValue, forKey: domainIdentifier.rawValue)
+        }
+    }
+
+    var fastEnumerationEnabled: Bool {
+        get { internalConfig[ConfigKey.fastEnumerationEnabled.rawValue] as? Bool ?? true }
+        set { internalConfig[ConfigKey.fastEnumerationEnabled.rawValue] = newValue }
+    }
+
+    lazy var fastEnumerationSet = internalConfig[ConfigKey.fastEnumerationEnabled.rawValue] != nil
+}

shell_integration/MacOSX/NextcloudIntegration/FileProviderExt/FileProviderEnumerator+SyncEngine.swift

@@ -196,23 +196,31 @@ extension FileProviderEnumerator {
         dispatchGroup.wait()
 
         guard criticalError == nil else {
+            Logger.enumeration.error(
+                "Received critical error stopping further scanning: \(criticalError!.errorDescription, privacy: .public)"
+            )
             return ([], [], [], [], error: criticalError)
         }
 
         var childDirectoriesToScan: [NextcloudItemMetadataTable] = []
-        var candidateMetadatas: [NextcloudItemMetadataTable] =
-            if scanChangesOnly {
-                allUpdatedMetadatas + allNewMetadatas
-            } else {
-                allMetadatas
-            }
+        var candidateMetadatas: [NextcloudItemMetadataTable]
+
+        if scanChangesOnly, fastEnumeration {
+            candidateMetadatas = allUpdatedMetadatas
+        } else if scanChangesOnly {
+            candidateMetadatas = allUpdatedMetadatas + allNewMetadatas
+        } else {
+            candidateMetadatas = allMetadatas
+        }
 
         for candidateMetadata in candidateMetadatas {
             if candidateMetadata.directory {
                 childDirectoriesToScan.append(candidateMetadata)
             }
         }
 
+        Logger.enumeration.debug("Candidate metadatas for further scan: \(candidateMetadatas, privacy: .public)")
+
         if childDirectoriesToScan.isEmpty {
             return (
                 metadatas: allMetadatas, newMetadatas: allNewMetadatas,
@@ -221,11 +229,12 @@ extension FileProviderEnumerator {
         }
 
         for childDirectory in childDirectoriesToScan {
+            Logger.enumeration.debug(
+                "About to recursively scan: \(childDirectory.urlBase, privacy: .public) with etag: \(childDirectory.etag, privacy: .public)"
+            )
             let childScanResult = scanRecursively(
-                childDirectory,
-                ncAccount: ncAccount,
-                ncKit: ncKit,
-                scanChangesOnly: scanChangesOnly)
+                childDirectory, ncAccount: ncAccount, ncKit: ncKit, scanChangesOnly: scanChangesOnly
+            )
 
             allMetadatas += childScanResult.metadatas
             allNewMetadatas += childScanResult.newMetadatas

shell_integration/MacOSX/NextcloudIntegration/FileProviderExt/FileProviderEnumerator.swift

@@ -28,21 +28,24 @@ class FileProviderEnumerator: NSObject, NSFileProviderEnumerator {
     private static let maxItemsPerFileProviderPage = 100
     let ncAccount: NextcloudAccount
     let ncKit: NextcloudKit
+    let fastEnumeration: Bool
     var serverUrl: String = ""
     var isInvalidated = false
 
     private static func isSystemIdentifier(_ identifier: NSFileProviderItemIdentifier) -> Bool {
         identifier == .rootContainer || identifier == .trashContainer || identifier == .workingSet
     }
-
+    
     init(
         enumeratedItemIdentifier: NSFileProviderItemIdentifier,
         ncAccount: NextcloudAccount,
-        ncKit: NextcloudKit
+        ncKit: NextcloudKit,
+        fastEnumeration: Bool = true
     ) {
         self.enumeratedItemIdentifier = enumeratedItemIdentifier
         self.ncAccount = ncAccount
         self.ncKit = ncKit
+        self.fastEnumeration = fastEnumeration
 
         if FileProviderEnumerator.isSystemIdentifier(enumeratedItemIdentifier) {
             Logger.enumeration.debug(

shell_integration/MacOSX/NextcloudIntegration/FileProviderExt/FileProviderExtension.swift

@@ -25,16 +25,17 @@ import OSLog
     lazy var ncKitBackground = NKBackground(nkCommonInstance: ncKit.nkCommonInstance)
     lazy var socketClient: LocalSocketClient? = {
         guard let containerUrl = pathForAppGroupContainer() else {
-            Logger.fileProviderExtension.critical("Won't start client, no container url")
-            return nil
+            Logger.fileProviderExtension.critical("Won't start socket client, no container url")
+            return nil;
         }
+
         let socketPath = containerUrl.appendingPathComponent(
             ".fileprovidersocket", conformingTo: .archive)
         let lineProcessor = FileProviderSocketLineProcessor(delegate: self)
         return LocalSocketClient(socketPath: socketPath.path, lineProcessor: lineProcessor)
     }()
 
-    let urlSessionIdentifier: String = "com.nextcloud.session.upload.fileproviderext"
+    let urlSessionIdentifier = "com.nextcloud.session.upload.fileproviderext"
     let urlSessionMaximumConnectionsPerHost = 5
     lazy var urlSession: URLSession = {
         let configuration = URLSessionConfiguration.background(withIdentifier: urlSessionIdentifier)
@@ -46,13 +47,28 @@ import OSLog
         configuration.sharedContainerIdentifier = appGroupIdentifier
 
         let session = URLSession(
-            configuration: configuration, delegate: ncKitBackground,
-            delegateQueue: OperationQueue.main)
+            configuration: configuration,
+            delegate: ncKitBackground,
+            delegateQueue: OperationQueue.main
+        )
         return session
     }()
 
+    // Whether or not we are going to recursively scan new folders when they are discovered.
+    // Apple's recommendation is that we should always scan the file hierarchy fully.
+    // This does lead to long load times when a file provider domain is initially configured.
+    // We can instead do a fast enumeration where we only scan folders as the user navigates through
+    // them, thereby avoiding this issue; the trade-off is that we will be unable to detect
+    // materialised file moves to unexplored folders, therefore deleting the item when we could have
+    // just moved it instead.
+    //
+    // Since it's not desirable to cancel a long recursive enumeration half-way through, we do the
+    // fast enumeration by default. We prompt the user on the client side to run a proper, full
+    // enumeration if they want for safety.
+    lazy var config = FileProviderConfig(domainIdentifier: domain.identifier)
+
     required init(domain: NSFileProviderDomain) {
-        // The containing application must create a domain using
+        // The containing application must create a domain using 
         // `NSFileProviderManager.add(_:, completionHandler:)`. The system will then launch the
         // application extension process, call `FileProviderExtension.init(domain:)` to instantiate
         // the extension for that domain, and call methods on the instance.
@@ -64,7 +80,8 @@ import OSLog
     func invalidate() {
         // TODO: cleanup any resources
         Logger.fileProviderExtension.debug(
-            "Extension for domain \(self.domain.displayName, privacy: .public) is being torn down")
+            "Extension for domain \(self.domain.displayName, privacy: .public) is being torn down"
+        )
     }
 
     // MARK: NSFileProviderReplicatedExtension protocol methods
@@ -787,7 +804,11 @@ import OSLog
         }
 
         return FileProviderEnumerator(
-            enumeratedItemIdentifier: containerItemIdentifier, ncAccount: ncAccount, ncKit: ncKit)
+            enumeratedItemIdentifier: containerItemIdentifier,
+            ncAccount: ncAccount,
+            ncKit: ncKit,
+            fastEnumeration: config.fastEnumerationEnabled
+        )
     }
 
     func materializedItemsDidChange(completionHandler: @escaping () -> Void) {

shell_integration/MacOSX/NextcloudIntegration/FileProviderExt/Services/ClientCommunicationProtocol.h

@@ -25,6 +25,8 @@
                         password:(NSString *)password;
 - (void)removeAccountConfig;
 - (void)createDebugLogStringWithCompletionHandler:(void(^)(NSString *debugLogString, NSError *error))completionHandler;
+- (void)getFastEnumerationStateWithCompletionHandler:(void(^)(BOOL enabled, BOOL set))completionHandler;
+- (void)setFastEnumerationEnabled:(BOOL)enabled;
 
 @end
 

shell_integration/MacOSX/NextcloudIntegration/FileProviderExt/Services/ClientCommunicationService.swift

@@ -77,4 +77,29 @@ class ClientCommunicationService: NSObject, NSFileProviderServiceSource, NSXPCLi
             completionHandler(logs.joined(separator: "\n"), nil)
         }
     }
+
+    func getFastEnumerationState(completionHandler: @escaping (Bool, Bool) -> Void) {
+        let enabled = fpExtension.config.fastEnumerationEnabled
+        let set = fpExtension.config.fastEnumerationSet
+        completionHandler(enabled, set)
+    }
+
+    func setFastEnumerationEnabled(_ enabled: Bool) {
+        fpExtension.config.fastEnumerationEnabled = enabled
+        Logger.fileProviderExtension.info("Fast enumeration setting changed to: \(enabled, privacy: .public)")
+
+        guard enabled else { return }
+        // If enabled, start full enumeration
+        guard let fpManager = NSFileProviderManager(for: fpExtension.domain) else {
+            let domainName = self.fpExtension.domain.displayName
+            Logger.fileProviderExtension.error("Could not get file provider manager for domain \(domainName, privacy: .public), cannot run enumeration after fast enumeration setting change")
+            return
+        }
+
+        fpManager.signalEnumerator(for: .workingSet) { error in
+            if error != nil {
+                Logger.fileProviderExtension.error("Error signalling enumerator for working set, received error: \(error!.localizedDescription, privacy: .public)")
+            }
+        }
+    }
 }

shell_integration/MacOSX/NextcloudIntegration/NextcloudIntegration.xcodeproj/project.pbxproj

@@ -40,6 +40,7 @@
 		53903D37295618A400D0B308 /* LineProcessor.h in Headers */ = {isa = PBXBuildFile; fileRef = 53903D36295618A400D0B308 /* LineProcessor.h */; settings = {ATTRIBUTES = (Public, ); }; };
 		539158AC27BE71A900816F56 /* FinderSyncSocketLineProcessor.m in Sources */ = {isa = PBXBuildFile; fileRef = 539158AB27BE71A900816F56 /* FinderSyncSocketLineProcessor.m */; };
 		53D056312970594F00988392 /* LocalFilesUtils.swift in Sources */ = {isa = PBXBuildFile; fileRef = 53D056302970594F00988392 /* LocalFilesUtils.swift */; };
+		53D666612B70C9A70042C03D /* FileProviderConfig.swift in Sources */ = {isa = PBXBuildFile; fileRef = 53D666602B70C9A70042C03D /* FileProviderConfig.swift */; };
 		53ED472029C5E64200795DB1 /* FileProviderEnumerator+SyncEngine.swift in Sources */ = {isa = PBXBuildFile; fileRef = 53ED471F29C5E64200795DB1 /* FileProviderEnumerator+SyncEngine.swift */; };
 		53ED472829C88E7000795DB1 /* NextcloudItemMetadataTable+NKFile.swift in Sources */ = {isa = PBXBuildFile; fileRef = 53ED472729C88E7000795DB1 /* NextcloudItemMetadataTable+NKFile.swift */; };
 		53ED473029C9CE0B00795DB1 /* FileProviderExtension+ClientInterface.swift in Sources */ = {isa = PBXBuildFile; fileRef = 53ED472F29C9CE0B00795DB1 /* FileProviderExtension+ClientInterface.swift */; };
@@ -172,6 +173,7 @@
 		539158B127BE891500816F56 /* LocalSocketClient.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = LocalSocketClient.h; sourceTree = "<group>"; };
 		539158B227BEC98A00816F56 /* LocalSocketClient.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = LocalSocketClient.m; sourceTree = "<group>"; };
 		53D056302970594F00988392 /* LocalFilesUtils.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LocalFilesUtils.swift; sourceTree = "<group>"; };
+		53D666602B70C9A70042C03D /* FileProviderConfig.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FileProviderConfig.swift; sourceTree = "<group>"; };
 		53ED471F29C5E64200795DB1 /* FileProviderEnumerator+SyncEngine.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "FileProviderEnumerator+SyncEngine.swift"; sourceTree = "<group>"; };
 		53ED472729C88E7000795DB1 /* NextcloudItemMetadataTable+NKFile.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "NextcloudItemMetadataTable+NKFile.swift"; sourceTree = "<group>"; };
 		53ED472F29C9CE0B00795DB1 /* FileProviderExtension+ClientInterface.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "FileProviderExtension+ClientInterface.swift"; sourceTree = "<group>"; };
@@ -272,9 +274,10 @@
 		538E396B27F4765000FA63D5 /* FileProviderExt */ = {
 			isa = PBXGroup;
 			children = (
-				5350E4C72B0C368B00F276CB /* Services */,
 				5318AD8F29BF406500CBB71C /* Database */,
 				5352E85929B7BFB4002CE85C /* Extensions */,
+				5350E4C72B0C368B00F276CB /* Services */,
+				53D666602B70C9A70042C03D /* FileProviderConfig.swift */,
 				538E397027F4765000FA63D5 /* FileProviderEnumerator.swift */,
 				53ED471F29C5E64200795DB1 /* FileProviderEnumerator+SyncEngine.swift */,
 				538E396C27F4765000FA63D5 /* FileProviderExtension.swift */,
@@ -592,6 +595,7 @@
 			buildActionMask = 2147483647;
 			files = (
 				5352E85B29B7BFE6002CE85C /* Progress+Extensions.swift in Sources */,
+				53D666612B70C9A70042C03D /* FileProviderConfig.swift in Sources */,
 				536EFC36295E3C1100F4CB13 /* NextcloudAccount.swift in Sources */,
 				53ED473029C9CE0B00795DB1 /* FileProviderExtension+ClientInterface.swift in Sources */,
 				538E396D27F4765000FA63D5 /* FileProviderExtension.swift in Sources */,

src/gui/macOS/fileprovider.h

@@ -37,10 +37,9 @@ class FileProvider : public QObject
     static FileProvider *instance();
     ~FileProvider() override;
 
-    static bool fileProviderAvailable();
+    [[nodiscard]] static bool fileProviderAvailable();
 
-public slots:
-    void createDebugArchiveForDomain(const QString &domainIdentifier, const QString &filename) const;
+    [[nodiscard]] FileProviderXPC *xpc() const;
 
 private slots:
     void configureXPC();

src/gui/macOS/fileprovider_mac.mm

@@ -97,9 +97,9 @@
     }
 }
 
-void FileProvider::createDebugArchiveForDomain(const QString &domainIdentifier, const QString &filename) const
+FileProviderXPC *FileProvider::xpc() const
 {
-    _xpc->createDebugArchiveForExtension(domainIdentifier, filename);
+    return _xpc.get();
 }
 
 } // namespace Mac

src/gui/macOS/fileprovidersettingscontroller.h

@@ -44,12 +44,15 @@ class FileProviderSettingsController : public QObject
     [[nodiscard]] Q_INVOKABLE float localStorageUsageGbForAccount(const QString &userIdAtHost) const;
     [[nodiscard]] unsigned long long remoteStorageUsageForAccount(const QString &userIdAtHost) const;
     [[nodiscard]] Q_INVOKABLE float remoteStorageUsageGbForAccount(const QString &userIdAtHost) const;
+    [[nodiscard]] Q_INVOKABLE bool fastEnumerationEnabledForAccount(const QString &userIdAtHost) const;
+    [[nodiscard]] Q_INVOKABLE bool fastEnumerationSetForAccount(const QString &userIdAtHost) const;
 
     [[nodiscard]] Q_INVOKABLE QAbstractListModel *materialisedItemsModelForAccount(const QString &userIdAtHost);
     [[nodiscard]] Q_INVOKABLE FileProviderDomainSyncStatus *domainSyncStatusForAccount(const QString &userIdAtHost) const;
 
 public slots:
     void setVfsEnabledForAccount(const QString &userIdAtHost, const bool setEnabled);
+    void setFastEnumerationEnabledForAccount(const QString &userIdAtHost, const bool setEnabled);
 
     void createEvictionWindowForAccount(const QString &userIdAtHost);
     void signalFileProviderDomain(const QString &userIdAtHost);
@@ -60,6 +63,8 @@ public slots:
     void localStorageUsageForAccountChanged(const QString &userIdAtHost);
     void remoteStorageUsageForAccountChanged(const QString &userIdAtHost);
     void materialisedItemsForAccountChanged(const QString &userIdAtHost);
+    void fastEnumerationEnabledForAccountChanged(const QString &userIdAtHost);
+    void fastEnumerationSetForAccountChanged(const QString &userIdAtHost);
 
 private:
     explicit FileProviderSettingsController(QObject *parent = nullptr);

src/gui/macOS/fileprovidersettingscontroller_mac.mm

@@ -362,6 +362,44 @@ void initialCheck()
     }
 }
 
+bool FileProviderSettingsController::fastEnumerationSetForAccount(const QString &userIdAtHost) const
+{
+    const auto xpc = FileProvider::instance()->xpc();
+    if (!xpc) {
+        return false;
+    }
+    if (const auto state = xpc->fastEnumerationStateForExtension(userIdAtHost)) {
+        return state->second;
+    }
+    return false;
+}
+
+bool FileProviderSettingsController::fastEnumerationEnabledForAccount(const QString &userIdAtHost) const
+{
+    const auto xpc = FileProvider::instance()->xpc();
+    if (!xpc) {
+        return true;
+    }
+    if (const auto fastEnumerationState = xpc->fastEnumerationStateForExtension(userIdAtHost)) {
+        return fastEnumerationState->first;
+    }
+    return true;
+}
+
+void FileProviderSettingsController::setFastEnumerationEnabledForAccount(const QString &userIdAtHost, const bool setEnabled)
+{
+    const auto xpc = FileProvider::instance()->xpc();
+    if (!xpc) {
+        // Reset state of UI elements
+        emit fastEnumerationEnabledForAccountChanged(userIdAtHost);
+        emit fastEnumerationSetForAccountChanged(userIdAtHost);
+        return;
+    }
+    xpc->setFastEnumerationEnabledForExtension(userIdAtHost, setEnabled);
+    emit fastEnumerationEnabledForAccountChanged(userIdAtHost);
+    emit fastEnumerationSetForAccountChanged(userIdAtHost);
+}
+
 unsigned long long FileProviderSettingsController::localStorageUsageForAccount(const QString &userIdAtHost) const
 {
     return d->localStorageUsageForAccount(userIdAtHost);
@@ -437,7 +475,13 @@ void initialCheck()
     if (filename.isEmpty()) {
         return;
     }
-    FileProvider::instance()->createDebugArchiveForDomain(userIdAtHost, filename);
+
+    const auto xpc = FileProvider::instance()->xpc();
+    if (!xpc) {
+        qCWarning(lcFileProviderSettingsController) << "Could not create debug archive, FileProviderXPC is not available.";
+        return;
+    }
+    xpc->createDebugArchiveForExtension(userIdAtHost, filename);
 }
 
 FileProviderDomainSyncStatus *FileProviderSettingsController::domainSyncStatusForAccount(const QString &userIdAtHost) const