Add incremental Messages archive and search

README.md

@@ -16,6 +16,7 @@ icloud-cli drive status
 icloud-cli drive containers --sort-by size
 icloud-cli account status
 icloud-cli snapshot
+icloud-cli providers list --format json
 icloud-cli permissions doctor
 icloud-cli shortcuts list --name Daily
 ```
@@ -24,7 +25,7 @@ The implementation reads local Safari session and metadata property lists from `
 
 If Safari session files are unreadable, the command exits with an error naming the file paths it tried. If the files are readable but empty, the error says no tabs were found instead of treating it as a permissions problem.
 
-Use `icloud-cli safari cloud-tabs probe` to check whether Safari's cross-device tab store is present and readable before using `icloud-cli safari cloud-tabs list --confirm-sensitive`. Use `icloud-cli permissions doctor` when a command reports missing or unreadable local stores; it probes source paths without reading payload content and gives the Full Disk Access hint per command. See [docs/cloudtabs.md](docs/cloudtabs.md) for the investigation notes, [docs/privacy.md](docs/privacy.md) for the privacy and permissions model, and [docs/openclaw-skill-contract.md](docs/openclaw-skill-contract.md) for the OpenClaw integration contract.
+Use `icloud-cli safari cloud-tabs probe` to check whether Safari's cross-device tab store is present and readable before using `icloud-cli safari cloud-tabs list --confirm-sensitive`. Use `icloud-cli permissions doctor` when a command reports missing or unreadable local stores; it probes source paths without reading payload content and gives the Full Disk Access hint per command. `icloud-cli providers list` exposes a static, versioned capability registry without reading live data; see [docs/provider-manifest.md](docs/provider-manifest.md) for its schema and compatibility policy. Recursive Drive and Finder-tag commands emit bounded crawl reports documented in [docs/crawl-budgets.md](docs/crawl-budgets.md), while [docs/provider-archives.md](docs/provider-archives.md) defines resumable metadata archives. See [docs/cloudtabs.md](docs/cloudtabs.md) for the investigation notes, [docs/privacy.md](docs/privacy.md) for the privacy and permissions model, and [docs/openclaw-skill-contract.md](docs/openclaw-skill-contract.md) for the OpenClaw integration contract.
 
 ## Build
 

Sources/ICloudCLICore/AppleMetadataInventories.swift

@@ -1173,11 +1173,29 @@ public struct FinderTagsReader: Sendable {
 
     public func items(tag: String, path: String?, limit: Int) throws -> [TaggedDriveItem] {
         let scanLimit = max(200, bounded(limit, defaultValue: 50, max: 1_000) * 5)
-        let files = try ICloudDriveInventoryReader(rootDirectory: driveRoot).listFiles(path: path, depth: Int.max, limit: scanLimit)
-        return files
-            .filter { $0.name.localizedCaseInsensitiveContains(tag) || $0.path.localizedCaseInsensitiveContains(".\(tag).") }
-            .prefix(max(1, limit))
+        let budget = CrawlBudget(scanLimit: scanLimit, wallClockLimitMilliseconds: CrawlBudget.defaultDrive.wallClockLimitMilliseconds)
+        return try itemsReport(tag: tag, path: path, limit: limit, budget: budget).data
+    }
+
+    public func itemsReport(tag: String, path: String?, limit: Int, budget: CrawlBudget = .defaultDrive) throws -> CrawlReport<[TaggedDriveItem]> {
+        let crawl = try ICloudDriveInventoryReader(rootDirectory: driveRoot).listFilesReport(path: path, depth: Int.max, budget: budget)
+        let resultLimit = max(1, limit)
+        let matches = crawl.data.filter { $0.name.localizedCaseInsensitiveContains(tag) || $0.path.localizedCaseInsensitiveContains(".\(tag).") }
+        let items = matches
+            .prefix(resultLimit)
             .map { TaggedDriveItem(path: $0.path, modifiedAt: $0.modifiedAt, iCloudStatus: $0.iCloudStatus) }
+        return CrawlReport(
+            providerId: "tags",
+            state: crawl.state,
+            data: items,
+            scannedCount: crawl.scannedCount,
+            resultCount: items.count,
+            totalAvailable: crawl.state == .complete ? matches.count : nil,
+            budget: budget,
+            resultLimit: resultLimit,
+            elapsedMilliseconds: crawl.elapsedMilliseconds,
+            nextAction: crawl.nextAction
+        )
     }
 }
 

Sources/ICloudCLICore/CacheWatch.swift

@@ -5,6 +5,13 @@ public struct CacheEnvelope: Codable, Equatable, Sendable {
     public let ok: Bool
     public let error: String?
     public let data: String?
+    public let failure: CacheFailure?
+}
+
+public struct CacheFailure: Codable, Equatable, Sendable {
+    public let code: String
+    public let providerId: String
+    public let guidance: String
 }
 
 public struct CacheStatusEntry: Codable, Equatable, Sendable {
@@ -13,6 +20,7 @@ public struct CacheStatusEntry: Codable, Equatable, Sendable {
     public let updatedAt: String?
     public let ok: Bool?
     public let error: String?
+    public let failure: CacheFailure?
 }
 
 public enum CacheWatchError: Error, LocalizedError, Equatable {
@@ -31,25 +39,33 @@ public struct CacheWatchStore: Sendable {
     public static let defaultCommands = ["safari-tabs", "drive-list", "photos-screenshots", "storage-status"]
 
     public let outputDirectory: URL
+    private let snapshotter: CacheSnapshotter
 
     public init(outputDirectory: URL = FileManager.default.homeDirectoryForCurrentUser.appendingPathComponent(".icloud-cli/cache")) {
         self.outputDirectory = outputDirectory
+        self.snapshotter = Self.defaultSnapshot
+    }
+
+    init(outputDirectory: URL, snapshotter: @escaping CacheSnapshotter) {
+        self.outputDirectory = outputDirectory
+        self.snapshotter = snapshotter
     }
 
-    public func refresh(commands: [String]) throws -> [CacheStatusEntry] {
+    public func refresh(commands: [String], budget: CrawlBudget = .defaultPolling) throws -> [CacheStatusEntry] {
         try ensureCacheDirectory()
         var statuses: [CacheStatusEntry] = []
         for command in commands {
             let envelope: CacheEnvelope
             do {
-                let data = try snapshot(command: command)
-                envelope = CacheEnvelope(updatedAt: now(), ok: true, error: nil, data: data)
+                let data = try snapshotter(command, budget)
+                envelope = CacheEnvelope(updatedAt: now(), ok: true, error: nil, data: data, failure: nil)
             } catch {
                 let previous = try? readEnvelope(command: command)
-                envelope = CacheEnvelope(updatedAt: now(), ok: false, error: error.localizedDescription, data: previous?.data)
+                let failure = redactedFailure(error, command: command)
+                envelope = CacheEnvelope(updatedAt: now(), ok: false, error: failure.message, data: previous?.data, failure: failure.detail)
             }
             try write(envelope, command: command)
-            statuses.append(CacheStatusEntry(command: command, path: fileURL(for: command).path, updatedAt: envelope.updatedAt, ok: envelope.ok, error: envelope.error))
+            statuses.append(CacheStatusEntry(command: command, path: fileURL(for: command).path, updatedAt: envelope.updatedAt, ok: envelope.ok, error: envelope.error, failure: envelope.failure))
         }
         return statuses
     }
@@ -64,7 +80,7 @@ public struct CacheWatchStore: Sendable {
         return files.filter { $0.pathExtension == "json" }.sorted { $0.lastPathComponent < $1.lastPathComponent }.map { url in
             let command = url.deletingPathExtension().lastPathComponent
             let envelope = try? readEnvelope(command: command)
-            return CacheStatusEntry(command: command, path: url.path, updatedAt: envelope?.updatedAt, ok: envelope?.ok, error: envelope?.error)
+            return CacheStatusEntry(command: command, path: url.path, updatedAt: envelope?.updatedAt, ok: envelope?.ok, error: envelope?.error, failure: envelope?.failure)
         }
     }
 
@@ -75,19 +91,38 @@ public struct CacheWatchStore: Sendable {
         try FileManager.default.setAttributes([.posixPermissions: 0o700], ofItemAtPath: outputDirectory.path)
     }
 
-    private func snapshot(command: String) throws -> String {
+    private static func defaultSnapshot(command: String, budget: CrawlBudget) throws -> String {
+        let startedAt = ProcessInfo.processInfo.systemUptime
+        let data: String
         switch command {
         case "safari-tabs":
-            return try json(SafariTabsReader().readTabs())
+            data = try json(SafariTabsReader().readTabs())
         case "drive-list":
-            return try json(ICloudDriveInventoryReader().listFiles())
+            let report = try ICloudDriveInventoryReader().listFilesReport(budget: budget)
+            guard report.state == .complete else { throw CrawlBudgetExceeded(providerId: "drive", state: report.state) }
+            data = try json(report)
         case "photos-screenshots":
-            return try json(PhotosInventoryReader().listScreenshots())
+            data = try json(PhotosInventoryReader().listScreenshots())
         case "storage-status":
-            return try json(ICloudStorageStatusReader().readStatus())
+            data = try json(ICloudStorageStatusReader().readStatus())
         default:
             throw CacheWatchError.missingCommand(command)
         }
+        let elapsed = Int((ProcessInfo.processInfo.systemUptime - startedAt) * 1_000)
+        guard elapsed < budget.wallClockLimitMilliseconds else {
+            throw CrawlBudgetExceeded(providerId: command.split(separator: "-").first.map(String.init) ?? "unknown", state: .timeout)
+        }
+        return data
+    }
+
+    private func redactedFailure(_ error: Error, command: String) -> (message: String, detail: CacheFailure) {
+        let providerId = command.split(separator: "-").first.map(String.init) ?? "unknown"
+        if let exceeded = error as? CrawlBudgetExceeded {
+            let code = exceeded.state == .timeout ? "timeout" : "partial"
+            let message = exceeded.state == .timeout ? "provider crawl timed out" : "provider crawl reached its scan budget"
+            return (message, CacheFailure(code: code, providerId: exceeded.providerId, guidance: "Narrow the provider scope or increase its explicit crawl budget."))
+        }
+        return ("provider crawl failed", CacheFailure(code: "provider-error", providerId: providerId, guidance: "Run the provider directly for local diagnostic details."))
     }
 
     private func write(_ envelope: CacheEnvelope, command: String) throws {
@@ -111,7 +146,7 @@ public struct CacheWatchStore: Sendable {
         outputDirectory.appendingPathComponent(command).appendingPathExtension("json")
     }
 
-    private func json<T: Encodable>(_ value: T) throws -> String {
+    private static func json<T: Encodable>(_ value: T) throws -> String {
         String(decoding: try JSONEncoder.pretty.encode(value), as: UTF8.self)
     }
 
@@ -120,6 +155,8 @@ public struct CacheWatchStore: Sendable {
     }
 }
 
+typealias CacheSnapshotter = @Sendable (String, CrawlBudget) throws -> String
+
 private extension JSONEncoder {
     static var pretty: JSONEncoder {
         let encoder = JSONEncoder()