From 8cb5cbff063f1a1ab6f0d008d9496032568da1aa Mon Sep 17 00:00:00 2001 From: Hermes Date: Sat, 11 Jul 2026 22:53:58 +0100 Subject: [PATCH 1/4] Snapshot live SQLite stores before querying Add a private read-only SQLite snapshot engine with WAL and SHM companions, query-only enforcement, busy and process timeouts, structured failures, and deterministic cleanup. Migrate Messages and Safari history and document remaining providers.\n\nCloses #90. --- Sources/ICloudCLICore/LocalInventories.swift | 52 ++++- Sources/ICloudCLICore/ProviderManifest.swift | 4 +- .../ICloudCLICore/SQLiteSnapshotQuery.swift | 132 +++++++++++++ .../ProviderManifestTests.swift | 2 + .../SQLiteSnapshotQueryTests.swift | 179 ++++++++++++++++++ docs/privacy.md | 2 + docs/sqlite-snapshots.md | 19 ++ 7 files changed, 384 insertions(+), 6 deletions(-) create mode 100644 Sources/ICloudCLICore/SQLiteSnapshotQuery.swift create mode 100644 Tests/ICloudCLICoreTests/SQLiteSnapshotQueryTests.swift create mode 100644 docs/sqlite-snapshots.md diff --git a/Sources/ICloudCLICore/LocalInventories.swift b/Sources/ICloudCLICore/LocalInventories.swift index ccb41c9..f17d15e 100644 --- a/Sources/ICloudCLICore/LocalInventories.swift +++ b/Sources/ICloudCLICore/LocalInventories.swift @@ -4,6 +4,8 @@ public enum LocalInventoryError: Error, LocalizedError, Equatable { case missingRoot(String) case missingStore(String) case permissionDenied(String) + case lockedStore(String) + case queryTimeout(String) case sensitiveConfirmationRequired(String) case unsupportedSchema(store: String, detail: String) case sqliteFailure(String) @@ -14,6 +16,8 @@ public enum LocalInventoryError: Error, LocalizedError, Equatable { case .missingStore(let path): return "Inventory store not available: \(path)" case .permissionDenied(let path): return "Permission denied reading local inventory store: \(path). Grant Full Disk Access to the calling terminal or agent process, then retry." + case .lockedStore(let path): return "Local inventory store is locked or busy: \(path)" + case .queryTimeout(let path): return "Timed out querying local inventory store: \(path)" case .sensitiveConfirmationRequired(let command): return "\(command) reads high-sensitivity local data; rerun with --confirm-sensitive" case .unsupportedSchema(let store, let detail): @@ -247,9 +251,22 @@ public struct NewsTopicEntry: Codable, Equatable, Sendable { public struct LocalSQLiteInventoryReader: Sendable { public let database: URL + private let snapshotsLiveStores: Bool + private let reportedStore: String + private let snapshotWorkspace: URL? public init(database: URL) { self.database = database + self.snapshotsLiveStores = true + self.reportedStore = database.path + self.snapshotWorkspace = nil + } + + private init(database: URL, snapshotsLiveStores: Bool, reportedStore: String, snapshotWorkspace: URL?) { + self.database = database + self.snapshotsLiveStores = snapshotsLiveStores + self.reportedStore = reportedStore + self.snapshotWorkspace = snapshotWorkspace } public func notes(folder: String?, modifiedSince: String?, includeBody: Bool) throws -> [NoteEntry] { @@ -375,6 +392,11 @@ public struct LocalSQLiteInventoryReader: Sendable { public func safariHistory(confirmSensitive: Bool, since: String?, until: String?, limit: Int, redactURLs: Bool) throws -> [SafariHistoryEntry] { guard confirmSensitive else { throw LocalInventoryError.sensitiveConfirmationRequired("icloud-cli safari history") } + if snapshotsLiveStores { + return try withSnapshot { reader in + try reader.safariHistory(confirmSensitive: true, since: since, until: until, limit: limit, redactURLs: redactURLs) + } + } if try tableExists("history_items"), try tableExists("history_visits") { return try appleSafariHistory(since: since, until: until, limit: limit, redactURLs: redactURLs) } @@ -414,11 +436,14 @@ public struct LocalSQLiteInventoryReader: Sendable { } public func messageConversations(limit: Int = 50) throws -> [MessageConversation] { + if snapshotsLiveStores { + return try withSnapshot { try $0.messageConversations(limit: limit) } + } if try tableExists("message_conversations") { return try query("SELECT chatIdentifier, displayName, participantCount, lastMessageAt, messageCount FROM message_conversations ORDER BY lastMessageAt DESC LIMIT \(bounded(limit, defaultValue: 50, max: 1000));") } guard try tableExists("chat"), try tableExists("message"), try tableExists("chat_message_join") else { - throw LocalInventoryError.unsupportedSchema(store: database.path, detail: "missing message_conversations or chat/message/chat_message_join tables") + throw LocalInventoryError.unsupportedSchema(store: reportedStore, detail: "missing message_conversations or chat/message/chat_message_join tables") } return try query(""" SELECT @@ -439,13 +464,18 @@ public struct LocalSQLiteInventoryReader: Sendable { public func recentMessages(confirmSensitive: Bool, includeBody: Bool, since: String?, limit: Int) throws -> [MessageRecentEntry] { guard confirmSensitive else { throw LocalInventoryError.sensitiveConfirmationRequired("icloud-cli messages recent") } + if snapshotsLiveStores { + return try withSnapshot { reader in + try reader.recentMessages(confirmSensitive: true, includeBody: includeBody, since: since, limit: limit) + } + } let floor = since ?? ISO8601DateFormatter().string(from: Date(timeIntervalSinceNow: -86_400)) let bodyColumn = includeBody ? "body" : "NULL AS body" if try tableExists("recent_messages") { return try query("SELECT chatIdentifier, sender, sentAt, isFromMe, \(bodyColumn) FROM recent_messages WHERE sentAt >= '\(sqlEscape(floor))' ORDER BY sentAt DESC LIMIT \(bounded(limit, defaultValue: 20, max: 1000));") } guard try tableExists("message"), try tableExists("chat"), try tableExists("chat_message_join") else { - throw LocalInventoryError.unsupportedSchema(store: database.path, detail: "missing recent_messages or message/chat/chat_message_join tables") + throw LocalInventoryError.unsupportedSchema(store: reportedStore, detail: "missing recent_messages or message/chat/chat_message_join tables") } let appleBodyColumn = includeBody ? "m.text" : "NULL AS body" let appleFloor = since.flatMap { Int64($0) } @@ -612,7 +642,12 @@ public struct LocalSQLiteInventoryReader: Sendable { } private func query(_ sql: String) throws -> [T] { - guard FileManager.default.fileExists(atPath: database.path) else { throw LocalInventoryError.missingStore(database.path) } + if !snapshotsLiveStores { + guard let snapshotWorkspace else { throw LocalInventoryError.sqliteFailure("SQLite snapshot workspace is unavailable") } + return try SQLiteSnapshotQueryEngine(source: database, reportedStore: reportedStore) + .querySnapshot(database, workspace: snapshotWorkspace, sql: sql) + } + guard FileManager.default.fileExists(atPath: database.path) else { throw LocalInventoryError.missingStore(reportedStore) } let process = Process() process.executableURL = URL(fileURLWithPath: "/usr/bin/sqlite3") process.arguments = ["-readonly", "-json", database.path, sql] @@ -627,7 +662,7 @@ public struct LocalSQLiteInventoryReader: Sendable { let data = outputPipe.fileHandleForReading.readDataToEndOfFile() let errorData = errorPipe.fileHandleForReading.readDataToEndOfFile() guard process.terminationStatus == 0 else { - throw sqliteError(from: errorData, store: database.path) + throw sqliteError(from: errorData, store: reportedStore) } if data.isEmpty { return [] } return try JSONDecoder().decode([T].self, from: data) @@ -643,6 +678,12 @@ public struct LocalSQLiteInventoryReader: Sendable { return rows.contains { $0.name == column } } + private func withSnapshot(_ operation: (LocalSQLiteInventoryReader) throws -> Result) throws -> Result { + try SQLiteSnapshotQueryEngine(source: database).withSnapshot { snapshot, workspace in + try operation(LocalSQLiteInventoryReader(database: snapshot, snapshotsLiveStores: false, reportedStore: reportedStore, snapshotWorkspace: workspace)) + } + } + private func contactColumnReference(_ candidates: [String]) -> String { for column in candidates { if (try? columnExists("ZABCDRECORD", column)) == true { @@ -796,6 +837,9 @@ func sqliteError(from errorData: Data, store: String) -> LocalInventoryError { if lowercased.contains("no such table") || lowercased.contains("no such column") { return .unsupportedSchema(store: store, detail: message) } + if lowercased.contains("database is locked") || lowercased.contains("database is busy") { + return .lockedStore(store) + } if lowercased.contains("file is not a database") || lowercased.contains("file is not in a database") { return .unsupportedSchema(store: store, detail: "not a SQLite database") } diff --git a/Sources/ICloudCLICore/ProviderManifest.swift b/Sources/ICloudCLICore/ProviderManifest.swift index 6cb28b3..d31d2dd 100644 --- a/Sources/ICloudCLICore/ProviderManifest.swift +++ b/Sources/ICloudCLICore/ProviderManifest.swift @@ -48,13 +48,13 @@ public enum ProviderRegistry { provider("home", "Home", .beta, .sqlite, .high, ["home accessories", "home homes", "home rooms", "home scenes"], ["accessories", "homes", "rooms", "scenes"]), provider("mail", "Mail", .beta, .sqlite, .high, ["mail accounts", "mail mailboxes", "mail recent"], ["accounts", "headers", "mailboxes"]), provider("maps", "Maps", .beta, .sqlite, .high, ["maps favorites", "maps recents"], ["favorites", "recents", "location"]), - provider("messages", "Messages", .beta, .sqlite, .high, ["messages conversations", "messages recent"], ["conversations", "messages", "date-filtering"]), + provider("messages", "Messages", .beta, .sqlite, .high, ["messages conversations", "messages recent"], ["consistent-snapshot", "conversations", "messages", "date-filtering"]), provider("music", "Music", .beta, .sqlite, .moderate, ["music playlists", "music status", "music tracks"], ["inventory", "playlists", "status"]), provider("news", "News", .beta, .sqlite, .moderate, ["news history", "news topics"], ["history", "topics", "date-filtering"]), provider("notes", "Notes", .beta, .sqlite, .high, ["notes accounts", "notes folders", "notes list", "notes shared", "notes tags"], ["accounts", "folders", "inventory", "sharing", "tags"]), provider("photos", "Photos", .beta, .mixed, .high, ["photos list", "photos screenshots", "photos shared-albums", "photos shared-library"], ["assets", "screenshots", "sharing"]), provider("reminders", "Reminders", .beta, .sqlite, .high, ["reminders assigned", "reminders flagged", "reminders list", "reminders lists", "reminders scheduled", "reminders today"], ["inventory", "lists", "smart-views"]), - provider("safari", "Safari", .beta, .mixed, .high, ["safari bookmarks", "safari cloud-tabs list", "safari cloud-tabs probe", "safari extensions list", "safari frequently-visited", "safari history", "safari profiles list", "safari reading-list", "safari tabs"], ["bookmarks", "cloud-tabs", "extensions", "history", "profiles", "tabs"]), + provider("safari", "Safari", .beta, .mixed, .high, ["safari bookmarks", "safari cloud-tabs list", "safari cloud-tabs probe", "safari extensions list", "safari frequently-visited", "safari history", "safari profiles list", "safari reading-list", "safari tabs"], ["bookmarks", "cloud-tabs", "consistent-snapshot", "extensions", "history", "profiles", "tabs"]), provider("shortcuts", "Shortcuts", .stable, .filesystem, .moderate, ["shortcuts list"], ["inventory", "search"], true), provider("stocks", "Stocks", .beta, .sqlite, .moderate, ["stocks groups", "stocks watchlist"], ["groups", "watchlist"]), provider("storage", "iCloud Storage", .stable, .preferences, .low, ["storage status"], ["quota", "status"], true), diff --git a/Sources/ICloudCLICore/SQLiteSnapshotQuery.swift b/Sources/ICloudCLICore/SQLiteSnapshotQuery.swift new file mode 100644 index 0000000..966cf92 --- /dev/null +++ b/Sources/ICloudCLICore/SQLiteSnapshotQuery.swift @@ -0,0 +1,132 @@ +import Darwin +import Foundation + +public struct SQLiteSnapshotQueryEngine: Sendable { + public let source: URL + public let timeout: TimeInterval + public let busyTimeoutMilliseconds: Int + public let temporaryRoot: URL + private let reportedStore: String + + public init( + source: URL, + timeout: TimeInterval = 10, + busyTimeoutMilliseconds: Int = 1_000, + temporaryRoot: URL = FileManager.default.temporaryDirectory, + reportedStore: String? = nil + ) { + self.source = source + self.timeout = max(0.001, timeout) + self.busyTimeoutMilliseconds = max(0, busyTimeoutMilliseconds) + self.temporaryRoot = temporaryRoot + self.reportedStore = reportedStore ?? source.path + } + + public func query(_ sql: String) throws -> [T] { + try withSnapshot { snapshot, directory in + try querySnapshot(snapshot, workspace: directory, sql: sql) + } + } + + func querySnapshot(_ snapshot: URL, workspace: URL, sql: String) throws -> [T] { + let identifier = UUID().uuidString + let output = workspace.appendingPathComponent("query-\(identifier).json") + let errors = workspace.appendingPathComponent("query-\(identifier).err") + FileManager.default.createFile(atPath: output.path, contents: nil, attributes: [.posixPermissions: 0o600]) + FileManager.default.createFile(atPath: errors.path, contents: nil, attributes: [.posixPermissions: 0o600]) + defer { + try? FileManager.default.removeItem(at: output) + try? FileManager.default.removeItem(at: errors) + } + let outputHandle = try FileHandle(forWritingTo: output) + let errorHandle = try FileHandle(forWritingTo: errors) + defer { + try? outputHandle.close() + try? errorHandle.close() + } + + let process = Process() + process.executableURL = URL(fileURLWithPath: "/usr/bin/sqlite3") + process.arguments = [ + "-readonly", "-json", + "-cmd", ".timeout \(busyTimeoutMilliseconds)", + "-cmd", "PRAGMA query_only=ON;", + snapshot.path, + sql, + ] + process.standardOutput = outputHandle + process.standardError = errorHandle + + let completed = DispatchSemaphore(value: 0) + process.terminationHandler = { _ in completed.signal() } + do { + try process.run() + } catch { + throw LocalInventoryError.sqliteFailure(error.localizedDescription) + } + guard completed.wait(timeout: .now() + timeout) == .success else { + process.terminate() + if completed.wait(timeout: .now() + 1) != .success { + Darwin.kill(process.processIdentifier, SIGKILL) + _ = completed.wait(timeout: .now() + 1) + } + throw LocalInventoryError.queryTimeout(reportedStore) + } + + outputHandle.synchronizeFile() + errorHandle.synchronizeFile() + let data = try Data(contentsOf: output) + let errorData = try Data(contentsOf: errors) + guard process.terminationStatus == 0 else { + throw sqliteError(from: errorData, store: reportedStore) + } + if data.isEmpty { return [] } + do { + return try JSONDecoder().decode([T].self, from: data) + } catch { + throw LocalInventoryError.sqliteFailure("SQLite returned an invalid JSON result") + } + } + + func withSnapshot(_ operation: (URL, URL) throws -> Result) throws -> Result { + guard FileManager.default.fileExists(atPath: source.path) else { + throw LocalInventoryError.missingStore(reportedStore) + } + + let directory = temporaryRoot.appendingPathComponent("icloud-cli-sqlite-\(UUID().uuidString)", isDirectory: true) + do { + try FileManager.default.createDirectory( + at: directory, + withIntermediateDirectories: true, + attributes: [.posixPermissions: 0o700] + ) + } catch { + throw LocalInventoryError.sqliteFailure("Unable to create private SQLite snapshot directory") + } + defer { try? FileManager.default.removeItem(at: directory) } + + let snapshot = directory.appendingPathComponent("snapshot.sqlite") + do { + try copy(source, to: snapshot) + for suffix in ["-wal", "-shm"] { + let companion = URL(fileURLWithPath: source.path + suffix) + guard FileManager.default.fileExists(atPath: companion.path) else { continue } + try copy(companion, to: URL(fileURLWithPath: snapshot.path + suffix)) + } + } catch { + if isPermissionError(error) { throw LocalInventoryError.permissionDenied(reportedStore) } + throw LocalInventoryError.sqliteFailure("Unable to create SQLite snapshot") + } + return try operation(snapshot, directory) + } + + private func copy(_ source: URL, to destination: URL) throws { + try FileManager.default.copyItem(at: source, to: destination) + try FileManager.default.setAttributes([.posixPermissions: 0o600], ofItemAtPath: destination.path) + } + + private func isPermissionError(_ error: Error) -> Bool { + let cocoa = error as NSError + return cocoa.domain == NSCocoaErrorDomain && [NSFileReadNoPermissionError, NSFileWriteNoPermissionError].contains(cocoa.code) + } +} diff --git a/Tests/ICloudCLICoreTests/ProviderManifestTests.swift b/Tests/ICloudCLICoreTests/ProviderManifestTests.swift index 74f6665..f480ded 100644 --- a/Tests/ICloudCLICoreTests/ProviderManifestTests.swift +++ b/Tests/ICloudCLICoreTests/ProviderManifestTests.swift @@ -21,6 +21,8 @@ import Testing ]) #expect(manifest.providers.allSatisfy { !$0.commands.isEmpty && !$0.capabilities.isEmpty }) #expect(manifest.providers.allSatisfy { $0.accessMode == .readOnly }) + #expect(manifest.providers.first { $0.id == "messages" }?.capabilities.contains("consistent-snapshot") == true) + #expect(manifest.providers.first { $0.id == "safari" }?.capabilities.contains("consistent-snapshot") == true) } @Test func providerManifestJSONContainsMetadataOnly() throws { diff --git a/Tests/ICloudCLICoreTests/SQLiteSnapshotQueryTests.swift b/Tests/ICloudCLICoreTests/SQLiteSnapshotQueryTests.swift new file mode 100644 index 0000000..a91284e --- /dev/null +++ b/Tests/ICloudCLICoreTests/SQLiteSnapshotQueryTests.swift @@ -0,0 +1,179 @@ +import Foundation +import Testing +@testable import ICloudCLICore + +private struct SnapshotValueRow: Decodable, Equatable { let value: String } + +@Test func snapshotQueryReadsDatabaseWithoutCompanionFilesAndCleansUp() throws { + let root = try temporarySQLiteSnapshotDirectory(named: "no-companions") + defer { try? FileManager.default.removeItem(at: root) } + let database = root.appendingPathComponent("source.db") + try runSnapshotFixtureSQL(database: database, sql: "CREATE TABLE values_table (value TEXT); INSERT INTO values_table VALUES ('one');") + let snapshots = root.appendingPathComponent("snapshots") + try FileManager.default.createDirectory(at: snapshots, withIntermediateDirectories: true) + + let rows: [SnapshotValueRow] = try SQLiteSnapshotQueryEngine(source: database, temporaryRoot: snapshots) + .query("SELECT value FROM values_table;") + + #expect(rows == [SnapshotValueRow(value: "one")]) + #expect(try FileManager.default.contentsOfDirectory(atPath: snapshots.path).isEmpty) +} + +@Test func snapshotQueryIncludesCommittedWALRows() throws { + let root = try temporarySQLiteSnapshotDirectory(named: "wal") + defer { try? FileManager.default.removeItem(at: root) } + let database = root.appendingPathComponent("source.db") + let writer = try openWALFixture(database: database) + defer { writer.stop() } + + #expect(FileManager.default.fileExists(atPath: database.path + "-wal")) + let rows: [SnapshotValueRow] = try SQLiteSnapshotQueryEngine(source: database) + .query("SELECT value FROM values_table ORDER BY value;") + #expect(rows == [SnapshotValueRow(value: "wal-row")]) +} + +@Test func snapshotQueryMapsSchemaDriftAndBusyFailures() { + let schema = sqliteError(from: Data("Error: no such table: missing".utf8), store: "/private/source.db") + #expect(schema == .unsupportedSchema(store: "/private/source.db", detail: "Error: no such table: missing")) + + let locked = sqliteError(from: Data("Error: database is locked".utf8), store: "/private/source.db") + #expect(locked == .lockedStore("/private/source.db")) +} + +@Test func snapshotQueryRejectsWritesAndLeavesSourceUnchanged() throws { + let root = try temporarySQLiteSnapshotDirectory(named: "query-only") + defer { try? FileManager.default.removeItem(at: root) } + let database = root.appendingPathComponent("source.db") + try runSnapshotFixtureSQL(database: database, sql: "CREATE TABLE values_table (value TEXT); INSERT INTO values_table VALUES ('one');") + let engine = SQLiteSnapshotQueryEngine(source: database) + + #expect(throws: LocalInventoryError.self) { + let _: [SnapshotValueRow] = try engine.query("INSERT INTO values_table VALUES ('two') RETURNING value;") + } + #expect(try snapshotFixtureScalar(database: database, sql: "SELECT COUNT(*) FROM values_table;") == "1") +} + +@Test func snapshotQueryReportsLockedStore() throws { + let root = try temporarySQLiteSnapshotDirectory(named: "locked") + defer { try? FileManager.default.removeItem(at: root) } + let database = root.appendingPathComponent("source.db") + try runSnapshotFixtureSQL(database: database, sql: "CREATE TABLE values_table (value TEXT); INSERT INTO values_table VALUES ('one');") + let engine = SQLiteSnapshotQueryEngine(source: database, busyTimeoutMilliseconds: 20) + + #expect(throws: LocalInventoryError.lockedStore(database.path)) { + try engine.withSnapshot { snapshot, workspace in + let locker = try lockSnapshotDatabase(snapshot) + defer { locker.stop() } + Thread.sleep(forTimeInterval: 0.05) + let _: [SnapshotValueRow] = try engine.querySnapshot(snapshot, workspace: workspace, sql: "SELECT value FROM values_table;") + } + } +} + +@Test func snapshotQueryTimesOutAndCleansUp() throws { + let root = try temporarySQLiteSnapshotDirectory(named: "timeout") + defer { try? FileManager.default.removeItem(at: root) } + let database = root.appendingPathComponent("source.db") + try runSnapshotFixtureSQL(database: database, sql: "CREATE TABLE values_table (value TEXT);") + let snapshots = root.appendingPathComponent("snapshots") + try FileManager.default.createDirectory(at: snapshots, withIntermediateDirectories: true) + let engine = SQLiteSnapshotQueryEngine(source: database, timeout: 0.01, temporaryRoot: snapshots) + + #expect(throws: LocalInventoryError.queryTimeout(database.path)) { + let _: [SnapshotValueRow] = try engine.query("WITH RECURSIVE loop(x) AS (VALUES(0) UNION ALL SELECT x + 1 FROM loop LIMIT 100000000) SELECT CAST(MAX(x) AS TEXT) AS value FROM loop;") + } + #expect(try FileManager.default.contentsOfDirectory(atPath: snapshots.path).isEmpty) +} + +@Test func snapshotQueryMapsCopyPermissionFailures() throws { + let root = try temporarySQLiteSnapshotDirectory(named: "permission") + defer { try? FileManager.default.removeItem(at: root) } + let database = root.appendingPathComponent("source.db") + try runSnapshotFixtureSQL(database: database, sql: "CREATE TABLE values_table (value TEXT);") + try FileManager.default.setAttributes([.posixPermissions: 0], ofItemAtPath: database.path) + defer { try? FileManager.default.setAttributes([.posixPermissions: 0o600], ofItemAtPath: database.path) } + + #expect(throws: LocalInventoryError.permissionDenied(database.path)) { + let _: [SnapshotValueRow] = try SQLiteSnapshotQueryEngine(source: database).query("SELECT value FROM values_table;") + } +} + +private final class WALFixtureWriter { + let process: Process + let input: FileHandle + + init(process: Process, input: FileHandle) { self.process = process; self.input = input } + + func stop() { + try? input.write(contentsOf: Data(".quit\n".utf8)) + try? input.close() + process.waitUntilExit() + } +} + +private func openWALFixture(database: URL) throws -> WALFixtureWriter { + let process = Process() + let input = Pipe() + process.executableURL = URL(fileURLWithPath: "/usr/bin/sqlite3") + process.arguments = [database.path] + process.standardInput = input + process.standardOutput = FileHandle.nullDevice + process.standardError = FileHandle.nullDevice + try process.run() + let commands = "PRAGMA journal_mode=WAL; PRAGMA wal_autocheckpoint=0; CREATE TABLE values_table (value TEXT); INSERT INTO values_table VALUES ('wal-row');\n" + try input.fileHandleForWriting.write(contentsOf: Data(commands.utf8)) + + let deadline = Date().addingTimeInterval(2) + while Date() < deadline { + if FileManager.default.fileExists(atPath: database.path + "-wal"), + (try? snapshotFixtureScalar(database: database, sql: "SELECT value FROM values_table;")) == "wal-row" { + break + } + Thread.sleep(forTimeInterval: 0.01) + } + return WALFixtureWriter(process: process, input: input.fileHandleForWriting) +} + +private func lockSnapshotDatabase(_ database: URL) throws -> WALFixtureWriter { + let process = Process() + let input = Pipe() + process.executableURL = URL(fileURLWithPath: "/usr/bin/sqlite3") + process.arguments = [database.path] + process.standardInput = input + process.standardOutput = FileHandle.nullDevice + process.standardError = FileHandle.nullDevice + try process.run() + try input.fileHandleForWriting.write(contentsOf: Data("PRAGMA locking_mode=EXCLUSIVE; BEGIN EXCLUSIVE; SELECT 1;\n".utf8)) + return WALFixtureWriter(process: process, input: input.fileHandleForWriting) +} + +private func snapshotFixtureScalar(database: URL, sql: String) throws -> String { + let process = Process() + let output = Pipe() + process.executableURL = URL(fileURLWithPath: "/usr/bin/sqlite3") + process.arguments = ["-readonly", database.path, sql] + process.standardOutput = output + process.standardError = FileHandle.nullDevice + try process.run() + process.waitUntilExit() + guard process.terminationStatus == 0 else { throw CocoaError(.fileReadUnknown) } + return String(decoding: output.fileHandleForReading.readDataToEndOfFile(), as: UTF8.self) + .trimmingCharacters(in: .whitespacesAndNewlines) +} + +private func temporarySQLiteSnapshotDirectory(named name: String) throws -> URL { + let url = FileManager.default.temporaryDirectory.appendingPathComponent("icloud-cli-snapshot-\(name)-\(UUID().uuidString)") + try FileManager.default.createDirectory(at: url, withIntermediateDirectories: true) + return url +} + +private func runSnapshotFixtureSQL(database: URL, sql: String) throws { + let process = Process() + process.executableURL = URL(fileURLWithPath: "/usr/bin/sqlite3") + process.arguments = [database.path, sql] + process.standardOutput = FileHandle.nullDevice + process.standardError = FileHandle.nullDevice + try process.run() + process.waitUntilExit() + guard process.terminationStatus == 0 else { throw CocoaError(.fileWriteUnknown) } +} diff --git a/docs/privacy.md b/docs/privacy.md index a98a721..f2c6dac 100644 --- a/docs/privacy.md +++ b/docs/privacy.md @@ -116,6 +116,8 @@ The second inventory wave adds local-only command trees for `account status`, `b These commands use preference/plist readers for account, backup, Family Sharing, permissions, and snapshot status, plus best-effort SQLite metadata readers for Apple private cache stores when a stable local table shape is known or supplied by fixtures. All adapters are read-only and local-only. They do not refresh iCloud, contact Apple services, control apps, trigger HomeKit accessories, execute shortcuts, read audio/photo/book/media payloads, or emit auth tokens. Private Apple schemas can vary across macOS releases, so commands fail closed when expected local tables are absent. +Messages and Safari history queries first copy the database and present WAL/SHM companions into a private temporary directory, query that copy in read-only/query-only mode with bounded busy and process timeouts, and delete it deterministically. See [sqlite-snapshots.md](sqlite-snapshots.md) for the migration and cleanup contract. + High-sensitivity gates: - `icloud-cli safari cloud-tabs list`, `icloud-cli mail recent`, and `icloud-cli health summary` require `--confirm-sensitive`. diff --git a/docs/sqlite-snapshots.md b/docs/sqlite-snapshots.md new file mode 100644 index 0000000..7eb947a --- /dev/null +++ b/docs/sqlite-snapshots.md @@ -0,0 +1,19 @@ +# Safe SQLite snapshots + +Apple applications often keep committed SQLite state across a database file and its write-ahead log. Querying the live path directly can produce inconsistent reads, wait indefinitely on a busy store, or interact with an Apple-owned database in ways the CLI does not intend. + +`SQLiteSnapshotQueryEngine` creates a mode-`0700` temporary directory, copies the database and any present `-wal` and `-shm` companions as mode-`0600` files, and queries only that private copy. The `sqlite3` process runs with `-readonly`, `PRAGMA query_only=ON`, a one-second busy timeout, and a ten-second process timeout. The snapshot, query output, and error output are deleted before the call returns or throws. Copied contents are never logged. + +Errors retain the original store path and distinguish missing stores, permission denial, unsupported schemas, locked or busy stores, and timeouts. The implementation does not checkpoint, lock, vacuum, or write to the Apple-owned source. + +## Migration status + +Messages conversations/recent messages and Safari history use the snapshot engine first because they read high-sensitivity, frequently changing databases. The remaining SQLite-backed providers still use the older read-only query path and should migrate incrementally with their provider-specific work: + +- Notes and Reminders +- Calendar, Contacts, Mail, and Maps +- Photos sharing and Find My metadata +- Books, Voice Memos, Home, Health, Freeform, Music, Stocks, Weather, and News +- Safari Cloud Tabs, profiles, and extensions where their selected store is SQLite + +Provider migrations should retain existing confirmation and redaction gates and add WAL-backed fixtures before switching paths. From 74f29b5880adc3c975bac598c9e790c214848055 Mon Sep 17 00:00:00 2001 From: Hermes Date: Sun, 12 Jul 2026 11:34:02 +0100 Subject: [PATCH 2/4] Bound recursive crawls and preserve partial results Add explicit scan and wall-clock budgets, structured crawl reports, cancellable Drive enumeration, Finder-tag budget metadata, and redacted watch partial-success envelopes. Extend the live audit with explicit budgets and aggregate outcomes.\n\nCloses #92. --- README.md | 2 +- .../AppleMetadataInventories.swift | 26 ++- Sources/ICloudCLICore/CacheWatch.swift | 61 ++++- Sources/ICloudCLICore/CommandLine.swift | 45 +++- Sources/ICloudCLICore/CommandRunner.swift | 17 +- Sources/ICloudCLICore/CrawlBudget.swift | 71 ++++++ Sources/ICloudCLICore/DriveInventory.swift | 213 +++++++++++++++--- Sources/ICloudCLICore/ProviderManifest.swift | 4 +- .../CommandRunnerHarnessTests.swift | 8 +- .../ICloudCLICoreTests/CrawlBudgetTests.swift | 128 +++++++++++ .../ProviderManifestTests.swift | 2 + docs/crawl-budgets.md | 18 ++ docs/privacy.md | 2 + scripts/live-command-audit.py | 37 ++- 14 files changed, 555 insertions(+), 79 deletions(-) create mode 100644 Sources/ICloudCLICore/CrawlBudget.swift create mode 100644 Tests/ICloudCLICoreTests/CrawlBudgetTests.swift create mode 100644 docs/crawl-budgets.md diff --git a/README.md b/README.md index 89833c0..9bf3cfc 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ The implementation reads local Safari session and metadata property lists from ` If Safari session files are unreadable, the command exits with an error naming the file paths it tried. If the files are readable but empty, the error says no tabs were found instead of treating it as a permissions problem. -Use `icloud-cli safari cloud-tabs probe` to check whether Safari's cross-device tab store is present and readable before using `icloud-cli safari cloud-tabs list --confirm-sensitive`. Use `icloud-cli permissions doctor` when a command reports missing or unreadable local stores; it probes source paths without reading payload content and gives the Full Disk Access hint per command. `icloud-cli providers list` exposes a static, versioned capability registry without reading live data; see [docs/provider-manifest.md](docs/provider-manifest.md) for its schema and compatibility policy. See [docs/cloudtabs.md](docs/cloudtabs.md) for the investigation notes, [docs/privacy.md](docs/privacy.md) for the privacy and permissions model, and [docs/openclaw-skill-contract.md](docs/openclaw-skill-contract.md) for the OpenClaw integration contract. +Use `icloud-cli safari cloud-tabs probe` to check whether Safari's cross-device tab store is present and readable before using `icloud-cli safari cloud-tabs list --confirm-sensitive`. Use `icloud-cli permissions doctor` when a command reports missing or unreadable local stores; it probes source paths without reading payload content and gives the Full Disk Access hint per command. `icloud-cli providers list` exposes a static, versioned capability registry without reading live data; see [docs/provider-manifest.md](docs/provider-manifest.md) for its schema and compatibility policy. Recursive Drive and Finder-tag commands emit bounded crawl reports documented in [docs/crawl-budgets.md](docs/crawl-budgets.md). See [docs/cloudtabs.md](docs/cloudtabs.md) for the investigation notes, [docs/privacy.md](docs/privacy.md) for the privacy and permissions model, and [docs/openclaw-skill-contract.md](docs/openclaw-skill-contract.md) for the OpenClaw integration contract. ## Build diff --git a/Sources/ICloudCLICore/AppleMetadataInventories.swift b/Sources/ICloudCLICore/AppleMetadataInventories.swift index 323d142..28b8b8e 100644 --- a/Sources/ICloudCLICore/AppleMetadataInventories.swift +++ b/Sources/ICloudCLICore/AppleMetadataInventories.swift @@ -1173,11 +1173,29 @@ public struct FinderTagsReader: Sendable { public func items(tag: String, path: String?, limit: Int) throws -> [TaggedDriveItem] { let scanLimit = max(200, bounded(limit, defaultValue: 50, max: 1_000) * 5) - let files = try ICloudDriveInventoryReader(rootDirectory: driveRoot).listFiles(path: path, depth: Int.max, limit: scanLimit) - return files - .filter { $0.name.localizedCaseInsensitiveContains(tag) || $0.path.localizedCaseInsensitiveContains(".\(tag).") } - .prefix(max(1, limit)) + let budget = CrawlBudget(scanLimit: scanLimit, wallClockLimitMilliseconds: CrawlBudget.defaultDrive.wallClockLimitMilliseconds) + return try itemsReport(tag: tag, path: path, limit: limit, budget: budget).data + } + + public func itemsReport(tag: String, path: String?, limit: Int, budget: CrawlBudget = .defaultDrive) throws -> CrawlReport<[TaggedDriveItem]> { + let crawl = try ICloudDriveInventoryReader(rootDirectory: driveRoot).listFilesReport(path: path, depth: Int.max, budget: budget) + let resultLimit = max(1, limit) + let matches = crawl.data.filter { $0.name.localizedCaseInsensitiveContains(tag) || $0.path.localizedCaseInsensitiveContains(".\(tag).") } + let items = matches + .prefix(resultLimit) .map { TaggedDriveItem(path: $0.path, modifiedAt: $0.modifiedAt, iCloudStatus: $0.iCloudStatus) } + return CrawlReport( + providerId: "tags", + state: crawl.state, + data: items, + scannedCount: crawl.scannedCount, + resultCount: items.count, + totalAvailable: crawl.state == .complete ? matches.count : nil, + budget: budget, + resultLimit: resultLimit, + elapsedMilliseconds: crawl.elapsedMilliseconds, + nextAction: crawl.nextAction + ) } } diff --git a/Sources/ICloudCLICore/CacheWatch.swift b/Sources/ICloudCLICore/CacheWatch.swift index d0b38e6..c9f5890 100644 --- a/Sources/ICloudCLICore/CacheWatch.swift +++ b/Sources/ICloudCLICore/CacheWatch.swift @@ -5,6 +5,13 @@ public struct CacheEnvelope: Codable, Equatable, Sendable { public let ok: Bool public let error: String? public let data: String? + public let failure: CacheFailure? +} + +public struct CacheFailure: Codable, Equatable, Sendable { + public let code: String + public let providerId: String + public let guidance: String } public struct CacheStatusEntry: Codable, Equatable, Sendable { @@ -13,6 +20,7 @@ public struct CacheStatusEntry: Codable, Equatable, Sendable { public let updatedAt: String? public let ok: Bool? public let error: String? + public let failure: CacheFailure? } public enum CacheWatchError: Error, LocalizedError, Equatable { @@ -31,25 +39,33 @@ public struct CacheWatchStore: Sendable { public static let defaultCommands = ["safari-tabs", "drive-list", "photos-screenshots", "storage-status"] public let outputDirectory: URL + private let snapshotter: CacheSnapshotter public init(outputDirectory: URL = FileManager.default.homeDirectoryForCurrentUser.appendingPathComponent(".icloud-cli/cache")) { self.outputDirectory = outputDirectory + self.snapshotter = Self.defaultSnapshot + } + + init(outputDirectory: URL, snapshotter: @escaping CacheSnapshotter) { + self.outputDirectory = outputDirectory + self.snapshotter = snapshotter } - public func refresh(commands: [String]) throws -> [CacheStatusEntry] { + public func refresh(commands: [String], budget: CrawlBudget = .defaultPolling) throws -> [CacheStatusEntry] { try ensureCacheDirectory() var statuses: [CacheStatusEntry] = [] for command in commands { let envelope: CacheEnvelope do { - let data = try snapshot(command: command) - envelope = CacheEnvelope(updatedAt: now(), ok: true, error: nil, data: data) + let data = try snapshotter(command, budget) + envelope = CacheEnvelope(updatedAt: now(), ok: true, error: nil, data: data, failure: nil) } catch { let previous = try? readEnvelope(command: command) - envelope = CacheEnvelope(updatedAt: now(), ok: false, error: error.localizedDescription, data: previous?.data) + let failure = redactedFailure(error, command: command) + envelope = CacheEnvelope(updatedAt: now(), ok: false, error: failure.message, data: previous?.data, failure: failure.detail) } try write(envelope, command: command) - statuses.append(CacheStatusEntry(command: command, path: fileURL(for: command).path, updatedAt: envelope.updatedAt, ok: envelope.ok, error: envelope.error)) + statuses.append(CacheStatusEntry(command: command, path: fileURL(for: command).path, updatedAt: envelope.updatedAt, ok: envelope.ok, error: envelope.error, failure: envelope.failure)) } return statuses } @@ -64,7 +80,7 @@ public struct CacheWatchStore: Sendable { return files.filter { $0.pathExtension == "json" }.sorted { $0.lastPathComponent < $1.lastPathComponent }.map { url in let command = url.deletingPathExtension().lastPathComponent let envelope = try? readEnvelope(command: command) - return CacheStatusEntry(command: command, path: url.path, updatedAt: envelope?.updatedAt, ok: envelope?.ok, error: envelope?.error) + return CacheStatusEntry(command: command, path: url.path, updatedAt: envelope?.updatedAt, ok: envelope?.ok, error: envelope?.error, failure: envelope?.failure) } } @@ -75,19 +91,38 @@ public struct CacheWatchStore: Sendable { try FileManager.default.setAttributes([.posixPermissions: 0o700], ofItemAtPath: outputDirectory.path) } - private func snapshot(command: String) throws -> String { + private static func defaultSnapshot(command: String, budget: CrawlBudget) throws -> String { + let startedAt = ProcessInfo.processInfo.systemUptime + let data: String switch command { case "safari-tabs": - return try json(SafariTabsReader().readTabs()) + data = try json(SafariTabsReader().readTabs()) case "drive-list": - return try json(ICloudDriveInventoryReader().listFiles()) + let report = try ICloudDriveInventoryReader().listFilesReport(budget: budget) + guard report.state == .complete else { throw CrawlBudgetExceeded(providerId: "drive", state: report.state) } + data = try json(report) case "photos-screenshots": - return try json(PhotosInventoryReader().listScreenshots()) + data = try json(PhotosInventoryReader().listScreenshots()) case "storage-status": - return try json(ICloudStorageStatusReader().readStatus()) + data = try json(ICloudStorageStatusReader().readStatus()) default: throw CacheWatchError.missingCommand(command) } + let elapsed = Int((ProcessInfo.processInfo.systemUptime - startedAt) * 1_000) + guard elapsed < budget.wallClockLimitMilliseconds else { + throw CrawlBudgetExceeded(providerId: command.split(separator: "-").first.map(String.init) ?? "unknown", state: .timeout) + } + return data + } + + private func redactedFailure(_ error: Error, command: String) -> (message: String, detail: CacheFailure) { + let providerId = command.split(separator: "-").first.map(String.init) ?? "unknown" + if let exceeded = error as? CrawlBudgetExceeded { + let code = exceeded.state == .timeout ? "timeout" : "partial" + let message = exceeded.state == .timeout ? "provider crawl timed out" : "provider crawl reached its scan budget" + return (message, CacheFailure(code: code, providerId: exceeded.providerId, guidance: "Narrow the provider scope or increase its explicit crawl budget.")) + } + return ("provider crawl failed", CacheFailure(code: "provider-error", providerId: providerId, guidance: "Run the provider directly for local diagnostic details.")) } private func write(_ envelope: CacheEnvelope, command: String) throws { @@ -111,7 +146,7 @@ public struct CacheWatchStore: Sendable { outputDirectory.appendingPathComponent(command).appendingPathExtension("json") } - private func json(_ value: T) throws -> String { + private static func json(_ value: T) throws -> String { String(decoding: try JSONEncoder.pretty.encode(value), as: UTF8.self) } @@ -120,6 +155,8 @@ public struct CacheWatchStore: Sendable { } } +typealias CacheSnapshotter = @Sendable (String, CrawlBudget) throws -> String + private extension JSONEncoder { static var pretty: JSONEncoder { let encoder = JSONEncoder() diff --git a/Sources/ICloudCLICore/CommandLine.swift b/Sources/ICloudCLICore/CommandLine.swift index 0528126..8daa7ee 100644 --- a/Sources/ICloudCLICore/CommandLine.swift +++ b/Sources/ICloudCLICore/CommandLine.swift @@ -51,13 +51,15 @@ public struct DriveListOptions: Equatable, Sendable { public var path: String? public var depth: Int public var showStatus: Bool + public var budget: CrawlBudget - public init(format: OutputFormat = .json, rootDirectory: URL = FileManager.default.homeDirectoryForCurrentUser.appendingPathComponent("Library/Mobile Documents"), path: String? = nil, depth: Int = 2, showStatus: Bool = false) { + public init(format: OutputFormat = .json, rootDirectory: URL = FileManager.default.homeDirectoryForCurrentUser.appendingPathComponent("Library/Mobile Documents"), path: String? = nil, depth: Int = 2, showStatus: Bool = false, budget: CrawlBudget = .defaultDrive) { self.format = format self.rootDirectory = rootDirectory self.path = path self.depth = depth self.showStatus = showStatus + self.budget = budget } } @@ -294,12 +296,14 @@ public struct WatchOptions: Equatable, Sendable { public var outputDirectory: URL public var commands: [String] public var once: Bool + public var budget: CrawlBudget - public init(intervalSeconds: Int = 60, outputDirectory: URL = FileManager.default.homeDirectoryForCurrentUser.appendingPathComponent(".icloud-cli/cache"), commands: [String] = CacheWatchStore.defaultCommands, once: Bool = false) { + public init(intervalSeconds: Int = 60, outputDirectory: URL = FileManager.default.homeDirectoryForCurrentUser.appendingPathComponent(".icloud-cli/cache"), commands: [String] = CacheWatchStore.defaultCommands, once: Bool = false, budget: CrawlBudget = .defaultPolling) { self.intervalSeconds = intervalSeconds self.outputDirectory = outputDirectory self.commands = commands self.once = once + self.budget = budget } } @@ -445,6 +449,7 @@ public struct MetadataOptions: Equatable, Sendable { public var raw: Bool public var downloadedOnly: Bool public var cloudOnly: Bool + public var crawlBudget: CrawlBudget public init( format: OutputFormat = .json, @@ -477,7 +482,8 @@ public struct MetadataOptions: Equatable, Sendable { includeURLs: Bool = false, raw: Bool = false, downloadedOnly: Bool = false, - cloudOnly: Bool = false + cloudOnly: Bool = false, + crawlBudget: CrawlBudget = .defaultDrive ) { self.format = format self.store = store @@ -510,6 +516,7 @@ public struct MetadataOptions: Equatable, Sendable { self.raw = raw self.downloadedOnly = downloadedOnly self.cloudOnly = cloudOnly + self.crawlBudget = crawlBudget } } @@ -930,6 +937,10 @@ public struct CLIParser: Sendable { case "--icloud-root": options.rootDirectory = try parseURL(after: token, in: tokens, at: &index) case "--path": options.path = try value(after: token, in: tokens, at: &index) case "--show-status": options.showStatus = true + case "--scan-limit": + options.budget = CrawlBudget(scanLimit: try positiveInteger(after: token, in: tokens, at: &index), wallClockLimitMilliseconds: options.budget.wallClockLimitMilliseconds) + case "--timeout-ms": + options.budget = CrawlBudget(scanLimit: options.budget.scanLimit, wallClockLimitMilliseconds: try positiveInteger(after: token, in: tokens, at: &index)) case "--depth": let rawValue = try value(after: token, in: tokens, at: &index) guard let depth = Int(rawValue), depth >= 0 else { throw CLIParseError.missingValue(token) } @@ -1232,6 +1243,10 @@ public struct CLIParser: Sendable { .map { String($0).trimmingCharacters(in: .whitespacesAndNewlines) } .filter { !$0.isEmpty } case "--once": options.once = true + case "--scan-limit": + options.budget = CrawlBudget(scanLimit: try positiveInteger(after: token, in: tokens, at: &index), wallClockLimitMilliseconds: options.budget.wallClockLimitMilliseconds) + case "--timeout-ms": + options.budget = CrawlBudget(scanLimit: options.budget.scanLimit, wallClockLimitMilliseconds: try positiveInteger(after: token, in: tokens, at: &index)) default: throw CLIParseError.unknownCommand(token) } index += 1 @@ -1302,6 +1317,10 @@ public struct CLIParser: Sendable { let limit = Int(try value(after: token, in: tokens, at: &index)) ?? options.limit options.limit = limit options.driveStatusLimit = limit + case "--scan-limit": + options.crawlBudget = CrawlBudget(scanLimit: try positiveInteger(after: token, in: tokens, at: &index), wallClockLimitMilliseconds: options.crawlBudget.wallClockLimitMilliseconds) + case "--timeout-ms": + options.crawlBudget = CrawlBudget(scanLimit: options.crawlBudget.scanLimit, wallClockLimitMilliseconds: try positiveInteger(after: token, in: tokens, at: &index)) case "--confirm-sensitive": options.confirmSensitive = true case "--include-attendees": options.includeAttendees = true case "--include-coordinates": options.includeCoordinates = true @@ -1324,6 +1343,12 @@ public struct CLIParser: Sendable { return format } + private func positiveInteger(after option: String, in tokens: [String], at index: inout Int) throws -> Int { + let rawValue = try value(after: option, in: tokens, at: &index) + guard let value = Int(rawValue), value > 0 else { throw CLIParseError.missingValue(option) } + return value + } + private func parseURL(after option: String, in tokens: [String], at index: inout Int) throws -> URL { URL(fileURLWithPath: NSString(string: try value(after: option, in: tokens, at: &index)).expandingTildeInPath) } @@ -1356,12 +1381,12 @@ Usage: icloud-cli devices list [--format json|text] [--cache-file PATH] icloud-cli wallet passes [--type PASS_TYPE] [--active-only] [--format json|text] [--passes-dir PATH] icloud-cli handoff list [--limit N] [--format json|text] [--handoff-dir PATH] - icloud-cli drive list [--path PATH] [--depth N] [--show-status] [--format json|text] [--icloud-root PATH] + icloud-cli drive list [--path PATH] [--depth N] [--scan-limit N] [--timeout-ms N] [--show-status] [--format json|text] [--icloud-root PATH] icloud-cli drive containers [--sort-by size|modified|name] [--format json|text] [--icloud-root PATH] - icloud-cli drive status [--path PATH] [--limit N] [--format json|text] [--icloud-root PATH] - icloud-cli drive errors [--path PATH] [--limit N] [--format json|text] [--icloud-root PATH] - icloud-cli drive shared [--path PATH] [--limit N] [--format json|text] [--icloud-root PATH] - icloud-cli drive recents [--since ISO8601] [--limit N] [--format json|text] [--icloud-root PATH] + icloud-cli drive status [--path PATH] [--scan-limit N] [--timeout-ms N] [--format json|text] [--icloud-root PATH] + icloud-cli drive errors [--path PATH] [--limit N] [--scan-limit N] [--timeout-ms N] [--format json|text] [--icloud-root PATH] + icloud-cli drive shared [--path PATH] [--limit N] [--scan-limit N] [--timeout-ms N] [--format json|text] [--icloud-root PATH] + icloud-cli drive recents [--since ISO8601] [--limit N] [--scan-limit N] [--timeout-ms N] [--format json|text] [--icloud-root PATH] icloud-cli shortcuts list [--name PATTERN] [--format json|text] [--shortcuts-dir PATH] icloud-cli photos screenshots [--format json|text] [--screenshots-dir PATH] icloud-cli photos list [--limit N] [--format json|text] [--photos-library PATH] @@ -1392,9 +1417,9 @@ Usage: icloud-cli stocks watchlist|groups [--format json|text] [--stocks-store PATH] icloud-cli weather favorites [--include-coordinates] [--format json|text] [--weather-store PATH] icloud-cli tags list [--format json|text] [--store PATH] - icloud-cli tags items --tag NAME [--path PATH] [--limit N] [--format json|text] [--icloud-root PATH] + icloud-cli tags items --tag NAME [--path PATH] [--limit N] [--scan-limit N] [--timeout-ms N] [--format json|text] [--icloud-root PATH] icloud-cli permissions doctor [--format json|text] - icloud-cli watch [--interval SECONDS] [--output-dir PATH] [--commands COMMAND,...] [--once] + icloud-cli watch [--interval SECONDS] [--scan-limit N] [--timeout-ms N] [--output-dir PATH] [--commands COMMAND,...] [--once] icloud-cli cache read COMMAND [--format json|text] [--output-dir PATH] icloud-cli cache status [--format json|text] [--output-dir PATH] icloud-cli safari tabs [--source all|current-session|last-session] [--profile NAME|all] [--format json|text] [--safari-dir PATH] diff --git a/Sources/ICloudCLICore/CommandRunner.swift b/Sources/ICloudCLICore/CommandRunner.swift index 2cf976c..88a47e9 100644 --- a/Sources/ICloudCLICore/CommandRunner.swift +++ b/Sources/ICloudCLICore/CommandRunner.swift @@ -52,8 +52,8 @@ public struct CommandRunner: Sendable { output(try render(containers, format: options.format)) return 0 case .driveList(let options): - let files = try ICloudDriveInventoryReader(rootDirectory: options.rootDirectory).listFiles(path: options.path, depth: options.depth) - output(try render(files, format: options.format)) + let report = try ICloudDriveInventoryReader(rootDirectory: options.rootDirectory).listFilesReport(path: options.path, depth: options.depth, budget: options.budget) + output(try render(report, format: options.format)) return 0 case .focusStatus(let options): let status = try FocusStatusReader(focusDirectory: options.focusDirectory).readStatus() @@ -156,7 +156,7 @@ public struct CommandRunner: Sendable { case .watch(let options): let store = CacheWatchStore(outputDirectory: options.outputDirectory) repeat { - let status = try store.refresh(commands: options.commands) + let status = try store.refresh(commands: options.commands, budget: options.budget) output(try render(status, format: .json)) if options.once { break } Thread.sleep(forTimeInterval: TimeInterval(options.intervalSeconds)) @@ -204,23 +204,24 @@ public struct CommandRunner: Sendable { return rendered case .driveStatus: let reader = ICloudDriveInventoryReader(rootDirectory: options.rootDirectory ?? FileManager.default.homeDirectoryForCurrentUser.appendingPathComponent("Library/Mobile Documents")) - return try render(try reader.syncStatus(path: options.path, limit: options.driveStatusLimit), format: options.format) + let budget = options.driveStatusLimit.map { CrawlBudget(scanLimit: $0, wallClockLimitMilliseconds: options.crawlBudget.wallClockLimitMilliseconds) } ?? options.crawlBudget + return try render(try reader.syncStatusReport(path: options.path, budget: budget), format: options.format) case .driveErrors: let reader = ICloudDriveInventoryReader(rootDirectory: options.rootDirectory ?? FileManager.default.homeDirectoryForCurrentUser.appendingPathComponent("Library/Mobile Documents")) - return try render(try reader.errorFiles(path: options.path, limit: options.limit), format: options.format) + return try render(try reader.errorFilesReport(path: options.path, limit: options.limit, budget: options.crawlBudget), format: options.format) case .driveShared: let reader = ICloudDriveInventoryReader(rootDirectory: options.rootDirectory ?? FileManager.default.homeDirectoryForCurrentUser.appendingPathComponent("Library/Mobile Documents")) - return try render(try reader.sharedItems(path: options.path, limit: options.limit), format: options.format) + return try render(try reader.sharedItemsReport(path: options.path, limit: options.limit, budget: options.crawlBudget), format: options.format) case .driveRecents: let reader = ICloudDriveInventoryReader(rootDirectory: options.rootDirectory ?? FileManager.default.homeDirectoryForCurrentUser.appendingPathComponent("Library/Mobile Documents")) - return try render(try reader.recentFiles(since: options.since, limit: options.limit), format: options.format) + return try render(try reader.recentFilesReport(since: options.since, limit: options.limit, budget: options.crawlBudget), format: options.format) case .tagsList: let reader = FinderTagsReader(preferencesFile: options.store ?? FinderTagsStoreResolver().resolvedPreferencesFile(), driveRoot: options.rootDirectory ?? FileManager.default.homeDirectoryForCurrentUser.appendingPathComponent("Library/Mobile Documents")) return try render(try reader.listTags(), format: options.format) case .taggedItems: guard let tag = options.tag else { throw CLIParseError.missingValue("--tag") } let reader = FinderTagsReader(preferencesFile: options.store ?? FinderTagsStoreResolver().resolvedPreferencesFile(), driveRoot: options.rootDirectory ?? FileManager.default.homeDirectoryForCurrentUser.appendingPathComponent("Library/Mobile Documents")) - return try render(try reader.items(tag: tag, path: options.path, limit: options.limit), format: options.format) + return try render(try reader.itemsReport(tag: tag, path: options.path, limit: options.limit, budget: options.crawlBudget), format: options.format) default: let store = options.store ?? LocalMetadataStoreReader.defaultStore(for: command) let rows = try LocalMetadataStoreReader(database: store).rows(for: command, options: options) diff --git a/Sources/ICloudCLICore/CrawlBudget.swift b/Sources/ICloudCLICore/CrawlBudget.swift new file mode 100644 index 0000000..677bc92 --- /dev/null +++ b/Sources/ICloudCLICore/CrawlBudget.swift @@ -0,0 +1,71 @@ +import Foundation + +public enum CrawlState: String, Codable, Equatable, Sendable { + case complete + case partial + case timeout +} + +public struct CrawlBudget: Codable, Equatable, Sendable { + public let scanLimit: Int + public let wallClockLimitMilliseconds: Int + + public init(scanLimit: Int, wallClockLimitMilliseconds: Int) { + self.scanLimit = max(1, scanLimit) + self.wallClockLimitMilliseconds = max(1, wallClockLimitMilliseconds) + } + + public static let defaultDrive = CrawlBudget(scanLimit: 5_000, wallClockLimitMilliseconds: 10_000) + public static let defaultPolling = CrawlBudget(scanLimit: 2_000, wallClockLimitMilliseconds: 8_000) +} + +public struct CrawlReport: Codable, Sendable { + public let schemaVersion: String + public let providerId: String + public let state: CrawlState + public let data: Payload + public let scannedCount: Int + public let resultCount: Int + public let totalAvailable: Int? + public let scanLimit: Int + public let resultLimit: Int? + public let wallClockLimitMilliseconds: Int + public let elapsedMilliseconds: Int + public let nextAction: String? + + public init( + providerId: String, + state: CrawlState, + data: Payload, + scannedCount: Int, + resultCount: Int, + totalAvailable: Int?, + budget: CrawlBudget, + resultLimit: Int? = nil, + elapsedMilliseconds: Int, + nextAction: String? + ) { + self.schemaVersion = "icloud-cli.crawl.v1" + self.providerId = providerId + self.state = state + self.data = data + self.scannedCount = scannedCount + self.resultCount = resultCount + self.totalAvailable = totalAvailable + self.scanLimit = budget.scanLimit + self.resultLimit = resultLimit + self.wallClockLimitMilliseconds = budget.wallClockLimitMilliseconds + self.elapsedMilliseconds = elapsedMilliseconds + self.nextAction = nextAction + } +} + +public struct CrawlBudgetExceeded: Error, Equatable, Sendable { + public let providerId: String + public let state: CrawlState + + public init(providerId: String, state: CrawlState) { + self.providerId = providerId + self.state = state + } +} diff --git a/Sources/ICloudCLICore/DriveInventory.swift b/Sources/ICloudCLICore/DriveInventory.swift index 1028241..2e64376 100644 --- a/Sources/ICloudCLICore/DriveInventory.swift +++ b/Sources/ICloudCLICore/DriveInventory.swift @@ -70,27 +70,63 @@ public enum DriveInventoryError: Error, LocalizedError, Equatable { public struct ICloudDriveInventoryReader: Sendable { public let rootDirectory: URL - public init(rootDirectory: URL = FileManager.default.homeDirectoryForCurrentUser.appendingPathComponent("Library/Mobile Documents")) { + private let now: @Sendable () -> TimeInterval + + public init( + rootDirectory: URL = FileManager.default.homeDirectoryForCurrentUser.appendingPathComponent("Library/Mobile Documents"), + now: @escaping @Sendable () -> TimeInterval = { ProcessInfo.processInfo.systemUptime } + ) { self.rootDirectory = rootDirectory.standardizedFileURL + self.now = now } public func listFiles(path requestedPath: String? = nil, depth: Int = 2, limit: Int? = nil) throws -> [ICloudDriveFile] { + let budget = CrawlBudget( + scanLimit: limit ?? CrawlBudget.defaultDrive.scanLimit, + wallClockLimitMilliseconds: CrawlBudget.defaultDrive.wallClockLimitMilliseconds + ) + return try listFilesReport(path: requestedPath, depth: depth, budget: budget).data + } + + public func listFilesReport( + path requestedPath: String? = nil, + depth: Int = 2, + budget: CrawlBudget = .defaultDrive + ) throws -> CrawlReport<[ICloudDriveFile]> { guard FileManager.default.fileExists(atPath: rootDirectory.path) else { throw DriveInventoryError.missingRoot(rootDirectory.path) } let startURL = try scopedURL(for: requestedPath) + let startedAt = now() let startValues = try? startURL.resourceValues(forKeys: [.isDirectoryKey, .fileSizeKey, .contentModificationDateKey]) if startValues?.isDirectory != true { - return [fileEntry(for: startURL, values: startValues)] + let file = fileEntry(for: startURL, values: startValues) + return CrawlReport(providerId: "drive", state: .complete, data: [file], scannedCount: 1, resultCount: 1, totalAvailable: 1, budget: budget, elapsedMilliseconds: elapsedMilliseconds(since: startedAt), nextAction: nil) } let maxDepth = max(0, depth) - let maxFiles = limit.map { max(1, $0) } - var result: [ICloudDriveFile] = [] - try walkFiles(at: startURL, currentDepth: 0, maxDepth: maxDepth, maxFiles: maxFiles, into: &result) - return result.sorted { lhs, rhs in lhs.path.localizedStandardCompare(rhs.path) == .orderedAscending } + let walk = walkFiles(at: startURL, maxDepth: maxDepth, budget: budget, startedAt: startedAt) + let files = walk.files.sorted { lhs, rhs in lhs.path.localizedStandardCompare(rhs.path) == .orderedAscending } + let state = walk.termination ?? .complete + return CrawlReport( + providerId: "drive", + state: state, + data: files, + scannedCount: walk.scannedCount, + resultCount: files.count, + totalAvailable: state == .complete ? walk.scannedCount : nil, + budget: budget, + elapsedMilliseconds: elapsedMilliseconds(since: startedAt), + nextAction: nextAction(for: state) + ) } public func syncStatus(path requestedPath: String? = nil, limit: Int? = nil) throws -> ICloudDriveSyncSummary { - let files = try listFiles(path: requestedPath, depth: Int.max, limit: limit) - return ICloudDriveSyncSummary( + let budget = CrawlBudget(scanLimit: limit ?? CrawlBudget.defaultDrive.scanLimit, wallClockLimitMilliseconds: CrawlBudget.defaultDrive.wallClockLimitMilliseconds) + return try syncStatusReport(path: requestedPath, budget: budget).data + } + + public func syncStatusReport(path requestedPath: String? = nil, budget: CrawlBudget = .defaultDrive) throws -> CrawlReport { + let crawl = try listFilesReport(path: requestedPath, depth: Int.max, budget: budget) + let files = crawl.data + let summary = ICloudDriveSyncSummary( downloadedCount: files.filter { $0.iCloudStatus == .downloaded }.count, cloudOnlyCount: files.filter { $0.iCloudStatus == .evicted || $0.iCloudStatus == .notDownloaded }.count, downloadingCount: files.filter { $0.iCloudStatus == .downloading }.count, @@ -99,39 +135,64 @@ public struct ICloudDriveInventoryReader: Sendable { errorCount: files.filter { $0.iCloudStatus == .error }.count, unknownCount: files.filter { $0.iCloudStatus == .unknown }.count ) + return crawl.replacingData(summary, resultCount: files.count, totalAvailable: crawl.totalAvailable) } public func errorFiles(path requestedPath: String? = nil, limit: Int = 500, scanLimit: Int? = nil) throws -> [ICloudDriveErrorEntry] { let resultLimit = max(1, limit) let traversalLimit = scanLimit ?? filteredDriveScanLimit(for: resultLimit) - return try listFiles(path: requestedPath, depth: Int.max, limit: traversalLimit) - .filter { $0.iCloudStatus == .error } + let budget = CrawlBudget(scanLimit: traversalLimit, wallClockLimitMilliseconds: CrawlBudget.defaultDrive.wallClockLimitMilliseconds) + return try errorFilesReport(path: requestedPath, limit: resultLimit, budget: budget).data + } + + public func errorFilesReport(path requestedPath: String? = nil, limit: Int = 500, budget: CrawlBudget = .defaultDrive) throws -> CrawlReport<[ICloudDriveErrorEntry]> { + let resultLimit = max(1, limit) + let crawl = try listFilesReport(path: requestedPath, depth: Int.max, budget: budget) + let matches = crawl.data.filter { $0.iCloudStatus == .error } + let results = matches .prefix(resultLimit) .map { ICloudDriveErrorEntry(path: $0.path, category: "icloud-sync-error") } + return crawl.replacingData(results, resultCount: results.count, totalAvailable: crawl.state == .complete ? matches.count : nil, resultLimit: resultLimit) } public func recentFiles(since: String? = nil, limit: Int = 50) throws -> [ICloudDriveFile] { + let resultLimit = max(1, limit) + let budget = CrawlBudget(scanLimit: max(200, bounded(limit, defaultValue: 50, max: 1_000) * 5), wallClockLimitMilliseconds: CrawlBudget.defaultDrive.wallClockLimitMilliseconds) + return try recentFilesReport(since: since, limit: resultLimit, budget: budget).data + } + + public func recentFilesReport(since: String? = nil, limit: Int = 50, budget: CrawlBudget = .defaultDrive) throws -> CrawlReport<[ICloudDriveFile]> { let floor = since.flatMap { ISO8601DateFormatter().date(from: $0) } - let scanLimit = max(200, bounded(limit, defaultValue: 50, max: 1_000) * 5) - return try listFiles(depth: Int.max, limit: scanLimit) - .filter { file in + let resultLimit = max(1, limit) + let crawl = try listFilesReport(depth: Int.max, budget: budget) + let matches = crawl.data.filter { file in guard let floor else { return true } return file.modifiedAt.map { $0 >= floor } ?? false } .sorted { ($0.modifiedAt ?? .distantPast, $0.path) > ($1.modifiedAt ?? .distantPast, $1.path) } - .prefix(max(1, limit)) + let results = matches + .prefix(resultLimit) .map { $0 } + return crawl.replacingData(results, resultCount: results.count, totalAvailable: crawl.state == .complete ? matches.count : nil, resultLimit: resultLimit) } public func sharedItems(path requestedPath: String? = nil, limit: Int = 500, scanLimit: Int? = nil) throws -> [ICloudDriveSharedItem] { let resultLimit = max(1, limit) let traversalLimit = scanLimit ?? filteredDriveScanLimit(for: resultLimit) - return try listFiles(path: requestedPath, depth: Int.max, limit: traversalLimit) - .filter { $0.path.localizedCaseInsensitiveContains(".shared") } + let budget = CrawlBudget(scanLimit: traversalLimit, wallClockLimitMilliseconds: CrawlBudget.defaultDrive.wallClockLimitMilliseconds) + return try sharedItemsReport(path: requestedPath, limit: resultLimit, budget: budget).data + } + + public func sharedItemsReport(path requestedPath: String? = nil, limit: Int = 500, budget: CrawlBudget = .defaultDrive) throws -> CrawlReport<[ICloudDriveSharedItem]> { + let resultLimit = max(1, limit) + let crawl = try listFilesReport(path: requestedPath, depth: Int.max, budget: budget) + let matches = crawl.data.filter { $0.path.localizedCaseInsensitiveContains(".shared") } + let results = matches .prefix(resultLimit) .map { file in ICloudDriveSharedItem(path: file.path, owner: nil, role: nil, dateShared: file.modifiedAt, iCloudStatus: file.iCloudStatus) } + return crawl.replacingData(results, resultCount: results.count, totalAvailable: crawl.state == .complete ? matches.count : nil, resultLimit: resultLimit) } public func listContainers(sortBy: DriveSortKey = .name) throws -> [ICloudDriveContainer] { @@ -161,26 +222,53 @@ public struct ICloudDriveInventoryReader: Sendable { max(200, bounded(resultLimit, defaultValue: 500, max: 2_000) * 5) } - private func walkFiles(at directory: URL, currentDepth: Int, maxDepth: Int, maxFiles: Int?, into result: inout [ICloudDriveFile]) throws { - if let maxFiles, result.count >= maxFiles { - return + private func walkFiles(at directory: URL, maxDepth: Int, budget: CrawlBudget, startedAt: TimeInterval) -> DriveWalkState { + let box = DriveWalkBox() + let completed = DispatchSemaphore(value: 0) + DispatchQueue.global(qos: .userInitiated).async { + enumerateFiles(at: directory, maxDepth: maxDepth, budget: budget, startedAt: startedAt, box: box) + completed.signal() + } + if completed.wait(timeout: .now() + .milliseconds(budget.wallClockLimitMilliseconds)) != .success { + box.finish(.timeout) } - let directoryValues = try? directory.resourceValues(forKeys: [.isDirectoryKey, .fileSizeKey, .contentModificationDateKey]) - if directoryValues?.isDirectory != true { - result.append(fileEntry(for: directory, values: directoryValues)) + return box.snapshot() + } + + private func enumerateFiles(at directory: URL, maxDepth: Int, budget: CrawlBudget, startedAt: TimeInterval, box: DriveWalkBox) { + let keys: [URLResourceKey] = [.isDirectoryKey, .fileSizeKey, .contentModificationDateKey] + guard let enumerator = FileManager.default.enumerator( + at: directory, + includingPropertiesForKeys: keys, + options: [], + errorHandler: { _, _ in true } + ) else { + box.finish(.complete) return } - let children = try FileManager.default.contentsOfDirectory(at: directory, includingPropertiesForKeys: [.isDirectoryKey, .fileSizeKey, .contentModificationDateKey], options: []) - for child in children { - if let maxFiles, result.count >= maxFiles { - return - } - let values = try? child.resourceValues(forKeys: [.isDirectoryKey, .fileSizeKey, .contentModificationDateKey]) + while true { + if elapsedMilliseconds(since: startedAt) >= budget.wallClockLimitMilliseconds { box.finish(.timeout); return } + if box.shouldStop { return } + guard let child = enumerator.nextObject() as? URL else { box.finish(.complete); return } + if box.shouldStop { return } + let values = try? child.resourceValues(forKeys: Set(keys)) if values?.isDirectory == true { - if currentDepth < maxDepth { try walkFiles(at: child, currentDepth: currentDepth + 1, maxDepth: maxDepth, maxFiles: maxFiles, into: &result) } + if enumerator.level > maxDepth { enumerator.skipDescendants() } continue } - result.append(fileEntry(for: child, values: values)) + guard box.append(fileEntry(for: child, values: values), scanLimit: budget.scanLimit) else { return } + } + } + + private func elapsedMilliseconds(since startedAt: TimeInterval) -> Int { + max(0, Int((now() - startedAt) * 1_000)) + } + + private func nextAction(for state: CrawlState) -> String? { + switch state { + case .complete: return nil + case .partial: return "Increase --scan-limit or narrow --path to inspect more items." + case .timeout: return "Increase --timeout-ms or narrow --path to finish the crawl." } } @@ -255,6 +343,71 @@ public struct ICloudDriveInventoryReader: Sendable { } } +private struct DriveWalkState { + var files: [ICloudDriveFile] = [] + var scannedCount = 0 + var termination: CrawlState? +} + +private final class DriveWalkBox: @unchecked Sendable { + private let lock = NSLock() + private var state = DriveWalkState() + + var shouldStop: Bool { + lock.lock(); defer { lock.unlock() } + return state.termination != nil + } + + func append(_ file: ICloudDriveFile, scanLimit: Int) -> Bool { + lock.lock(); defer { lock.unlock() } + guard state.termination == nil else { return false } + guard state.scannedCount < scanLimit else { + state.termination = .partial + return false + } + state.files.append(file) + state.scannedCount += 1 + if state.scannedCount >= scanLimit { + state.termination = .partial + return false + } + return true + } + + func finish(_ termination: CrawlState) { + lock.lock(); defer { lock.unlock() } + guard state.termination == nil else { return } + state.termination = termination + } + + func snapshot() -> DriveWalkState { + lock.lock(); defer { lock.unlock() } + return state + } +} + +private extension CrawlReport { + func replacingData( + _ data: NewPayload, + resultCount: Int, + totalAvailable: Int?, + resultLimit: Int? = nil + ) -> CrawlReport { + CrawlReport( + providerId: providerId, + state: state, + data: data, + scannedCount: scannedCount, + resultCount: resultCount, + totalAvailable: totalAvailable, + budget: CrawlBudget(scanLimit: scanLimit, wallClockLimitMilliseconds: wallClockLimitMilliseconds), + resultLimit: resultLimit, + elapsedMilliseconds: elapsedMilliseconds, + nextAction: nextAction + ) + } +} + private func bounded(_ value: Int, defaultValue: Int, max: Int) -> Int { guard value > 0 else { return defaultValue } return Swift.min(value, max) diff --git a/Sources/ICloudCLICore/ProviderManifest.swift b/Sources/ICloudCLICore/ProviderManifest.swift index d31d2dd..aa36f79 100644 --- a/Sources/ICloudCLICore/ProviderManifest.swift +++ b/Sources/ICloudCLICore/ProviderManifest.swift @@ -38,7 +38,7 @@ public enum ProviderRegistry { provider("calendar", "Calendar", .beta, .sqlite, .high, ["calendar accounts", "calendar list", "calendar events"], ["accounts", "calendars", "events", "date-filtering"]), provider("contacts", "Contacts", .beta, .sqlite, .high, ["contacts list"], ["inventory", "search"]), provider("devices", "iCloud Devices", .stable, .preferences, .moderate, ["devices list"], ["inventory", "status"], true), - provider("drive", "iCloud Drive", .stable, .filesystem, .moderate, ["drive containers", "drive errors", "drive list", "drive recents", "drive shared", "drive status"], ["containers", "inventory", "recents", "sharing", "sync-status"], true), + provider("drive", "iCloud Drive", .stable, .filesystem, .moderate, ["drive containers", "drive errors", "drive list", "drive recents", "drive shared", "drive status"], ["bounded-crawl", "containers", "inventory", "recents", "sharing", "sync-status"], true), provider("family", "Family Sharing", .stable, .preferences, .high, ["family status"], ["membership", "status"]), provider("findmy", "Find My", .experimental, .sqlite, .high, ["findmy devices", "findmy people"], ["devices", "people", "location"]), provider("focus", "Focus", .stable, .preferences, .moderate, ["focus status"], ["status"], true), @@ -58,7 +58,7 @@ public enum ProviderRegistry { provider("shortcuts", "Shortcuts", .stable, .filesystem, .moderate, ["shortcuts list"], ["inventory", "search"], true), provider("stocks", "Stocks", .beta, .sqlite, .moderate, ["stocks groups", "stocks watchlist"], ["groups", "watchlist"]), provider("storage", "iCloud Storage", .stable, .preferences, .low, ["storage status"], ["quota", "status"], true), - provider("tags", "Finder Tags", .beta, .mixed, .moderate, ["tags items", "tags list"], ["inventory", "search", "tags"]), + provider("tags", "Finder Tags", .beta, .mixed, .moderate, ["tags items", "tags list"], ["bounded-crawl", "inventory", "search", "tags"]), provider("voice-memos", "Voice Memos", .beta, .sqlite, .high, ["voice-memos list"], ["folders", "inventory", "date-filtering"]), provider("wallet", "Wallet", .beta, .filesystem, .high, ["wallet passes"], ["inventory", "type-filtering"]), provider("weather", "Weather", .beta, .sqlite, .high, ["weather favorites"], ["favorites", "location"]), diff --git a/Tests/ICloudCLICoreTests/CommandRunnerHarnessTests.swift b/Tests/ICloudCLICoreTests/CommandRunnerHarnessTests.swift index 42422b8..6b63bb1 100644 --- a/Tests/ICloudCLICoreTests/CommandRunnerHarnessTests.swift +++ b/Tests/ICloudCLICoreTests/CommandRunnerHarnessTests.swift @@ -128,7 +128,7 @@ import Testing #expect(FileManager.default.fileExists(atPath: snapshotOutput.path)) #expect(sink.errors.isEmpty) #expect(sink.output.count == commands.count) - #expect(sink.output.last?.contains("Unsupported cache command: unknown-command") == true) + #expect(sink.output.last?.contains("provider crawl failed") == true) } @Test func driveStatusCommandHonorsExplicitTraversalLimit() throws { @@ -148,7 +148,7 @@ import Testing #expect(runner.run(arguments: ["icloud-cli", "drive", "status", "--icloud-root", driveRoot.path, "--limit", "1"]) == 0) let output = try #require(sink.output.first) - let summary = try JSONDecoder().decode(ICloudDriveSyncSummary.self, from: Data(output.utf8)) + let summary = try JSONDecoder().decode(CrawlReport.self, from: Data(output.utf8)).data #expect(summary.downloadedCount + summary.cloudOnlyCount + summary.downloadingCount + summary.uploadedCount + summary.uploadingCount + summary.errorCount + summary.unknownCount == 1) } @@ -197,8 +197,8 @@ import Testing #expect(sink.output.count == 2) let decoder = JSONDecoder() - let full = try decoder.decode(ICloudDriveSyncSummary.self, from: Data(sink.output[0].utf8)) - let capped = try decoder.decode(ICloudDriveSyncSummary.self, from: Data(sink.output[1].utf8)) + let full = try decoder.decode(CrawlReport.self, from: Data(sink.output[0].utf8)).data + let capped = try decoder.decode(CrawlReport.self, from: Data(sink.output[1].utf8)).data #expect(full.downloadedCount + full.cloudOnlyCount + full.downloadingCount + full.uploadedCount + full.uploadingCount + full.errorCount + full.unknownCount == 2) #expect(capped.downloadedCount + capped.cloudOnlyCount + capped.downloadingCount + capped.uploadedCount + capped.uploadingCount + capped.errorCount + capped.unknownCount == 1) diff --git a/Tests/ICloudCLICoreTests/CrawlBudgetTests.swift b/Tests/ICloudCLICoreTests/CrawlBudgetTests.swift new file mode 100644 index 0000000..12ac1aa --- /dev/null +++ b/Tests/ICloudCLICoreTests/CrawlBudgetTests.swift @@ -0,0 +1,128 @@ +import Foundation +import Testing +@testable import ICloudCLICore + +@Test func driveCrawlReportsScanBudgetPartialState() throws { + let root = try crawlFixture(named: "drive-scan", fileCount: 12) + defer { try? FileManager.default.removeItem(at: root.deletingLastPathComponent()) } + + let report = try ICloudDriveInventoryReader(rootDirectory: root).listFilesReport( + depth: Int.max, + budget: CrawlBudget(scanLimit: 3, wallClockLimitMilliseconds: 5_000) + ) + + #expect(report.state == .partial) + #expect(report.scannedCount == 3) + #expect(report.resultCount == 3) + #expect(report.totalAvailable == nil) + #expect(report.scanLimit == 3) + #expect(report.nextAction?.contains("--scan-limit") == true) +} + +@Test func driveCrawlReportsDeterministicTimeout() throws { + let root = try crawlFixture(named: "drive-timeout", fileCount: 4) + defer { try? FileManager.default.removeItem(at: root.deletingLastPathComponent()) } + let clock = StepClock(values: [0, 0, 0.010, 0.010]) + + let report = try ICloudDriveInventoryReader(rootDirectory: root, now: { clock.next() }).listFilesReport( + depth: Int.max, + budget: CrawlBudget(scanLimit: 100, wallClockLimitMilliseconds: 5) + ) + + #expect(report.state == .timeout) + #expect(report.scannedCount < 4) + #expect(report.totalAvailable == nil) + #expect(report.nextAction?.contains("--timeout-ms") == true) +} + +@Test func finderTagsPreserveScanAndResultLimits() throws { + let root = try crawlFixture(named: "tags", fileCount: 8, taggedIndexes: [1, 5]) + defer { try? FileManager.default.removeItem(at: root.deletingLastPathComponent()) } + + let report = try FinderTagsReader(driveRoot: root).itemsReport( + tag: "Red", + path: nil, + limit: 1, + budget: CrawlBudget(scanLimit: 6, wallClockLimitMilliseconds: 5_000) + ) + + #expect(report.providerId == "tags") + #expect(report.scannedCount == 6) + #expect(report.resultCount == 1) + #expect(report.resultLimit == 1) + #expect(report.totalAvailable == nil) +} + +@Test func completedFinderTagCrawlReportsTotalMatchingRows() throws { + let root = try crawlFixture(named: "tags-total", fileCount: 8, taggedIndexes: [1, 5]) + defer { try? FileManager.default.removeItem(at: root.deletingLastPathComponent()) } + + let report = try FinderTagsReader(driveRoot: root).itemsReport( + tag: "Red", + path: nil, + limit: 1, + budget: CrawlBudget(scanLimit: 20, wallClockLimitMilliseconds: 5_000) + ) + + #expect(report.state == .complete) + #expect(report.scannedCount == 8) + #expect(report.resultCount == 1) + #expect(report.totalAvailable == 2) +} + +@Test func watchContinuesAfterTimeoutWithRedactedFailure() throws { + let root = FileManager.default.temporaryDirectory.appendingPathComponent("icloud-cli-watch-budget-\(UUID().uuidString)") + defer { try? FileManager.default.removeItem(at: root) } + let snapshotter: CacheSnapshotter = { command, _ in + if command == "drive-list" { throw CrawlBudgetExceeded(providerId: "drive", state: .timeout) } + return "[]" + } + let store = CacheWatchStore(outputDirectory: root, snapshotter: snapshotter) + + let statuses = try store.refresh( + commands: ["drive-list", "photos-screenshots"], + budget: CrawlBudget(scanLimit: 5, wallClockLimitMilliseconds: 5) + ) + + #expect(statuses.count == 2) + #expect(statuses[0].ok == false) + #expect(statuses[0].failure?.code == "timeout") + #expect(statuses[0].failure?.providerId == "drive") + #expect(statuses[0].error == "provider crawl timed out") + #expect(!statuses[0].error!.contains(root.path)) + #expect(statuses[1].ok == true) + #expect(try store.read(command: "drive-list").failure?.guidance.contains("budget") == true) +} + +@Test func parsesExplicitCrawlBudgets() throws { + let drive = try CLIParser().parse(arguments: ["icloud-cli", "drive", "list", "--scan-limit", "25", "--timeout-ms", "900"]) + guard case .driveList(let driveOptions) = drive else { Issue.record("Expected drive list"); return } + #expect(driveOptions.budget == CrawlBudget(scanLimit: 25, wallClockLimitMilliseconds: 900)) + + let watch = try CLIParser().parse(arguments: ["icloud-cli", "watch", "--once", "--scan-limit", "40", "--timeout-ms", "700"]) + guard case .watch(let watchOptions) = watch else { Issue.record("Expected watch"); return } + #expect(watchOptions.budget == CrawlBudget(scanLimit: 40, wallClockLimitMilliseconds: 700)) +} + +private final class StepClock: @unchecked Sendable { + private var values: [TimeInterval] + private let lock = NSLock() + init(values: [TimeInterval]) { self.values = values } + func next() -> TimeInterval { + lock.lock(); defer { lock.unlock() } + if values.count > 1 { return values.removeFirst() } + return values.first ?? 0 + } +} + +private func crawlFixture(named name: String, fileCount: Int, taggedIndexes: Set = []) throws -> URL { + let parent = FileManager.default.temporaryDirectory.appendingPathComponent("icloud-cli-crawl-\(name)-\(UUID().uuidString)") + let root = parent.appendingPathComponent("MobileDocuments") + let container = root.appendingPathComponent("com~example~App") + try FileManager.default.createDirectory(at: container, withIntermediateDirectories: true) + for index in 0.. list[tuple[str, list[str]]]: ("devices list", ["devices", "list", "--format", "json"]), ("wallet passes", ["wallet", "passes", "--format", "json"]), ("handoff list", ["handoff", "list", "--limit", "10", "--format", "json"]), - ("drive list", ["drive", "list", "--depth", "1", "--format", "json"]), + ("drive list", ["drive", "list", "--depth", "1", "--scan-limit", "2000", "--timeout-ms", "10000", "--format", "json"]), ("drive containers", ["drive", "containers", "--sort-by", "name", "--format", "json"]), - ("drive status", ["drive", "status", "--limit", "25", "--format", "json"]), - ("drive errors", ["drive", "errors", "--limit", "25", "--format", "json"]), - ("drive shared", ["drive", "shared", "--limit", "25", "--format", "json"]), - ("drive recents", ["drive", "recents", "--limit", "25", "--format", "json"]), + ("drive status", ["drive", "status", "--scan-limit", "2000", "--timeout-ms", "10000", "--format", "json"]), + ("drive errors", ["drive", "errors", "--limit", "25", "--scan-limit", "2000", "--timeout-ms", "10000", "--format", "json"]), + ("drive shared", ["drive", "shared", "--limit", "25", "--scan-limit", "2000", "--timeout-ms", "10000", "--format", "json"]), + ("drive recents", ["drive", "recents", "--limit", "25", "--scan-limit", "2000", "--timeout-ms", "10000", "--format", "json"]), ("shortcuts list", ["shortcuts", "list", "--format", "json"]), ("photos screenshots", ["photos", "screenshots", "--format", "json"]), ("photos list", ["photos", "list", "--limit", "25", "--format", "json"]), @@ -81,7 +82,7 @@ def command_catalog(cache_dir: str, tag: str) -> list[tuple[str, list[str]]]: ("stocks groups", ["stocks", "groups", "--format", "json"]), ("weather favorites", ["weather", "favorites", "--format", "json"]), ("tags list", ["tags", "list", "--format", "json"]), - ("tags items", ["tags", "items", "--tag", tag, "--limit", "25", "--format", "json"]), + ("tags items", ["tags", "items", "--tag", tag, "--limit", "25", "--scan-limit", "2000", "--timeout-ms", "10000", "--format", "json"]), ("permissions doctor", ["permissions", "doctor", "--format", "json"]), ("safari tabs", ["safari", "tabs", "--source", "all", "--format", "json"]), ("safari history", ["safari", "history", "--confirm-sensitive", "--limit", "10", "--redact-urls", "--format", "json"]), @@ -93,7 +94,7 @@ def command_catalog(cache_dir: str, tag: str) -> list[tuple[str, list[str]]]: ("safari profiles list", ["safari", "profiles", "list", "--format", "json"]), ("safari extensions list", ["safari", "extensions", "list", "--format", "json"]), ("cache status", ["cache", "status", "--format", "json", "--output-dir", cache_dir]), - ("watch once", ["watch", "--once", "--output-dir", cache_dir]), + ("watch once", ["watch", "--once", "--scan-limit", "2000", "--timeout-ms", "10000", "--output-dir", cache_dir]), ("cache read drive-list", ["cache", "read", "drive-list", "--format", "json", "--output-dir", cache_dir]), ] @@ -102,6 +103,9 @@ def payload_shape(payload: Any) -> tuple[int, str, str]: if isinstance(payload, list): return len(payload), "array", "" if isinstance(payload, dict): + if payload.get("schemaVersion") == "icloud-cli.crawl.v1" and "data" in payload: + count, shape, _ = payload_shape(payload["data"]) + return count, f"crawl:{shape}", scalar_note(payload) for key in ( "items", "files", @@ -178,7 +182,10 @@ def classify(exe: Path, args: list[str], timeout: int) -> tuple[str, str, int, s return "EMPTY", "0", 0, "-", "no stdout" try: - count, shape, note = payload_shape(json.loads(stdout)) + payload = json.loads(stdout) + count, shape, note = payload_shape(payload) + if isinstance(payload, dict) and payload.get("state") == "timeout": + return "TIMEOUT", "0", count, shape, note status = "OK_DATA" if count > 0 else "OK_EMPTY" return status, "0", count, shape, note except json.JSONDecodeError: @@ -198,10 +205,24 @@ def main() -> int: return 2 with tempfile.TemporaryDirectory(prefix="icloud-cli-live-cache-") as cache_dir: + totals: Counter[str] = Counter() print("command\tstatus\texit\tcount\tshape\tnote") for name, command_args in command_catalog(cache_dir, args.tag): status, exit_code, count, shape, note = classify(exe, command_args, args.timeout) print(f"{name}\t{status}\t{exit_code}\t{count}\t{shape}\t{note}") + if status == "TIMEOUT": + totals["timeout"] += 1 + elif status == "FAIL": + totals["fail"] += 1 + elif status in {"EMPTY", "OK_EMPTY"}: + totals["empty"] += 1 + else: + totals["pass"] += 1 + print( + "summary\t" + f"pass={totals['pass']}\tempty={totals['empty']}\t" + f"fail={totals['fail']}\ttimeout={totals['timeout']}" + ) return 0 From 5b46bde7aae53bfc0b1cd4e7125cc3cb64d21cf3 Mon Sep 17 00:00:00 2001 From: Hermes Date: Sun, 12 Jul 2026 11:48:44 +0100 Subject: [PATCH 3/4] Add resumable per-provider metadata archives Define versioned archive sync, storage, status, migration, retention, and privacy contracts with idempotent upserts, explicit tombstones, bounded attempts, persisted freshness and counts, private atomic files, and fail-closed sensitivity gates.\n\nCloses #93. --- README.md | 2 +- Sources/ICloudCLICore/CommandLine.swift | 81 ++++ Sources/ICloudCLICore/CommandRunner.swift | 12 + Sources/ICloudCLICore/ProviderArchive.swift | 434 ++++++++++++++++++ Sources/ICloudCLICore/ProviderManifest.swift | 16 +- .../ProviderArchiveTests.swift | 209 +++++++++ docs/privacy.md | 2 + docs/provider-archives.md | 26 ++ 8 files changed, 773 insertions(+), 9 deletions(-) create mode 100644 Sources/ICloudCLICore/ProviderArchive.swift create mode 100644 Tests/ICloudCLICoreTests/ProviderArchiveTests.swift create mode 100644 docs/provider-archives.md diff --git a/README.md b/README.md index 9bf3cfc..7a2d228 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ The implementation reads local Safari session and metadata property lists from ` If Safari session files are unreadable, the command exits with an error naming the file paths it tried. If the files are readable but empty, the error says no tabs were found instead of treating it as a permissions problem. -Use `icloud-cli safari cloud-tabs probe` to check whether Safari's cross-device tab store is present and readable before using `icloud-cli safari cloud-tabs list --confirm-sensitive`. Use `icloud-cli permissions doctor` when a command reports missing or unreadable local stores; it probes source paths without reading payload content and gives the Full Disk Access hint per command. `icloud-cli providers list` exposes a static, versioned capability registry without reading live data; see [docs/provider-manifest.md](docs/provider-manifest.md) for its schema and compatibility policy. Recursive Drive and Finder-tag commands emit bounded crawl reports documented in [docs/crawl-budgets.md](docs/crawl-budgets.md). See [docs/cloudtabs.md](docs/cloudtabs.md) for the investigation notes, [docs/privacy.md](docs/privacy.md) for the privacy and permissions model, and [docs/openclaw-skill-contract.md](docs/openclaw-skill-contract.md) for the OpenClaw integration contract. +Use `icloud-cli safari cloud-tabs probe` to check whether Safari's cross-device tab store is present and readable before using `icloud-cli safari cloud-tabs list --confirm-sensitive`. Use `icloud-cli permissions doctor` when a command reports missing or unreadable local stores; it probes source paths without reading payload content and gives the Full Disk Access hint per command. `icloud-cli providers list` exposes a static, versioned capability registry without reading live data; see [docs/provider-manifest.md](docs/provider-manifest.md) for its schema and compatibility policy. Recursive Drive and Finder-tag commands emit bounded crawl reports documented in [docs/crawl-budgets.md](docs/crawl-budgets.md), while [docs/provider-archives.md](docs/provider-archives.md) defines resumable metadata archives. See [docs/cloudtabs.md](docs/cloudtabs.md) for the investigation notes, [docs/privacy.md](docs/privacy.md) for the privacy and permissions model, and [docs/openclaw-skill-contract.md](docs/openclaw-skill-contract.md) for the OpenClaw integration contract. ## Build diff --git a/Sources/ICloudCLICore/CommandLine.swift b/Sources/ICloudCLICore/CommandLine.swift index 8daa7ee..9baee1a 100644 --- a/Sources/ICloudCLICore/CommandLine.swift +++ b/Sources/ICloudCLICore/CommandLine.swift @@ -1,5 +1,33 @@ import Foundation +public struct ArchiveSyncOptions: Equatable, Sendable { + public var providerId: String + public var input: URL + public var archiveDirectory: URL + public var budget: CrawlBudget + public var format: OutputFormat + + public init(providerId: String, input: URL, archiveDirectory: URL = FileManager.default.homeDirectoryForCurrentUser.appendingPathComponent(".icloud-cli/archives"), budget: CrawlBudget = .defaultPolling, format: OutputFormat = .json) { + self.providerId = providerId + self.input = input + self.archiveDirectory = archiveDirectory + self.budget = budget + self.format = format + } +} + +public struct ArchiveStatusOptions: Equatable, Sendable { + public var providerId: String + public var archiveDirectory: URL + public var format: OutputFormat + + public init(providerId: String, archiveDirectory: URL = FileManager.default.homeDirectoryForCurrentUser.appendingPathComponent(".icloud-cli/archives"), format: OutputFormat = .json) { + self.providerId = providerId + self.archiveDirectory = archiveDirectory + self.format = format + } +} + public enum OutputFormat: String, Sendable { case json case text @@ -521,6 +549,8 @@ public struct MetadataOptions: Equatable, Sendable { } public enum CLICommand: Equatable, Sendable { + case archiveStatus(ArchiveStatusOptions) + case archiveSync(ArchiveSyncOptions) case cacheRead(CacheOptions) case cacheStatus(CacheOptions) case cloudTabsProbe(CloudTabsProbeOptions) @@ -584,6 +614,15 @@ public struct CLIParser: Sendable { if tokens == ["--version"] || tokens == ["-V"] { return .version } let topCommand = tokens.removeFirst() + if topCommand == "archive" { + guard let subcommand = tokens.first else { throw CLIParseError.unknownCommand("archive") } + tokens.removeFirst() + switch subcommand { + case "sync": return .archiveSync(try parseArchiveSyncOptions(tokens)) + case "status": return .archiveStatus(try parseArchiveStatusOptions(tokens)) + default: throw CLIParseError.unknownCommand("archive \(subcommand)") + } + } if topCommand == "providers" { guard let subcommand = tokens.first else { throw CLIParseError.unknownCommand("providers") } tokens.removeFirst() @@ -894,6 +933,45 @@ public struct CLIParser: Sendable { return options } + private func parseArchiveSyncOptions(_ tokens: [String]) throws -> ArchiveSyncOptions { + guard let providerId = tokens.first, !providerId.hasPrefix("--") else { throw CLIParseError.missingValue("PROVIDER") } + var input: URL? + var archiveDirectory = FileManager.default.homeDirectoryForCurrentUser.appendingPathComponent(".icloud-cli/archives") + var budget = CrawlBudget.defaultPolling + var format = OutputFormat.json + var index = 1 + while index < tokens.count { + let token = tokens[index] + switch token { + case "--input": input = try parseURL(after: token, in: tokens, at: &index) + case "--archive-dir": archiveDirectory = try parseURL(after: token, in: tokens, at: &index) + case "--scan-limit": budget = CrawlBudget(scanLimit: try positiveInteger(after: token, in: tokens, at: &index), wallClockLimitMilliseconds: budget.wallClockLimitMilliseconds) + case "--timeout-ms": budget = CrawlBudget(scanLimit: budget.scanLimit, wallClockLimitMilliseconds: try positiveInteger(after: token, in: tokens, at: &index)) + case "--format": format = try parseFormat(after: token, in: tokens, at: &index) + default: throw CLIParseError.unknownCommand(token) + } + index += 1 + } + guard let input else { throw CLIParseError.missingValue("--input") } + return ArchiveSyncOptions(providerId: providerId, input: input, archiveDirectory: archiveDirectory, budget: budget, format: format) + } + + private func parseArchiveStatusOptions(_ tokens: [String]) throws -> ArchiveStatusOptions { + guard let providerId = tokens.first, !providerId.hasPrefix("--") else { throw CLIParseError.missingValue("PROVIDER") } + var options = ArchiveStatusOptions(providerId: providerId) + var index = 1 + while index < tokens.count { + let token = tokens[index] + switch token { + case "--archive-dir": options.archiveDirectory = try parseURL(after: token, in: tokens, at: &index) + case "--format": options.format = try parseFormat(after: token, in: tokens, at: &index) + default: throw CLIParseError.unknownCommand(token) + } + index += 1 + } + return options + } + private func parseSafariBookmarksOptions(_ tokens: [String]) throws -> SafariBookmarksOptions { var options = SafariBookmarksOptions(); var index = 0 while index < tokens.count { @@ -1370,6 +1448,8 @@ icloud-cli \(version) Created by OMT-Global. Usage: + icloud-cli archive sync PROVIDER --input PATH [--archive-dir PATH] [--scan-limit N] [--timeout-ms N] [--format json|text] + icloud-cli archive status PROVIDER [--archive-dir PATH] [--format json|text] icloud-cli providers list [--format json|text] icloud-cli providers external-manifest [--format json|text] icloud-cli snapshot [--include COMMAND,...] [--redaction safe|raw] [--output PATH] [--format json|text] @@ -1433,6 +1513,7 @@ Usage: icloud-cli safari extensions list [--profile NAME] [--format json|text] [--safari-store PATH] Commands: + archive Incrementally sync and inspect private per-provider metadata archives. providers list List the versioned, machine-readable provider capability manifest. providers external-manifest Emit the local OpenClaw control-plane contract. snapshot Emit a conservative redacted operator status payload. diff --git a/Sources/ICloudCLICore/CommandRunner.swift b/Sources/ICloudCLICore/CommandRunner.swift index 88a47e9..49fa4f6 100644 --- a/Sources/ICloudCLICore/CommandRunner.swift +++ b/Sources/ICloudCLICore/CommandRunner.swift @@ -19,6 +19,18 @@ public struct CommandRunner: Sendable { public func run(arguments: [String]) -> Int32 { do { switch try parser.parse(arguments: arguments) { + case .archiveSync(let options): + let values = try options.input.resourceValues(forKeys: [.fileSizeKey]) + if let size = values.fileSize, size > 16 * 1_024 * 1_024 { throw ProviderArchiveError.inputTooLarge(Int64(size)) } + let batch = try JSONDecoder().decode(ArchiveSyncBatch.self, from: Data(contentsOf: options.input)) + guard batch.providerId == options.providerId else { throw ProviderArchiveError.providerMismatch(expected: options.providerId, actual: batch.providerId) } + let result = try ProviderArchiveStore(rootDirectory: options.archiveDirectory).sync(batch, budget: options.budget) + output(try render(result, format: options.format)) + return 0 + case .archiveStatus(let options): + let status = try ProviderArchiveStore(rootDirectory: options.archiveDirectory).status(providerId: options.providerId) + output(try render(status, format: options.format)) + return 0 case .help: output(CLIHelp.root()) return 0 diff --git a/Sources/ICloudCLICore/ProviderArchive.swift b/Sources/ICloudCLICore/ProviderArchive.swift new file mode 100644 index 0000000..ffd466d --- /dev/null +++ b/Sources/ICloudCLICore/ProviderArchive.swift @@ -0,0 +1,434 @@ +import Foundation + +public enum ArchiveFreshness: String, Codable, Equatable, Sendable { + case fresh + case stale + case neverSucceeded = "never-succeeded" +} + +public struct ArchiveFailure: Codable, Equatable, Sendable { + public let code: String + public let guidance: String + + public init(code: String, guidance: String) { + self.code = code + self.guidance = guidance + } +} + +public indirect enum ArchiveValue: Codable, Equatable, Sendable { + case string(String) + case int(Int) + case double(Double) + case bool(Bool) + case null + case array([ArchiveValue]) + case object([String: ArchiveValue]) + + public init(from decoder: Decoder) throws { + let container = try decoder.singleValueContainer() + if container.decodeNil() { self = .null } + else if let value = try? container.decode(Bool.self) { self = .bool(value) } + else if let value = try? container.decode(Int.self) { self = .int(value) } + else if let value = try? container.decode(Double.self) { self = .double(value) } + else if let value = try? container.decode([ArchiveValue].self) { self = .array(value) } + else if let value = try? container.decode([String: ArchiveValue].self) { self = .object(value) } + else { self = .string(try container.decode(String.self)) } + } + + public func encode(to encoder: Encoder) throws { + var container = encoder.singleValueContainer() + switch self { + case .string(let value): try container.encode(value) + case .int(let value): try container.encode(value) + case .double(let value): try container.encode(value) + case .bool(let value): try container.encode(value) + case .null: try container.encodeNil() + case .array(let value): try container.encode(value) + case .object(let value): try container.encode(value) + } + } +} + +public struct ArchiveInputRecord: Codable, Equatable, Sendable { + public let id: String + public let sourceModifiedAt: String? + public let fields: [String: ArchiveValue] + + public init(id: String, sourceModifiedAt: String?, fields: [String: ArchiveValue]) { + self.id = id + self.sourceModifiedAt = sourceModifiedAt + self.fields = fields + } +} + +public struct ArchiveSyncBatch: Codable, Equatable, Sendable { + public let schemaVersion: String + public let providerId: String + public let providerSchemaVersion: String + public let sourceFingerprint: String + public let cursor: String? + public let records: [ArchiveInputRecord] + public let deletedIds: [String] + public let failure: ArchiveFailure? + + public init( + schemaVersion: String, + providerId: String, + providerSchemaVersion: String, + sourceFingerprint: String, + cursor: String?, + records: [ArchiveInputRecord], + deletedIds: [String], + failure: ArchiveFailure? + ) { + self.schemaVersion = schemaVersion + self.providerId = providerId + self.providerSchemaVersion = providerSchemaVersion + self.sourceFingerprint = sourceFingerprint + self.cursor = cursor + self.records = records + self.deletedIds = deletedIds + self.failure = failure + } +} + +public struct ArchivedRecord: Codable, Equatable, Sendable { + public let id: String + public let sourceModifiedAt: String? + public let archivedAt: String + public let tombstonedAt: String? + public let fields: [String: ArchiveValue]? +} + +public struct ProviderArchiveDocument: Codable, Equatable, Sendable { + public let schemaVersion: String + public let providerId: String + public let providerSchemaVersion: String + public let cursor: String? + public let sourceFingerprint: String? + public let lastAttemptAt: String? + public let lastSuccessAt: String? + public let freshness: ArchiveFreshness + public let activeItemCount: Int + public let tombstoneCount: Int + public let totalRecordCount: Int + public let failure: ArchiveFailure? + public let records: [ArchivedRecord] +} + +public struct ArchiveProviderStatus: Codable, Equatable, Sendable { + public let schemaVersion: String + public let providerId: String + public let providerSchemaVersion: String + public let cursor: String? + public let sourceFingerprint: String? + public let lastAttemptAt: String? + public let lastSuccessAt: String? + public let freshness: ArchiveFreshness + public let activeItemCount: Int + public let tombstoneCount: Int + public let totalRecordCount: Int + public let failure: ArchiveFailure? +} + +public struct ArchiveSyncResult: Codable, Equatable, Sendable { + public let schemaVersion: String + public let providerId: String + public let state: CrawlState + public let scannedCount: Int + public let upsertedCount: Int + public let tombstonedCount: Int + public let status: ArchiveProviderStatus +} + +public struct ArchiveRetentionPolicy: Codable, Equatable, Sendable { + public let tombstoneLifetimeSeconds: TimeInterval + public let maximumRecords: Int + + public init(tombstoneLifetimeSeconds: TimeInterval = 30 * 86_400, maximumRecords: Int = 100_000) { + self.tombstoneLifetimeSeconds = max(0, tombstoneLifetimeSeconds) + self.maximumRecords = max(1, maximumRecords) + } +} + +public enum ProviderArchiveError: Error, LocalizedError, Equatable { + case invalidBatchSchema(String) + case invalidProviderId(String) + case providerNotArchivable(String) + case providerMismatch(expected: String, actual: String) + case sensitiveField(String) + case missingArchive(String) + case inputTooLarge(Int64) + case unsupportedArchiveSchema(String) + + public var errorDescription: String? { + switch self { + case .invalidBatchSchema(let schema): return "Unsupported archive sync schema: \(schema)" + case .invalidProviderId(let provider): return "Invalid archive provider id: \(provider)" + case .providerNotArchivable(let provider): return "Provider is not approved for metadata archiving: \(provider)" + case .providerMismatch(let expected, let actual): return "Archive batch provider mismatch: expected \(expected), found \(actual)" + case .sensitiveField(let field): return "Archive batch contains a high-sensitivity field that is not opted in: \(field)" + case .missingArchive(let provider): return "No provider archive found for \(provider)" + case .inputTooLarge(let bytes): return "Archive sync input exceeds the 16 MiB safety limit: \(bytes) bytes" + case .unsupportedArchiveSchema(let schema): return "Unsupported provider archive schema: \(schema)" + } + } +} + +public struct ProviderArchiveStore: Sendable { + public let rootDirectory: URL + public let retention: ArchiveRetentionPolicy + public let freshnessLifetimeSeconds: TimeInterval + private let now: @Sendable () -> Date + private let monotonic: @Sendable () -> TimeInterval + + public init( + rootDirectory: URL = FileManager.default.homeDirectoryForCurrentUser.appendingPathComponent(".icloud-cli/archives"), + retention: ArchiveRetentionPolicy = ArchiveRetentionPolicy(), + freshnessLifetimeSeconds: TimeInterval = 3_600, + now: @escaping @Sendable () -> Date = { Date() }, + monotonic: @escaping @Sendable () -> TimeInterval = { ProcessInfo.processInfo.systemUptime } + ) { + self.rootDirectory = rootDirectory + self.retention = retention + self.freshnessLifetimeSeconds = max(1, freshnessLifetimeSeconds) + self.now = now + self.monotonic = monotonic + } + + public func sync(_ batch: ArchiveSyncBatch, budget: CrawlBudget) throws -> ArchiveSyncResult { + try validate(batch) + try ensureRootDirectory() + let attemptedAt = now() + let attemptedAtString = iso8601(attemptedAt) + let startedAt = monotonic() + var document = try loadOrCreate(batch: batch) + var records = Dictionary(uniqueKeysWithValues: document.records.map { ($0.id, $0) }) + var scannedCount = 0 + var upsertedCount = 0 + var tombstonedCount = 0 + var budgetFailure: ArchiveFailure? + + for record in batch.records.sorted(by: { $0.id < $1.id }) { + guard withinBudget(scannedCount: scannedCount, startedAt: startedAt, budget: budget) else { + budgetFailure = budgetExceededFailure(startedAt: startedAt, budget: budget) + break + } + scannedCount += 1 + if let existing = records[record.id], existing.tombstonedAt == nil, + existing.sourceModifiedAt == record.sourceModifiedAt, existing.fields == record.fields { + continue + } + records[record.id] = ArchivedRecord(id: record.id, sourceModifiedAt: record.sourceModifiedAt, archivedAt: attemptedAtString, tombstonedAt: nil, fields: record.fields) + upsertedCount += 1 + } + + if budgetFailure == nil { + for id in Array(Set(batch.deletedIds)).sorted() { + guard withinBudget(scannedCount: scannedCount, startedAt: startedAt, budget: budget) else { + budgetFailure = budgetExceededFailure(startedAt: startedAt, budget: budget) + break + } + scannedCount += 1 + guard let existing = records[id], existing.tombstonedAt == nil else { continue } + records[id] = ArchivedRecord(id: id, sourceModifiedAt: existing.sourceModifiedAt, archivedAt: existing.archivedAt, tombstonedAt: attemptedAtString, fields: nil) + tombstonedCount += 1 + } + } + + let failure = batch.failure.map(redactedFailure) ?? budgetFailure + let completed = failure == nil + let retained = applyRetention(Array(records.values), now: attemptedAt) + let lastSuccessAt = completed ? attemptedAtString : document.lastSuccessAt + let activeItemCount = retained.filter { $0.tombstonedAt == nil }.count + document = ProviderArchiveDocument( + schemaVersion: "icloud-cli.archive.v1", + providerId: batch.providerId, + providerSchemaVersion: batch.providerSchemaVersion, + cursor: completed ? batch.cursor : document.cursor, + sourceFingerprint: completed ? batch.sourceFingerprint : document.sourceFingerprint, + lastAttemptAt: attemptedAtString, + lastSuccessAt: lastSuccessAt, + freshness: freshness(lastSuccessAt: lastSuccessAt, now: attemptedAt), + activeItemCount: activeItemCount, + tombstoneCount: retained.count - activeItemCount, + totalRecordCount: retained.count, + failure: failure, + records: retained + ) + try write(document) + return ArchiveSyncResult( + schemaVersion: "icloud-cli.archive-sync-result.v1", + providerId: batch.providerId, + state: completed ? .complete : .partial, + scannedCount: scannedCount, + upsertedCount: upsertedCount, + tombstonedCount: tombstonedCount, + status: status(for: document, now: attemptedAt) + ) + } + + public func read(providerId: String) throws -> ProviderArchiveDocument { + try validateProviderId(providerId) + let url = fileURL(providerId: providerId) + guard FileManager.default.fileExists(atPath: url.path) else { throw ProviderArchiveError.missingArchive(providerId) } + let data = try Data(contentsOf: url) + if let document = try? JSONDecoder().decode(ProviderArchiveDocument.self, from: data) { + guard document.schemaVersion == "icloud-cli.archive.v1" else { throw ProviderArchiveError.unsupportedArchiveSchema(document.schemaVersion) } + return document + } + let legacy = try JSONDecoder().decode(LegacyArchiveV0.self, from: data) + guard legacy.schemaVersion == "icloud-cli.archive.v0" else { throw ProviderArchiveError.unsupportedArchiveSchema(legacy.schemaVersion) } + let migrated = ProviderArchiveDocument( + schemaVersion: "icloud-cli.archive.v1", + providerId: legacy.providerId, + providerSchemaVersion: legacy.providerSchemaVersion, + cursor: legacy.cursor, + sourceFingerprint: legacy.sourceFingerprint, + lastAttemptAt: legacy.updatedAt, + lastSuccessAt: legacy.updatedAt, + freshness: .stale, + activeItemCount: legacy.records.filter { $0.tombstonedAt == nil }.count, + tombstoneCount: legacy.records.filter { $0.tombstonedAt != nil }.count, + totalRecordCount: legacy.records.count, + failure: nil, + records: legacy.records + ) + try write(migrated) + return migrated + } + + public func status(providerId: String) throws -> ArchiveProviderStatus { + status(for: try read(providerId: providerId), now: now()) + } + + private func validate(_ batch: ArchiveSyncBatch) throws { + guard batch.schemaVersion == "icloud-cli.archive-sync.v1" else { throw ProviderArchiveError.invalidBatchSchema(batch.schemaVersion) } + try validateProviderId(batch.providerId) + guard ProviderRegistry.manifest.providers.first(where: { $0.id == batch.providerId })?.capabilities.contains("archive-metadata") == true else { + throw ProviderArchiveError.providerNotArchivable(batch.providerId) + } + for record in batch.records { + if let field = sensitiveField(in: record.fields) { throw ProviderArchiveError.sensitiveField(field) } + } + } + + private func validateProviderId(_ providerId: String) throws { + let allowed = providerId.unicodeScalars.allSatisfy { CharacterSet.lowercaseLetters.contains($0) || CharacterSet.decimalDigits.contains($0) || $0 == "-" } + guard allowed, !providerId.isEmpty else { throw ProviderArchiveError.invalidProviderId(providerId) } + } + + private func sensitiveField(in fields: [String: ArchiveValue]) -> String? { + let forbidden = Set(["body", "content", "mediadata", "pixeldata", "audiodata", "attachment", "attachments"]) + for (key, value) in fields { + if forbidden.contains(key.lowercased()) { return key } + switch value { + case .object(let nested): if let match = sensitiveField(in: nested) { return match } + case .array(let values): + for case .object(let nested) in values { + if let match = sensitiveField(in: nested) { return match } + } + default: break + } + } + return nil + } + + private func redactedFailure(_ failure: ArchiveFailure) -> ArchiveFailure { + let allowed = failure.code.lowercased().filter { $0.isLetter || $0.isNumber || $0 == "-" } + return ArchiveFailure(code: String(allowed.prefix(64)).isEmpty ? "provider-error" : String(allowed.prefix(64)), guidance: "Retry the provider with a narrower scope or inspect it directly for local diagnostic details.") + } + + private func loadOrCreate(batch: ArchiveSyncBatch) throws -> ProviderArchiveDocument { + if FileManager.default.fileExists(atPath: fileURL(providerId: batch.providerId).path) { + return try read(providerId: batch.providerId) + } + return ProviderArchiveDocument(schemaVersion: "icloud-cli.archive.v1", providerId: batch.providerId, providerSchemaVersion: batch.providerSchemaVersion, cursor: nil, sourceFingerprint: nil, lastAttemptAt: nil, lastSuccessAt: nil, freshness: .neverSucceeded, activeItemCount: 0, tombstoneCount: 0, totalRecordCount: 0, failure: nil, records: []) + } + + private func withinBudget(scannedCount: Int, startedAt: TimeInterval, budget: CrawlBudget) -> Bool { + scannedCount < budget.scanLimit && Int((monotonic() - startedAt) * 1_000) < budget.wallClockLimitMilliseconds + } + + private func budgetExceededFailure(startedAt: TimeInterval, budget: CrawlBudget) -> ArchiveFailure { + let timedOut = Int((monotonic() - startedAt) * 1_000) >= budget.wallClockLimitMilliseconds + return ArchiveFailure( + code: timedOut ? "timeout" : "scan-limit", + guidance: timedOut ? "Increase the explicit sync timeout or reduce the input batch." : "Resume with the remaining records or increase the explicit scan limit." + ) + } + + private func applyRetention(_ records: [ArchivedRecord], now: Date) -> [ArchivedRecord] { + let tombstoneFloor = now.addingTimeInterval(-retention.tombstoneLifetimeSeconds) + return records + .filter { record in + guard let tombstonedAt = record.tombstonedAt, let date = ISO8601DateFormatter().date(from: tombstonedAt) else { return true } + return date >= tombstoneFloor + } + .sorted { ($0.archivedAt, $0.id) > ($1.archivedAt, $1.id) } + .prefix(retention.maximumRecords) + .sorted { $0.id < $1.id } + } + + private func freshness(lastSuccessAt: String?, now: Date) -> ArchiveFreshness { + guard let lastSuccessAt, let date = ISO8601DateFormatter().date(from: lastSuccessAt) else { return .neverSucceeded } + return now.timeIntervalSince(date) <= freshnessLifetimeSeconds ? .fresh : .stale + } + + private func status(for document: ProviderArchiveDocument, now: Date) -> ArchiveProviderStatus { + return ArchiveProviderStatus( + schemaVersion: "icloud-cli.archive-status.v1", + providerId: document.providerId, + providerSchemaVersion: document.providerSchemaVersion, + cursor: document.cursor, + sourceFingerprint: document.sourceFingerprint, + lastAttemptAt: document.lastAttemptAt, + lastSuccessAt: document.lastSuccessAt, + freshness: freshness(lastSuccessAt: document.lastSuccessAt, now: now), + activeItemCount: document.activeItemCount, + tombstoneCount: document.tombstoneCount, + totalRecordCount: document.totalRecordCount, + failure: document.failure + ) + } + + private func ensureRootDirectory() throws { + if !FileManager.default.fileExists(atPath: rootDirectory.path) { + try FileManager.default.createDirectory(at: rootDirectory, withIntermediateDirectories: true, attributes: [.posixPermissions: 0o700]) + } + try FileManager.default.setAttributes([.posixPermissions: 0o700], ofItemAtPath: rootDirectory.path) + var values = URLResourceValues() + values.isExcludedFromBackup = true + var directory = rootDirectory + try? directory.setResourceValues(values) + } + + private func write(_ document: ProviderArchiveDocument) throws { + try ensureRootDirectory() + let encoder = JSONEncoder() + encoder.outputFormatting = [.prettyPrinted, .sortedKeys] + let data = try encoder.encode(document) + let target = fileURL(providerId: document.providerId) + try data.write(to: target, options: [.atomic]) + try FileManager.default.setAttributes([.posixPermissions: 0o600], ofItemAtPath: target.path) + } + + private func fileURL(providerId: String) -> URL { + rootDirectory.appendingPathComponent(providerId).appendingPathExtension("json") + } + + private func iso8601(_ date: Date) -> String { + ISO8601DateFormatter().string(from: date) + } +} + +private struct LegacyArchiveV0: Codable { + let schemaVersion: String + let providerId: String + let providerSchemaVersion: String + let cursor: String? + let sourceFingerprint: String? + let updatedAt: String + let records: [ArchivedRecord] +} diff --git a/Sources/ICloudCLICore/ProviderManifest.swift b/Sources/ICloudCLICore/ProviderManifest.swift index aa36f79..dc9da22 100644 --- a/Sources/ICloudCLICore/ProviderManifest.swift +++ b/Sources/ICloudCLICore/ProviderManifest.swift @@ -32,18 +32,18 @@ public enum ProviderRegistry { ) public static let providers: [ProviderDescriptor] = [ - provider("account", "iCloud Account", .stable, .preferences, .low, ["account status"], ["status"], true), - provider("backup", "iCloud Backup", .stable, .preferences, .moderate, ["backup status"], ["status"], true), + provider("account", "iCloud Account", .stable, .preferences, .low, ["account status"], ["archive-metadata", "status"], true), + provider("backup", "iCloud Backup", .stable, .preferences, .moderate, ["backup status"], ["archive-metadata", "status"], true), provider("books", "Apple Books", .beta, .sqlite, .moderate, ["books collections", "books list"], ["collections", "inventory", "highlights"]), provider("calendar", "Calendar", .beta, .sqlite, .high, ["calendar accounts", "calendar list", "calendar events"], ["accounts", "calendars", "events", "date-filtering"]), provider("contacts", "Contacts", .beta, .sqlite, .high, ["contacts list"], ["inventory", "search"]), - provider("devices", "iCloud Devices", .stable, .preferences, .moderate, ["devices list"], ["inventory", "status"], true), - provider("drive", "iCloud Drive", .stable, .filesystem, .moderate, ["drive containers", "drive errors", "drive list", "drive recents", "drive shared", "drive status"], ["bounded-crawl", "containers", "inventory", "recents", "sharing", "sync-status"], true), + provider("devices", "iCloud Devices", .stable, .preferences, .moderate, ["devices list"], ["archive-metadata", "inventory", "status"], true), + provider("drive", "iCloud Drive", .stable, .filesystem, .moderate, ["drive containers", "drive errors", "drive list", "drive recents", "drive shared", "drive status"], ["archive-metadata", "bounded-crawl", "containers", "inventory", "recents", "sharing", "sync-status"], true), provider("family", "Family Sharing", .stable, .preferences, .high, ["family status"], ["membership", "status"]), provider("findmy", "Find My", .experimental, .sqlite, .high, ["findmy devices", "findmy people"], ["devices", "people", "location"]), - provider("focus", "Focus", .stable, .preferences, .moderate, ["focus status"], ["status"], true), + provider("focus", "Focus", .stable, .preferences, .moderate, ["focus status"], ["archive-metadata", "status"], true), provider("freeform", "Freeform", .beta, .sqlite, .moderate, ["freeform list"], ["inventory", "date-filtering"]), - provider("handoff", "Handoff", .beta, .filesystem, .moderate, ["handoff list"], ["activity", "recents"], true), + provider("handoff", "Handoff", .beta, .filesystem, .moderate, ["handoff list"], ["activity", "archive-metadata", "recents"], true), provider("health", "Health", .experimental, .sqlite, .high, ["health summary"], ["aggregate-summary"]), provider("home", "Home", .beta, .sqlite, .high, ["home accessories", "home homes", "home rooms", "home scenes"], ["accessories", "homes", "rooms", "scenes"]), provider("mail", "Mail", .beta, .sqlite, .high, ["mail accounts", "mail mailboxes", "mail recent"], ["accounts", "headers", "mailboxes"]), @@ -55,9 +55,9 @@ public enum ProviderRegistry { provider("photos", "Photos", .beta, .mixed, .high, ["photos list", "photos screenshots", "photos shared-albums", "photos shared-library"], ["assets", "screenshots", "sharing"]), provider("reminders", "Reminders", .beta, .sqlite, .high, ["reminders assigned", "reminders flagged", "reminders list", "reminders lists", "reminders scheduled", "reminders today"], ["inventory", "lists", "smart-views"]), provider("safari", "Safari", .beta, .mixed, .high, ["safari bookmarks", "safari cloud-tabs list", "safari cloud-tabs probe", "safari extensions list", "safari frequently-visited", "safari history", "safari profiles list", "safari reading-list", "safari tabs"], ["bookmarks", "cloud-tabs", "consistent-snapshot", "extensions", "history", "profiles", "tabs"]), - provider("shortcuts", "Shortcuts", .stable, .filesystem, .moderate, ["shortcuts list"], ["inventory", "search"], true), + provider("shortcuts", "Shortcuts", .stable, .filesystem, .moderate, ["shortcuts list"], ["archive-metadata", "inventory", "search"], true), provider("stocks", "Stocks", .beta, .sqlite, .moderate, ["stocks groups", "stocks watchlist"], ["groups", "watchlist"]), - provider("storage", "iCloud Storage", .stable, .preferences, .low, ["storage status"], ["quota", "status"], true), + provider("storage", "iCloud Storage", .stable, .preferences, .low, ["storage status"], ["archive-metadata", "quota", "status"], true), provider("tags", "Finder Tags", .beta, .mixed, .moderate, ["tags items", "tags list"], ["bounded-crawl", "inventory", "search", "tags"]), provider("voice-memos", "Voice Memos", .beta, .sqlite, .high, ["voice-memos list"], ["folders", "inventory", "date-filtering"]), provider("wallet", "Wallet", .beta, .filesystem, .high, ["wallet passes"], ["inventory", "type-filtering"]), diff --git a/Tests/ICloudCLICoreTests/ProviderArchiveTests.swift b/Tests/ICloudCLICoreTests/ProviderArchiveTests.swift new file mode 100644 index 0000000..f8367a1 --- /dev/null +++ b/Tests/ICloudCLICoreTests/ProviderArchiveTests.swift @@ -0,0 +1,209 @@ +import Foundation +import Testing +@testable import ICloudCLICore + +@Test func archiveSyncPersistsCursorAndResumesIdempotently() throws { + let root = try archiveTestDirectory(named: "resume") + defer { try? FileManager.default.removeItem(at: root) } + let clock = ArchiveTestClock(date: ISO8601DateFormatter().date(from: "2026-07-12T10:00:00Z")!) + let store = ProviderArchiveStore(rootDirectory: root, now: { clock.date }, monotonic: { 0 }) + let batch = archiveBatch(records: [archiveRecord("one", title: "First")], cursor: "cursor-1") + + let first = try store.sync(batch, budget: CrawlBudget(scanLimit: 10, wallClockLimitMilliseconds: 1_000)) + let restarted = ProviderArchiveStore(rootDirectory: root, now: { clock.date }, monotonic: { 0 }) + let second = try restarted.sync(batch, budget: CrawlBudget(scanLimit: 10, wallClockLimitMilliseconds: 1_000)) + let status = try restarted.status(providerId: "drive") + + #expect(first.upsertedCount == 1) + #expect(second.upsertedCount == 0) + #expect(status.cursor == "cursor-1") + #expect(status.sourceFingerprint == "source-v1") + #expect(status.activeItemCount == 1) + #expect(status.lastAttemptAt == "2026-07-12T10:00:00Z") + #expect(status.lastSuccessAt == "2026-07-12T10:00:00Z") +} + +@Test func archiveSyncHandlesExplicitTombstones() throws { + let root = try archiveTestDirectory(named: "tombstone") + defer { try? FileManager.default.removeItem(at: root) } + let clock = ArchiveTestClock(date: Date(timeIntervalSince1970: 1_000)) + let store = ProviderArchiveStore(rootDirectory: root, now: { clock.date }, monotonic: { 0 }) + _ = try store.sync(archiveBatch(records: [archiveRecord("one", title: "First")]), budget: .defaultPolling) + clock.date = Date(timeIntervalSince1970: 2_000) + + let result = try store.sync(archiveBatch(records: [], deletedIds: ["one"], cursor: "cursor-2"), budget: .defaultPolling) + let document = try store.read(providerId: "drive") + + #expect(result.tombstonedCount == 1) + #expect(document.records.count == 1) + #expect(document.records[0].fields == nil) + #expect(document.records[0].tombstonedAt != nil) + #expect(document.activeItemCount == 0) + #expect(document.tombstoneCount == 1) + #expect(try store.status(providerId: "drive").activeItemCount == 0) +} + +@Test func archivePartialFailurePreservesLastSuccessAndCursor() throws { + let root = try archiveTestDirectory(named: "partial") + defer { try? FileManager.default.removeItem(at: root) } + let clock = ArchiveTestClock(date: Date(timeIntervalSince1970: 1_000)) + let store = ProviderArchiveStore(rootDirectory: root, now: { clock.date }, monotonic: { 0 }) + _ = try store.sync(archiveBatch(records: [archiveRecord("one", title: "First")], cursor: "good"), budget: .defaultPolling) + clock.date = Date(timeIntervalSince1970: 2_000) + let failed = archiveBatch(records: [archiveRecord("two", title: "Second")], cursor: "unsafe", failure: ArchiveFailure(code: "source-timeout", guidance: "Retry with a narrower source scope.")) + + let result = try store.sync(failed, budget: .defaultPolling) + let status = try store.status(providerId: "drive") + + #expect(result.state == .partial) + #expect(status.cursor == "good") + #expect(status.lastSuccessAt == ISO8601DateFormatter().string(from: Date(timeIntervalSince1970: 1_000))) + #expect(status.lastAttemptAt == ISO8601DateFormatter().string(from: Date(timeIntervalSince1970: 2_000))) + #expect(status.failure?.code == "source-timeout") + #expect(status.failure?.guidance != "Retry with a narrower source scope.") +} + +@Test func archiveRetentionPurgesExpiredTombstones() throws { + let root = try archiveTestDirectory(named: "retention") + defer { try? FileManager.default.removeItem(at: root) } + let clock = ArchiveTestClock(date: Date(timeIntervalSince1970: 1_000)) + let store = ProviderArchiveStore(rootDirectory: root, retention: ArchiveRetentionPolicy(tombstoneLifetimeSeconds: 100, maximumRecords: 100), now: { clock.date }, monotonic: { 0 }) + _ = try store.sync(archiveBatch(records: [archiveRecord("one", title: "First")]), budget: .defaultPolling) + _ = try store.sync(archiveBatch(records: [], deletedIds: ["one"]), budget: .defaultPolling) + clock.date = Date(timeIntervalSince1970: 1_200) + + _ = try store.sync(archiveBatch(records: [archiveRecord("two", title: "Second")]), budget: .defaultPolling) + + #expect(try store.read(providerId: "drive").records.map(\.id) == ["two"]) +} + +@Test func archiveMigratesV0Documents() throws { + let root = try archiveTestDirectory(named: "migration") + defer { try? FileManager.default.removeItem(at: root) } + try FileManager.default.createDirectory(at: root, withIntermediateDirectories: true) + let legacy = """ + {"schemaVersion":"icloud-cli.archive.v0","providerId":"drive","providerSchemaVersion":"drive.v0","cursor":"legacy","sourceFingerprint":"old","updatedAt":"2026-01-01T00:00:00Z","records":[]} + """ + try Data(legacy.utf8).write(to: root.appendingPathComponent("drive.json")) + let store = ProviderArchiveStore(rootDirectory: root) + + let document = try store.read(providerId: "drive") + + #expect(document.schemaVersion == "icloud-cli.archive.v1") + #expect(document.cursor == "legacy") + #expect(try String(contentsOf: root.appendingPathComponent("drive.json"), encoding: .utf8).contains("icloud-cli.archive.v1")) +} + +@Test func archiveStorageUsesPrivatePermissions() throws { + let root = try archiveTestDirectory(named: "permissions") + defer { try? FileManager.default.removeItem(at: root) } + let store = ProviderArchiveStore(rootDirectory: root) + _ = try store.sync(archiveBatch(records: [archiveRecord("one", title: "First")]), budget: .defaultPolling) + + let directoryMode = try #require((try FileManager.default.attributesOfItem(atPath: root.path)[.posixPermissions] as? NSNumber)?.intValue) + let fileMode = try #require((try FileManager.default.attributesOfItem(atPath: root.appendingPathComponent("drive.json").path)[.posixPermissions] as? NSNumber)?.intValue) + #expect(directoryMode & 0o777 == 0o700) + #expect(fileMode & 0o777 == 0o600) +} + +@Test func archiveRejectsHighSensitivityProvidersWithoutOptIn() throws { + let root = try archiveTestDirectory(named: "sensitive") + defer { try? FileManager.default.removeItem(at: root) } + let store = ProviderArchiveStore(rootDirectory: root) + let batch = ArchiveSyncBatch(schemaVersion: "icloud-cli.archive-sync.v1", providerId: "messages", providerSchemaVersion: "messages.v1", sourceFingerprint: "source", cursor: nil, records: [], deletedIds: [], failure: nil) + + #expect(throws: ProviderArchiveError.providerNotArchivable("messages")) { + try store.sync(batch, budget: .defaultPolling) + } +} + +@Test func archiveRejectsNestedSensitiveFieldsWithoutEmbeddingJSONStrings() throws { + let root = try archiveTestDirectory(named: "nested-sensitive") + defer { try? FileManager.default.removeItem(at: root) } + let record = ArchiveInputRecord(id: "one", sourceModifiedAt: nil, fields: [ + "metadata": .object(["body": .string("private")]), + ]) + + #expect(throws: ProviderArchiveError.sensitiveField("body")) { + try ProviderArchiveStore(rootDirectory: root).sync(archiveBatch(records: [record]), budget: .defaultPolling) + } +} + +@Test func archiveSyncReturnsPartialAtExplicitScanBudget() throws { + let root = try archiveTestDirectory(named: "bounded") + defer { try? FileManager.default.removeItem(at: root) } + let store = ProviderArchiveStore(rootDirectory: root, monotonic: { 0 }) + let batch = archiveBatch(records: [archiveRecord("one", title: "One"), archiveRecord("two", title: "Two")], cursor: "not-safe-yet") + + let result = try store.sync(batch, budget: CrawlBudget(scanLimit: 1, wallClockLimitMilliseconds: 1_000)) + + #expect(result.state == .partial) + #expect(result.scannedCount == 1) + #expect(result.status.cursor == nil) + #expect(result.status.failure?.code == "scan-limit") +} + +@Test func archiveSyncReturnsPartialAtExplicitTimeout() throws { + let root = try archiveTestDirectory(named: "timeout") + defer { try? FileManager.default.removeItem(at: root) } + let clock = ArchiveMonotonicClock(values: [0, 0, 0.1, 0.1]) + let store = ProviderArchiveStore(rootDirectory: root, monotonic: { clock.next() }) + let batch = archiveBatch(records: [archiveRecord("one", title: "One"), archiveRecord("two", title: "Two")]) + + let result = try store.sync(batch, budget: CrawlBudget(scanLimit: 10, wallClockLimitMilliseconds: 50)) + + #expect(result.state == .partial) + #expect(result.status.failure?.code == "timeout") + #expect(result.status.cursor == nil) +} + +@Test func parsesAndRunsArchiveSyncAndStatusCommands() throws { + let root = try archiveTestDirectory(named: "commands") + defer { try? FileManager.default.removeItem(at: root) } + try FileManager.default.createDirectory(at: root, withIntermediateDirectories: true) + let input = root.appendingPathComponent("batch.json") + try JSONEncoder().encode(archiveBatch(records: [archiveRecord("one", title: "One")])).write(to: input) + let archiveDirectory = root.appendingPathComponent("archives") + + let parsed = try CLIParser().parse(arguments: ["icloud-cli", "archive", "sync", "drive", "--input", input.path, "--archive-dir", archiveDirectory.path, "--scan-limit", "10", "--timeout-ms", "500"]) + guard case .archiveSync(let options) = parsed else { Issue.record("Expected archive sync"); return } + #expect(options.providerId == "drive") + #expect(options.budget == CrawlBudget(scanLimit: 10, wallClockLimitMilliseconds: 500)) + + final class Sink: @unchecked Sendable { var output: [String] = []; var errors: [String] = [] } + let sink = Sink() + let runner = CommandRunner(output: { sink.output.append($0) }, errorOutput: { sink.errors.append($0) }) + #expect(runner.run(arguments: ["icloud-cli", "archive", "sync", "drive", "--input", input.path, "--archive-dir", archiveDirectory.path]) == 0) + #expect(runner.run(arguments: ["icloud-cli", "archive", "status", "drive", "--archive-dir", archiveDirectory.path]) == 0) + #expect(sink.errors.isEmpty) + #expect(try JSONDecoder().decode(ArchiveSyncResult.self, from: Data(sink.output[0].utf8)).status.activeItemCount == 1) + #expect(try JSONDecoder().decode(ArchiveProviderStatus.self, from: Data(sink.output[1].utf8)).providerId == "drive") +} + +private final class ArchiveTestClock: @unchecked Sendable { + var date: Date + init(date: Date) { self.date = date } +} + +private final class ArchiveMonotonicClock: @unchecked Sendable { + private var values: [TimeInterval] + private let lock = NSLock() + init(values: [TimeInterval]) { self.values = values } + func next() -> TimeInterval { + lock.lock(); defer { lock.unlock() } + if values.count > 1 { return values.removeFirst() } + return values.first ?? 0 + } +} + +private func archiveBatch(records: [ArchiveInputRecord], deletedIds: [String] = [], cursor: String = "cursor", failure: ArchiveFailure? = nil) -> ArchiveSyncBatch { + ArchiveSyncBatch(schemaVersion: "icloud-cli.archive-sync.v1", providerId: "drive", providerSchemaVersion: "drive.metadata.v1", sourceFingerprint: "source-v1", cursor: cursor, records: records, deletedIds: deletedIds, failure: failure) +} + +private func archiveRecord(_ id: String, title: String) -> ArchiveInputRecord { + ArchiveInputRecord(id: id, sourceModifiedAt: "2026-07-12T09:00:00Z", fields: ["title": .string(title)]) +} + +private func archiveTestDirectory(named name: String) throws -> URL { + FileManager.default.temporaryDirectory.appendingPathComponent("icloud-cli-archive-\(name)-\(UUID().uuidString)") +} diff --git a/docs/privacy.md b/docs/privacy.md index 5731812..3e9fd28 100644 --- a/docs/privacy.md +++ b/docs/privacy.md @@ -120,6 +120,8 @@ Messages and Safari history queries first copy the database and present WAL/SHM Recursive Drive and Finder-tag crawls use explicit scan and wall-clock budgets and return structured partial/timeout metadata rather than hanging. Polling records only redacted failure codes and guidance when a crawl cannot finish. See [crawl-budgets.md](crawl-budgets.md) for the output contract. +Resumable archives are restricted to providers that explicitly declare metadata archiving support. They use private local permissions, bounded sync, tombstones, retention, redacted failure state, and no automatic backup; high-sensitivity bodies and media fail closed. See [provider-archives.md](provider-archives.md) for the contract and encrypted-backup guidance. + High-sensitivity gates: - `icloud-cli safari cloud-tabs list`, `icloud-cli mail recent`, and `icloud-cli health summary` require `--confirm-sensitive`. diff --git a/docs/provider-archives.md b/docs/provider-archives.md new file mode 100644 index 0000000..51563ef --- /dev/null +++ b/docs/provider-archives.md @@ -0,0 +1,26 @@ +# Resumable provider archives + +`icloud-cli archive sync PROVIDER --input BATCH.json` persists a bounded, incremental metadata archive under `~/.icloud-cli/archives` by default. `icloud-cli archive status PROVIDER` returns freshness and health metadata without returning archived records. + +## Contracts + +Archive sync input uses `icloud-cli.archive-sync.v1`. Each batch declares a stable provider id, a provider-owned schema version, a source fingerprint, an optional resume cursor, nested metadata records, explicit deleted ids, and an optional structured failure. Payload fields are JSON values, not JSON serialized into strings. + +Stored documents use `icloud-cli.archive.v1`. The shared envelope owns cursor, fingerprint, attempt/success timestamps, freshness, counts, failure state, tombstones, and retention. Providers continue to own their `providerSchemaVersion` and field meanings. The reader migrates the legacy synthetic `icloud-cli.archive.v0` envelope and writes the migrated v1 document atomically. + +Only providers declaring the `archive-metadata` capability may sync. The initial allowlist is account, backup, devices, Drive, Focus, Handoff, Shortcuts, and storage metadata. High-sensitivity providers and body/media fields fail closed until a later provider issue explicitly opts them in. + +## Sync behavior + +- Upserts are keyed by provider record id and are idempotent when source timestamp and fields are unchanged. +- `deletedIds` create tombstones rather than silently dropping records. +- A cursor and source fingerprint advance only after a complete batch. +- Partial source failures and scan/time budgets preserve the last successful cursor while recording redacted failure code and guidance. +- `--scan-limit` and `--timeout-ms` bound each sync attempt. +- Status reports last attempt, last success, freshness, active/tombstone/total counts, cursor, fingerprint, and structured failure. + +## Storage, retention, and backup + +The archive directory is mode `0700`; documents are mode `0600` and replaced atomically. The CLI requests the macOS backup-exclusion resource flag on the directory, but backup tools and filesystems may not honor it, so operators should explicitly exclude this path. Operators who need disaster recovery should back it up only into an encrypted, access-controlled destination and restore the whole provider document rather than editing cursors manually. + +Tombstones are retained for 30 days by default, and each provider archive is capped at 100,000 records. Retention runs after each sync. Archive code does not log record fields, identifiers, source paths, or failure payloads; errors expose stable codes and operator guidance only. From 17c03d7fa7d21950f7fc247449dfa67ae71f3ed1 Mon Sep 17 00:00:00 2001 From: Hermes Date: Sun, 12 Jul 2026 22:57:08 +0100 Subject: [PATCH 4/4] Add incremental Messages archive search Archive bounded Messages metadata from consistent snapshots with cursors, deduplication, deletion tombstones, source fingerprints, and versioned search. Keep bodies behind explicit retention and confirmation gates and exclude all action capabilities. --- Sources/ICloudCLICore/CommandLine.swift | 70 +++++++++ Sources/ICloudCLICore/CommandRunner.swift | 9 ++ Sources/ICloudCLICore/MessagesArchive.swift | 147 ++++++++++++++++++ Sources/ICloudCLICore/ProviderArchive.swift | 37 ++++- Sources/ICloudCLICore/ProviderManifest.swift | 2 +- .../MessagesArchiveTests.swift | 88 +++++++++++ .../ProviderArchiveTests.swift | 4 +- docs/messages-archive.md | 9 ++ docs/privacy.md | 1 + 9 files changed, 362 insertions(+), 5 deletions(-) create mode 100644 Sources/ICloudCLICore/MessagesArchive.swift create mode 100644 Tests/ICloudCLICoreTests/MessagesArchiveTests.swift create mode 100644 docs/messages-archive.md diff --git a/Sources/ICloudCLICore/CommandLine.swift b/Sources/ICloudCLICore/CommandLine.swift index 9baee1a..b8d926c 100644 --- a/Sources/ICloudCLICore/CommandLine.swift +++ b/Sources/ICloudCLICore/CommandLine.swift @@ -277,6 +277,27 @@ public struct MessagesOptions: Equatable, Sendable { } } +public struct MessagesArchiveOptions: Equatable, Sendable { + public var format: OutputFormat = .json + public var chatDatabase = FileManager.default.homeDirectoryForCurrentUser.appendingPathComponent("Library/Messages/chat.db") + public var archiveDirectory = FileManager.default.homeDirectoryForCurrentUser.appendingPathComponent(".icloud-cli/archives") + public var confirmSensitive = false + public var includeBody = false + public var bodyRetentionDays: Int? + public var limit = 1_000 + public init() {} +} + +public struct MessagesSearchOptions: Equatable, Sendable { + public let query: String + public var format: OutputFormat = .json + public var archiveDirectory = FileManager.default.homeDirectoryForCurrentUser.appendingPathComponent(".icloud-cli/archives") + public var confirmSensitive = false + public var includeBody = false + public var limit = 50 + public init(query: String) { self.query = query } +} + public struct ContactsListOptions: Equatable, Sendable { public var format: OutputFormat public var addressBookDatabase: URL @@ -564,6 +585,8 @@ public enum CLICommand: Equatable, Sendable { case mapsRecents(MapsOptions) case messagesConversations(MessagesOptions) case messagesRecent(MessagesOptions) + case messagesArchive(MessagesArchiveOptions) + case messagesSearch(MessagesSearchOptions) case metadata(MetadataCommand, MetadataOptions) case newsHistory(NewsOptions) case newsTopics(NewsOptions) @@ -843,8 +866,13 @@ public struct CLIParser: Sendable { guard let messagesCommand = tokens.first else { throw CLIParseError.unknownCommand("messages") } tokens.removeFirst() switch messagesCommand { + case "archive": return .messagesArchive(try parseMessagesArchiveOptions(tokens)) case "conversations": return .messagesConversations(try parseMessagesOptions(tokens)) case "recent": return .messagesRecent(try parseMessagesOptions(tokens)) + case "search": + guard let query = tokens.first else { throw CLIParseError.missingValue("query") } + tokens.removeFirst() + return .messagesSearch(try parseMessagesSearchOptions(query: query, tokens: tokens)) default: throw CLIParseError.unknownCommand((["messages", messagesCommand] + tokens).joined(separator: " ")) } } @@ -1258,6 +1286,42 @@ public struct CLIParser: Sendable { return options } + private func parseMessagesArchiveOptions(_ tokens: [String]) throws -> MessagesArchiveOptions { + var options = MessagesArchiveOptions(); var index = 0 + while index < tokens.count { + let token = tokens[index] + switch token { + case "--format": options.format = try parseFormat(after: token, in: tokens, at: &index) + case "--chat-db": options.chatDatabase = try parseURL(after: token, in: tokens, at: &index) + case "--archive-dir": options.archiveDirectory = try parseURL(after: token, in: tokens, at: &index) + case "--confirm-sensitive": options.confirmSensitive = true + case "--include-body": options.includeBody = true + case "--body-retention-days": options.bodyRetentionDays = Int(try value(after: token, in: tokens, at: &index)) + case "--limit": options.limit = Int(try value(after: token, in: tokens, at: &index)) ?? options.limit + default: throw CLIParseError.unknownCommand(token) + } + index += 1 + } + return options + } + + private func parseMessagesSearchOptions(query: String, tokens: [String]) throws -> MessagesSearchOptions { + var options = MessagesSearchOptions(query: query); var index = 0 + while index < tokens.count { + let token = tokens[index] + switch token { + case "--format": options.format = try parseFormat(after: token, in: tokens, at: &index) + case "--archive-dir": options.archiveDirectory = try parseURL(after: token, in: tokens, at: &index) + case "--confirm-sensitive": options.confirmSensitive = true + case "--include-body": options.includeBody = true + case "--limit": options.limit = Int(try value(after: token, in: tokens, at: &index)) ?? options.limit + default: throw CLIParseError.unknownCommand(token) + } + index += 1 + } + return options + } + private func parseContactsListOptions(_ tokens: [String]) throws -> ContactsListOptions { var options = ContactsListOptions(); var index = 0 while index < tokens.count { @@ -1484,6 +1548,8 @@ Usage: icloud-cli mail recent [--confirm-sensitive] [--account NAME] [--mailbox NAME] [--limit N] [--format json|text] [--mail-store PATH] icloud-cli messages conversations [--limit N] [--format json|text] [--chat-db PATH] icloud-cli messages recent [--confirm-sensitive] [--include-body] [--since ISO8601] [--limit N] [--format json|text] [--chat-db PATH] + icloud-cli messages archive --confirm-sensitive [--include-body --body-retention-days N] [--limit N] [--format json|text] [--chat-db PATH] [--archive-dir PATH] + icloud-cli messages search QUERY [--include-body --confirm-sensitive] [--limit N] [--format json|text] [--archive-dir PATH] icloud-cli maps favorites [--format json|text] [--maps-store PATH] icloud-cli maps recents [--limit N] [--format json|text] [--maps-store PATH] icloud-cli news history [--since ISO8601] [--limit N] [--format json|text] [--news-store PATH] @@ -1544,6 +1610,10 @@ Commands: List message conversation metadata without message bodies. messages recent List recent messages only after --confirm-sensitive. + messages archive + Incrementally archive bounded message metadata from a consistent snapshot. + messages search + Search the local Messages archive; bodies remain separately gated. maps favorites List Maps saved-place metadata. maps recents List Maps recent-place metadata. news history List News reading-history metadata. diff --git a/Sources/ICloudCLICore/CommandRunner.swift b/Sources/ICloudCLICore/CommandRunner.swift index 49fa4f6..ed50325 100644 --- a/Sources/ICloudCLICore/CommandRunner.swift +++ b/Sources/ICloudCLICore/CommandRunner.swift @@ -91,6 +91,15 @@ public struct CommandRunner: Sendable { let messages = try LocalSQLiteInventoryReader(database: options.chatDatabase).recentMessages(confirmSensitive: options.confirmSensitive, includeBody: options.includeBody, since: options.since, limit: options.limit) output(try render(messages, format: options.format)) return 0 + case .messagesArchive(let options): + guard options.confirmSensitive else { throw LocalInventoryError.sensitiveConfirmationRequired("icloud-cli messages archive") } + let result = try MessagesArchiveAdapter(archiveDirectory: options.archiveDirectory).sync(database: options.chatDatabase, includeBodies: options.includeBody, bodyRetentionDays: options.bodyRetentionDays, limit: options.limit) + output(try render(result, format: options.format)) + return 0 + case .messagesSearch(let options): + let hits = try MessagesArchiveAdapter(archiveDirectory: options.archiveDirectory).search(query: options.query, includeBodies: options.includeBody, confirmSensitive: options.confirmSensitive, limit: options.limit) + output(try render(hits, format: options.format)) + return 0 case .metadata(let command, let options): output(try runMetadata(command: command, options: options)) return 0 diff --git a/Sources/ICloudCLICore/MessagesArchive.swift b/Sources/ICloudCLICore/MessagesArchive.swift new file mode 100644 index 0000000..147e28d --- /dev/null +++ b/Sources/ICloudCLICore/MessagesArchive.swift @@ -0,0 +1,147 @@ +import Foundation + +public enum MessagesArchiveError: Error, LocalizedError, Equatable { + case bodyRetentionRequired + case sensitiveConfirmationRequired + + public var errorDescription: String? { + switch self { + case .bodyRetentionRequired: "Archiving message bodies requires --body-retention-days with a positive bounded value." + case .sensitiveConfirmationRequired: "Searching archived message bodies requires --confirm-sensitive." + } + } +} + +public struct MessagesArchiveHit: Codable, Equatable, Sendable { + public let schemaVersion: String + public let messageId: String + public let chatIdentifier: String + public let sender: String? + public let sentAt: String? + public let isFromMe: Bool + public let body: String? +} + +private struct MessagesArchiveRow: Decodable { + let messageId: String + let chatIdentifier: String + let sender: String? + let sentAt: String? + let isFromMe: Int + let body: String? +} + +public struct MessagesArchiveAdapter: Sendable { + public let archiveDirectory: URL + private let now: @Sendable () -> Date + + public init(archiveDirectory: URL = FileManager.default.homeDirectoryForCurrentUser.appendingPathComponent(".icloud-cli/archives"), now: @escaping @Sendable () -> Date = { Date() }) { + self.archiveDirectory = archiveDirectory + self.now = now + } + + public func sync(database: URL, includeBodies: Bool, bodyRetentionDays: Int?, limit: Int) throws -> ArchiveSyncResult { + if includeBodies && (bodyRetentionDays == nil || bodyRetentionDays! <= 0 || bodyRetentionDays! > 365) { + throw MessagesArchiveError.bodyRetentionRequired + } + let store = ProviderArchiveStore( + rootDirectory: archiveDirectory, + retention: ArchiveRetentionPolicy(tombstoneLifetimeSeconds: TimeInterval((bodyRetentionDays ?? 30) * 86_400), maximumRecords: 100_000) + ) + let previous = try? store.read(providerId: "messages") + let rows = try readRows(database: database, after: previous?.cursor, includeBodies: includeBodies, limit: limit) + let currentIds = try readAllIds(database: database, limit: limit + 1) + var records = rows.map { row in + var fields: [String: ArchiveValue] = [ + "chatIdentifier": .string(row.chatIdentifier), + "sender": row.sender.map(ArchiveValue.string) ?? .null, + "sentAt": row.sentAt.map(ArchiveValue.string) ?? .null, + "isFromMe": .bool(row.isFromMe != 0), + ] + if includeBodies { fields["body"] = row.body.map(ArchiveValue.string) ?? .null } + return ArchiveInputRecord(id: row.messageId, sourceModifiedAt: row.sentAt, fields: fields) + } + let expiry = bodyRetentionDays.map { now().addingTimeInterval(-TimeInterval($0 * 86_400)) } + for record in previous?.records ?? [] where record.tombstonedAt == nil { + guard var fields = record.fields, fields["body"] != nil else { continue } + let bodyExpired = !includeBodies || expiry.map { cutoff in + ISO8601DateFormatter().date(from: record.archivedAt).map { archivedAt in archivedAt < cutoff } ?? true + } == true + if bodyExpired { + fields.removeValue(forKey: "body") + records.append(ArchiveInputRecord(id: record.id, sourceModifiedAt: record.sourceModifiedAt, fields: fields)) + } + } + let cursor = rows.compactMap(\.sentAt).max() ?? previous?.cursor + let values = try database.resourceValues(forKeys: [.fileSizeKey, .contentModificationDateKey]) + let fingerprint = "size:\(values.fileSize ?? 0);modified:\(values.contentModificationDate?.timeIntervalSince1970 ?? 0)" + let archivedIds = Set(previous?.records.filter { $0.tombstonedAt == nil }.map(\.id) ?? []) + let deletedIds = currentIds.count <= limit ? Array(archivedIds.subtracting(currentIds)) : [] + return try store.sync( + ArchiveSyncBatch(schemaVersion: "icloud-cli.archive-sync.v1", providerId: "messages", providerSchemaVersion: "icloud-cli.messages-archive.v1", sourceFingerprint: fingerprint, cursor: cursor, records: records, deletedIds: deletedIds, failure: nil, sensitiveFields: includeBodies ? ["body"] : nil), + budget: CrawlBudget(scanLimit: min(max(limit, 1), 10_000), wallClockLimitMilliseconds: 10_000) + ) + } + + public func search(query: String, includeBodies: Bool, confirmSensitive: Bool = false, limit: Int) throws -> [MessagesArchiveHit] { + if includeBodies && !confirmSensitive { throw MessagesArchiveError.sensitiveConfirmationRequired } + let needle = query.lowercased() + return try ProviderArchiveStore(rootDirectory: archiveDirectory).read(providerId: "messages").records + .filter { $0.tombstonedAt == nil } + .compactMap { record -> MessagesArchiveHit? in + guard let fields = record.fields, case .string(let chatIdentifier) = fields["chatIdentifier"] else { return nil } + let sender = fields["sender"]?.stringValue + let sentAt = fields["sentAt"]?.stringValue + let isFromMe = fields["isFromMe"]?.boolValue ?? false + let body = includeBodies ? fields["body"]?.stringValue : nil + let searchable = [record.id, chatIdentifier, sender, body].compactMap { $0 }.joined(separator: " ").lowercased() + guard needle.isEmpty || searchable.contains(needle) else { return nil } + return MessagesArchiveHit(schemaVersion: "icloud-cli.messages-search.v1", messageId: record.id, chatIdentifier: chatIdentifier, sender: sender, sentAt: sentAt, isFromMe: isFromMe, body: body) + } + .sorted { ($0.sentAt ?? "", $0.messageId) > ($1.sentAt ?? "", $1.messageId) } + .prefix(min(max(limit, 1), 1_000)) + .map { $0 } + } + + private func readRows(database: URL, after cursor: String?, includeBodies: Bool, limit: Int) throws -> [MessagesArchiveRow] { + let engine = SQLiteSnapshotQueryEngine(source: database) + let floor = cursor.map { "WHERE sentAt > '\(sqlLiteral($0))'" } ?? "" + let body = includeBodies ? "body" : "NULL AS body" + do { + return try engine.query("SELECT messageId, chatIdentifier, sender, sentAt, isFromMe, \(body) FROM recent_messages \(floor) ORDER BY sentAt ASC LIMIT \(min(max(limit, 1), 10000));") + } catch LocalInventoryError.unsupportedSchema { + throw LocalInventoryError.unsupportedSchema(store: database.path, detail: "unsupported Messages schema") + } catch { + let appleFloor = cursor.map { "WHERE m.date > \(Int64($0) ?? 0)" } ?? "" + let appleBody = includeBodies ? "m.text" : "NULL" + return try engine.query(""" + SELECT CAST(m.ROWID AS TEXT) AS messageId, + COALESCE(c.chat_identifier, c.guid, h.id, 'unknown') AS chatIdentifier, + h.id AS sender, CAST(m.date AS TEXT) AS sentAt, + COALESCE(m.is_from_me, 0) AS isFromMe, \(appleBody) AS body + FROM message m + LEFT JOIN handle h ON h.ROWID = m.handle_id + LEFT JOIN chat_message_join cmj ON cmj.message_id = m.ROWID + LEFT JOIN chat c ON c.ROWID = cmj.chat_id + \(appleFloor) ORDER BY m.date ASC LIMIT \(min(max(limit, 1), 10000)); + """) + } + } + + private func readAllIds(database: URL, limit: Int) throws -> Set { + struct Identifier: Decodable { let id: String } + let engine = SQLiteSnapshotQueryEngine(source: database) + if let rows: [Identifier] = try? engine.query("SELECT messageId AS id FROM recent_messages ORDER BY messageId LIMIT \(limit);") { + return Set(rows.map(\.id)) + } + let rows: [Identifier] = try engine.query("SELECT CAST(ROWID AS TEXT) AS id FROM message ORDER BY ROWID LIMIT \(limit);") + return Set(rows.map(\.id)) + } + + private func sqlLiteral(_ value: String) -> String { value.replacingOccurrences(of: "'", with: "''") } +} + +private extension ArchiveValue { + var stringValue: String? { if case .string(let value) = self { value } else { nil } } + var boolValue: Bool? { if case .bool(let value) = self { value } else { nil } } +} diff --git a/Sources/ICloudCLICore/ProviderArchive.swift b/Sources/ICloudCLICore/ProviderArchive.swift index ffd466d..197c84a 100644 --- a/Sources/ICloudCLICore/ProviderArchive.swift +++ b/Sources/ICloudCLICore/ProviderArchive.swift @@ -71,6 +71,36 @@ public struct ArchiveSyncBatch: Codable, Equatable, Sendable { public let records: [ArchiveInputRecord] public let deletedIds: [String] public let failure: ArchiveFailure? + public let sensitiveFields: [String]? + + private enum CodingKeys: String, CodingKey { + case schemaVersion, providerId, providerSchemaVersion, sourceFingerprint, cursor, records, deletedIds, failure + } + + public init(from decoder: Decoder) throws { + let values = try decoder.container(keyedBy: CodingKeys.self) + schemaVersion = try values.decode(String.self, forKey: .schemaVersion) + providerId = try values.decode(String.self, forKey: .providerId) + providerSchemaVersion = try values.decode(String.self, forKey: .providerSchemaVersion) + sourceFingerprint = try values.decode(String.self, forKey: .sourceFingerprint) + cursor = try values.decodeIfPresent(String.self, forKey: .cursor) + records = try values.decode([ArchiveInputRecord].self, forKey: .records) + deletedIds = try values.decode([String].self, forKey: .deletedIds) + failure = try values.decodeIfPresent(ArchiveFailure.self, forKey: .failure) + sensitiveFields = nil + } + + public func encode(to encoder: Encoder) throws { + var values = encoder.container(keyedBy: CodingKeys.self) + try values.encode(schemaVersion, forKey: .schemaVersion) + try values.encode(providerId, forKey: .providerId) + try values.encode(providerSchemaVersion, forKey: .providerSchemaVersion) + try values.encode(sourceFingerprint, forKey: .sourceFingerprint) + try values.encodeIfPresent(cursor, forKey: .cursor) + try values.encode(records, forKey: .records) + try values.encode(deletedIds, forKey: .deletedIds) + try values.encodeIfPresent(failure, forKey: .failure) + } public init( schemaVersion: String, @@ -80,7 +110,8 @@ public struct ArchiveSyncBatch: Codable, Equatable, Sendable { cursor: String?, records: [ArchiveInputRecord], deletedIds: [String], - failure: ArchiveFailure? + failure: ArchiveFailure?, + sensitiveFields: [String]? = nil ) { self.schemaVersion = schemaVersion self.providerId = providerId @@ -90,6 +121,7 @@ public struct ArchiveSyncBatch: Codable, Equatable, Sendable { self.records = records self.deletedIds = deletedIds self.failure = failure + self.sensitiveFields = sensitiveFields } } @@ -309,8 +341,9 @@ public struct ProviderArchiveStore: Sendable { guard ProviderRegistry.manifest.providers.first(where: { $0.id == batch.providerId })?.capabilities.contains("archive-metadata") == true else { throw ProviderArchiveError.providerNotArchivable(batch.providerId) } + let approvedSensitive = Set(batch.sensitiveFields ?? []) for record in batch.records { - if let field = sensitiveField(in: record.fields) { throw ProviderArchiveError.sensitiveField(field) } + if let field = sensitiveField(in: record.fields), !(batch.providerId == "messages" && approvedSensitive.contains(field)) { throw ProviderArchiveError.sensitiveField(field) } } } diff --git a/Sources/ICloudCLICore/ProviderManifest.swift b/Sources/ICloudCLICore/ProviderManifest.swift index dc9da22..9ced583 100644 --- a/Sources/ICloudCLICore/ProviderManifest.swift +++ b/Sources/ICloudCLICore/ProviderManifest.swift @@ -48,7 +48,7 @@ public enum ProviderRegistry { provider("home", "Home", .beta, .sqlite, .high, ["home accessories", "home homes", "home rooms", "home scenes"], ["accessories", "homes", "rooms", "scenes"]), provider("mail", "Mail", .beta, .sqlite, .high, ["mail accounts", "mail mailboxes", "mail recent"], ["accounts", "headers", "mailboxes"]), provider("maps", "Maps", .beta, .sqlite, .high, ["maps favorites", "maps recents"], ["favorites", "recents", "location"]), - provider("messages", "Messages", .beta, .sqlite, .high, ["messages conversations", "messages recent"], ["consistent-snapshot", "conversations", "messages", "date-filtering"]), + provider("messages", "Messages", .beta, .sqlite, .high, ["messages archive", "messages conversations", "messages recent", "messages search"], ["archive-metadata", "consistent-snapshot", "conversations", "date-filtering", "incremental", "messages", "search"]), provider("music", "Music", .beta, .sqlite, .moderate, ["music playlists", "music status", "music tracks"], ["inventory", "playlists", "status"]), provider("news", "News", .beta, .sqlite, .moderate, ["news history", "news topics"], ["history", "topics", "date-filtering"]), provider("notes", "Notes", .beta, .sqlite, .high, ["notes accounts", "notes folders", "notes list", "notes shared", "notes tags"], ["accounts", "folders", "inventory", "sharing", "tags"]), diff --git a/Tests/ICloudCLICoreTests/MessagesArchiveTests.swift b/Tests/ICloudCLICoreTests/MessagesArchiveTests.swift new file mode 100644 index 0000000..3a50d51 --- /dev/null +++ b/Tests/ICloudCLICoreTests/MessagesArchiveTests.swift @@ -0,0 +1,88 @@ +import Foundation +import Testing +@testable import ICloudCLICore + +@Test func messagesArchiveIsIncrementalDeduplicatedAndPreservesGroupChats() throws { + let root = try messagesArchiveFixture("incremental") + defer { try? FileManager.default.removeItem(at: root) } + let database = root.appendingPathComponent("chat.db") + try makeMessagesArchiveDatabase(database) + let archive = root.appendingPathComponent("archive") + let adapter = MessagesArchiveAdapter(archiveDirectory: archive) + + let first = try adapter.sync(database: database, includeBodies: false, bodyRetentionDays: nil, limit: 100) + let second = try adapter.sync(database: database, includeBodies: false, bodyRetentionDays: nil, limit: 100) + + #expect(first.status.activeItemCount == 2) + #expect(second.upsertedCount == 0) + #expect(second.status.cursor == "200") + let hits = try adapter.search(query: "group", includeBodies: false, limit: 10) + #expect(hits.map(\.chatIdentifier) == ["chat-group"]) + #expect(hits.first?.body == nil) + + let process = Process(); process.executableURL = URL(fileURLWithPath: "/usr/bin/sqlite3"); process.arguments = [database.path, "DELETE FROM recent_messages WHERE messageId='m1';"] + try process.run(); process.waitUntilExit() + let deleted = try adapter.sync(database: database, includeBodies: false, bodyRetentionDays: nil, limit: 100) + #expect(deleted.tombstonedCount == 1) +} + +@Test func messagesArchiveBodiesRequireExplicitRetentionAndSearchConfirmation() throws { + let root = try messagesArchiveFixture("bodies") + defer { try? FileManager.default.removeItem(at: root) } + let database = root.appendingPathComponent("chat.db") + try makeMessagesArchiveDatabase(database) + let adapter = MessagesArchiveAdapter(archiveDirectory: root.appendingPathComponent("archive")) + + #expect(throws: MessagesArchiveError.bodyRetentionRequired) { + try adapter.sync(database: database, includeBodies: true, bodyRetentionDays: nil, limit: 10) + } + _ = try adapter.sync(database: database, includeBodies: true, bodyRetentionDays: 7, limit: 10) + #expect(throws: MessagesArchiveError.sensitiveConfirmationRequired) { + try adapter.search(query: "secret", includeBodies: true, confirmSensitive: false, limit: 10) + } + #expect(try adapter.search(query: "secret", includeBodies: true, confirmSensitive: true, limit: 10).count == 1) + + let future = MessagesArchiveAdapter(archiveDirectory: root.appendingPathComponent("archive"), now: { Date(timeIntervalSinceNow: 8 * 86_400) }) + _ = try future.sync(database: database, includeBodies: true, bodyRetentionDays: 7, limit: 10) + #expect(try future.search(query: "secret", includeBodies: true, confirmSensitive: true, limit: 10).isEmpty) +} + +@Test func parsesMessagesArchiveAndSearchCommands() throws { + let archive = try CLIParser().parse(arguments: ["icloud-cli", "messages", "archive", "--confirm-sensitive", "--include-body", "--body-retention-days", "7", "--limit", "50"]) + guard case .messagesArchive(let options) = archive else { Issue.record("Expected messages archive"); return } + #expect(options.confirmSensitive && options.includeBody && options.bodyRetentionDays == 7 && options.limit == 50) + + let search = try CLIParser().parse(arguments: ["icloud-cli", "messages", "search", "group", "--limit", "5"]) + guard case .messagesSearch(let options) = search else { Issue.record("Expected messages search"); return } + #expect(options.query == "group" && options.limit == 5) +} + +@Test func externalArchiveBatchCannotSelfAuthorizeMessageBodies() throws { + let data = Data(""" + {"schemaVersion":"icloud-cli.archive-sync.v1","providerId":"messages","providerSchemaVersion":"messages.v1","sourceFingerprint":"source","records":[{"id":"m1","fields":{"body":"secret"}}],"deletedIds":[],"sensitiveFields":["body"]} + """.utf8) + let batch = try JSONDecoder().decode(ArchiveSyncBatch.self, from: data) + #expect(batch.sensitiveFields == nil) + let root = try messagesArchiveFixture("untrusted-batch") + defer { try? FileManager.default.removeItem(at: root) } + #expect(throws: ProviderArchiveError.sensitiveField("body")) { + try ProviderArchiveStore(rootDirectory: root).sync(batch, budget: .defaultPolling) + } +} + +private func messagesArchiveFixture(_ name: String) throws -> URL { + let url = FileManager.default.temporaryDirectory.appendingPathComponent("icloud-cli-messages-\(name)-\(UUID().uuidString)") + try FileManager.default.createDirectory(at: url, withIntermediateDirectories: true) + return url +} + +private func makeMessagesArchiveDatabase(_ url: URL) throws { + let sql = """ + CREATE TABLE recent_messages (messageId TEXT, chatIdentifier TEXT, sender TEXT, sentAt TEXT, isFromMe INTEGER, body TEXT); + INSERT INTO recent_messages VALUES ('m1', 'chat-direct', '+15550001', '100', 0, 'ordinary'); + INSERT INTO recent_messages VALUES ('m2', 'chat-group', '+15550002', '200', 1, 'group secret'); + """ + let process = Process(); process.executableURL = URL(fileURLWithPath: "/usr/bin/sqlite3"); process.arguments = [url.path, sql] + try process.run(); process.waitUntilExit() + #expect(process.terminationStatus == 0) +} diff --git a/Tests/ICloudCLICoreTests/ProviderArchiveTests.swift b/Tests/ICloudCLICoreTests/ProviderArchiveTests.swift index f8367a1..4fd4f45 100644 --- a/Tests/ICloudCLICoreTests/ProviderArchiveTests.swift +++ b/Tests/ICloudCLICoreTests/ProviderArchiveTests.swift @@ -110,9 +110,9 @@ import Testing let root = try archiveTestDirectory(named: "sensitive") defer { try? FileManager.default.removeItem(at: root) } let store = ProviderArchiveStore(rootDirectory: root) - let batch = ArchiveSyncBatch(schemaVersion: "icloud-cli.archive-sync.v1", providerId: "messages", providerSchemaVersion: "messages.v1", sourceFingerprint: "source", cursor: nil, records: [], deletedIds: [], failure: nil) + let batch = ArchiveSyncBatch(schemaVersion: "icloud-cli.archive-sync.v1", providerId: "contacts", providerSchemaVersion: "contacts.v1", sourceFingerprint: "source", cursor: nil, records: [], deletedIds: [], failure: nil) - #expect(throws: ProviderArchiveError.providerNotArchivable("messages")) { + #expect(throws: ProviderArchiveError.providerNotArchivable("contacts")) { try store.sync(batch, budget: .defaultPolling) } } diff --git a/docs/messages-archive.md b/docs/messages-archive.md new file mode 100644 index 0000000..4240c7f --- /dev/null +++ b/docs/messages-archive.md @@ -0,0 +1,9 @@ +# Messages archive and search + +The Messages provider uses an internal, read-only adapter rather than delegating to `imsgcrawl` or `imsg`. This keeps the versioned output, snapshot, archive, retention, and privacy contracts in one executable while leaving send, react, and other actions outside the crawler boundary. It never loads private frameworks, injects a library, disables SIP, or modifies Apple's database. + +`messages archive --confirm-sensitive` copies `chat.db` and any present WAL/SHM companions into private temporary storage through the shared SQLite snapshot engine. It reads a bounded batch in ascending message-date order, persists stable message and conversation identifiers, advances a cursor only after a successful archive sync, and stores a source size/modification fingerprint. Repeated batches deduplicate by message id and fields. A complete bounded identifier scan creates tombstones for disappeared messages; when the source exceeds the explicit limit, deletion inference is skipped rather than risking false tombstones. Group chats retain their chat identifier. + +Bodies are omitted by default. `--include-body` is accepted only with `--body-retention-days 1...365`; searching bodies also requires `--confirm-sensitive`. Metadata search returns `icloud-cli.messages-search.v1` rows and never exposes bodies unless both flags are present. The shared archive remains local, private, bounded, and excluded from backup on a best-effort basis. + +The Apple Messages schema is private and varies by macOS release. The adapter supports the current public fixture shape and the established `message`, `chat`, `chat_message_join`, and `handle` tables, failing closed when neither shape is present. Attachments and attributed-body blobs are unsupported and never archived. diff --git a/docs/privacy.md b/docs/privacy.md index 3e9fd28..b74cfd5 100644 --- a/docs/privacy.md +++ b/docs/privacy.md @@ -55,6 +55,7 @@ OpenClaw integrations should default to local retention. Exporting raw browsing | `icloud-cli reminders list` / `reminders lists` | Local Reminders metadata under `~/Library/Reminders` by default | Normal user file access or Full Disk Access depending on macOS privacy posture. Reminder titles and notes are sensitive; logs should summarize list counts. | | `icloud-cli safari history` | `~/Library/Safari/History.db` | Full Disk Access is expected. Requires `--confirm-sensitive`; `--redact-urls` is recommended for automated export. This command is not part of default watch polling. | | `icloud-cli messages conversations` / `messages recent` | `~/Library/Messages/chat.db` | Full Disk Access is expected. Recent message reads require `--confirm-sensitive`; bodies require `--include-body`. Message commands are not part of default watch polling. | +| `icloud-cli messages archive` / `messages search` | A consistent private snapshot of `chat.db`, then `~/.icloud-cli/archives/messages.json` | Archive creation requires `--confirm-sensitive`. Bodies require both `--include-body` and an explicit bounded retention period; body search additionally requires `--confirm-sensitive`. The archive remains local with mode `0600`. | | `icloud-cli contacts list` | Local AddressBook SQLite metadata under `~/Library/Application Support/AddressBook` by default | Contacts permission or Full Disk Access may be needed. Notes are omitted unless `--include-notes` is passed. | | `icloud-cli maps favorites` / `maps recents` | Local Maps cache under `~/Library/Containers/com.apple.Maps` by default | Location data is sensitive. Home/work categories are high-sensitivity and logs must omit coordinates. | | `icloud-cli news history` / `news topics` | Local News cache under `~/Library/Containers/com.apple.news` by default | Reading history is interest-graph data. Logs should summarize source/topic counts rather than URLs. |