Snapshot live SQLite stores before querying

Sources/ICloudCLICore/LocalInventories.swift

@@ -4,6 +4,8 @@ public enum LocalInventoryError: Error, LocalizedError, Equatable {
     case missingRoot(String)
     case missingStore(String)
     case permissionDenied(String)
+    case lockedStore(String)
+    case queryTimeout(String)
     case sensitiveConfirmationRequired(String)
     case unsupportedSchema(store: String, detail: String)
     case sqliteFailure(String)
@@ -14,6 +16,8 @@ public enum LocalInventoryError: Error, LocalizedError, Equatable {
         case .missingStore(let path): return "Inventory store not available: \(path)"
         case .permissionDenied(let path):
             return "Permission denied reading local inventory store: \(path). Grant Full Disk Access to the calling terminal or agent process, then retry."
+        case .lockedStore(let path): return "Local inventory store is locked or busy: \(path)"
+        case .queryTimeout(let path): return "Timed out querying local inventory store: \(path)"
         case .sensitiveConfirmationRequired(let command):
             return "\(command) reads high-sensitivity local data; rerun with --confirm-sensitive"
         case .unsupportedSchema(let store, let detail):
@@ -247,9 +251,22 @@ public struct NewsTopicEntry: Codable, Equatable, Sendable {
 
 public struct LocalSQLiteInventoryReader: Sendable {
     public let database: URL
+    private let snapshotsLiveStores: Bool
+    private let reportedStore: String
+    private let snapshotWorkspace: URL?
 
     public init(database: URL) {
         self.database = database
+        self.snapshotsLiveStores = true
+        self.reportedStore = database.path
+        self.snapshotWorkspace = nil
+    }
+
+    private init(database: URL, snapshotsLiveStores: Bool, reportedStore: String, snapshotWorkspace: URL?) {
+        self.database = database
+        self.snapshotsLiveStores = snapshotsLiveStores
+        self.reportedStore = reportedStore
+        self.snapshotWorkspace = snapshotWorkspace
     }
 
     public func notes(folder: String?, modifiedSince: String?, includeBody: Bool) throws -> [NoteEntry] {
@@ -375,6 +392,11 @@ public struct LocalSQLiteInventoryReader: Sendable {
 
     public func safariHistory(confirmSensitive: Bool, since: String?, until: String?, limit: Int, redactURLs: Bool) throws -> [SafariHistoryEntry] {
         guard confirmSensitive else { throw LocalInventoryError.sensitiveConfirmationRequired("icloud-cli safari history") }
+        if snapshotsLiveStores {
+            return try withSnapshot { reader in
+                try reader.safariHistory(confirmSensitive: true, since: since, until: until, limit: limit, redactURLs: redactURLs)
+            }
+        }
         if try tableExists("history_items"), try tableExists("history_visits") {
             return try appleSafariHistory(since: since, until: until, limit: limit, redactURLs: redactURLs)
         }
@@ -414,11 +436,14 @@ public struct LocalSQLiteInventoryReader: Sendable {
     }
 
     public func messageConversations(limit: Int = 50) throws -> [MessageConversation] {
+        if snapshotsLiveStores {
+            return try withSnapshot { try $0.messageConversations(limit: limit) }
+        }
         if try tableExists("message_conversations") {
             return try query("SELECT chatIdentifier, displayName, participantCount, lastMessageAt, messageCount FROM message_conversations ORDER BY lastMessageAt DESC LIMIT \(bounded(limit, defaultValue: 50, max: 1000));")
         }
         guard try tableExists("chat"), try tableExists("message"), try tableExists("chat_message_join") else {
-            throw LocalInventoryError.unsupportedSchema(store: database.path, detail: "missing message_conversations or chat/message/chat_message_join tables")
+            throw LocalInventoryError.unsupportedSchema(store: reportedStore, detail: "missing message_conversations or chat/message/chat_message_join tables")
         }
         return try query("""
             SELECT
@@ -439,13 +464,18 @@ public struct LocalSQLiteInventoryReader: Sendable {
 
     public func recentMessages(confirmSensitive: Bool, includeBody: Bool, since: String?, limit: Int) throws -> [MessageRecentEntry] {
         guard confirmSensitive else { throw LocalInventoryError.sensitiveConfirmationRequired("icloud-cli messages recent") }
+        if snapshotsLiveStores {
+            return try withSnapshot { reader in
+                try reader.recentMessages(confirmSensitive: true, includeBody: includeBody, since: since, limit: limit)
+            }
+        }
         let floor = since ?? ISO8601DateFormatter().string(from: Date(timeIntervalSinceNow: -86_400))
         let bodyColumn = includeBody ? "body" : "NULL AS body"
         if try tableExists("recent_messages") {
             return try query("SELECT chatIdentifier, sender, sentAt, isFromMe, \(bodyColumn) FROM recent_messages WHERE sentAt >= '\(sqlEscape(floor))' ORDER BY sentAt DESC LIMIT \(bounded(limit, defaultValue: 20, max: 1000));")
         }
         guard try tableExists("message"), try tableExists("chat"), try tableExists("chat_message_join") else {
-            throw LocalInventoryError.unsupportedSchema(store: database.path, detail: "missing recent_messages or message/chat/chat_message_join tables")
+            throw LocalInventoryError.unsupportedSchema(store: reportedStore, detail: "missing recent_messages or message/chat/chat_message_join tables")
         }
         let appleBodyColumn = includeBody ? "m.text" : "NULL AS body"
         let appleFloor = since.flatMap { Int64($0) }
@@ -612,7 +642,12 @@ public struct LocalSQLiteInventoryReader: Sendable {
     }
 
     private func query<T: Decodable>(_ sql: String) throws -> [T] {
-        guard FileManager.default.fileExists(atPath: database.path) else { throw LocalInventoryError.missingStore(database.path) }
+        if !snapshotsLiveStores {
+            guard let snapshotWorkspace else { throw LocalInventoryError.sqliteFailure("SQLite snapshot workspace is unavailable") }
+            return try SQLiteSnapshotQueryEngine.production(source: database, reportedStore: reportedStore)
+                .querySnapshot(database, workspace: snapshotWorkspace, sql: sql)
+        }
+        guard FileManager.default.fileExists(atPath: database.path) else { throw LocalInventoryError.missingStore(reportedStore) }
         let process = Process()
         process.executableURL = URL(fileURLWithPath: "/usr/bin/sqlite3")
         process.arguments = ["-readonly", "-json", database.path, sql]
@@ -627,7 +662,7 @@ public struct LocalSQLiteInventoryReader: Sendable {
         let data = outputPipe.fileHandleForReading.readDataToEndOfFile()
         let errorData = errorPipe.fileHandleForReading.readDataToEndOfFile()
         guard process.terminationStatus == 0 else {
-            throw sqliteError(from: errorData, store: database.path)
+            throw sqliteError(from: errorData, store: reportedStore)
         }
         if data.isEmpty { return [] }
         return try JSONDecoder().decode([T].self, from: data)
@@ -643,6 +678,12 @@ public struct LocalSQLiteInventoryReader: Sendable {
         return rows.contains { $0.name == column }
     }
 
+    private func withSnapshot<Result>(_ operation: (LocalSQLiteInventoryReader) throws -> Result) throws -> Result {
+        try SQLiteSnapshotQueryEngine.production(source: database).withSnapshot { snapshot, workspace in
+            try operation(LocalSQLiteInventoryReader(database: snapshot, snapshotsLiveStores: false, reportedStore: reportedStore, snapshotWorkspace: workspace))
+        }
+    }
+
     private func contactColumnReference(_ candidates: [String]) -> String {
         for column in candidates {
             if (try? columnExists("ZABCDRECORD", column)) == true {
@@ -796,6 +837,9 @@ func sqliteError(from errorData: Data, store: String) -> LocalInventoryError {
     if lowercased.contains("no such table") || lowercased.contains("no such column") {
         return .unsupportedSchema(store: store, detail: message)
     }
+    if lowercased.contains("database is locked") || lowercased.contains("database is busy") {
+        return .lockedStore(store)
+    }
     if lowercased.contains("file is not a database") || lowercased.contains("file is not in a database") {
         return .unsupportedSchema(store: store, detail: "not a SQLite database")
     }

Sources/ICloudCLICore/ProviderManifest.swift

@@ -48,13 +48,13 @@ public enum ProviderRegistry {
         provider("home", "Home", .beta, .sqlite, .high, ["home accessories", "home homes", "home rooms", "home scenes"], ["accessories", "homes", "rooms", "scenes"]),
         provider("mail", "Mail", .beta, .sqlite, .high, ["mail accounts", "mail mailboxes", "mail recent"], ["accounts", "headers", "mailboxes"]),
         provider("maps", "Maps", .beta, .sqlite, .high, ["maps favorites", "maps recents"], ["favorites", "recents", "location"]),
-        provider("messages", "Messages", .beta, .sqlite, .high, ["messages conversations", "messages recent"], ["conversations", "messages", "date-filtering"]),
+        provider("messages", "Messages", .beta, .sqlite, .high, ["messages conversations", "messages recent"], ["consistent-snapshot", "conversations", "messages", "date-filtering"]),
         provider("music", "Music", .beta, .sqlite, .moderate, ["music playlists", "music status", "music tracks"], ["inventory", "playlists", "status"]),
         provider("news", "News", .beta, .sqlite, .moderate, ["news history", "news topics"], ["history", "topics", "date-filtering"]),
         provider("notes", "Notes", .beta, .sqlite, .high, ["notes accounts", "notes folders", "notes list", "notes shared", "notes tags"], ["accounts", "folders", "inventory", "sharing", "tags"]),
         provider("photos", "Photos", .beta, .mixed, .high, ["photos list", "photos screenshots", "photos shared-albums", "photos shared-library"], ["assets", "screenshots", "sharing"]),
         provider("reminders", "Reminders", .beta, .sqlite, .high, ["reminders assigned", "reminders flagged", "reminders list", "reminders lists", "reminders scheduled", "reminders today"], ["inventory", "lists", "smart-views"]),
-        provider("safari", "Safari", .beta, .mixed, .high, ["safari bookmarks", "safari cloud-tabs list", "safari cloud-tabs probe", "safari extensions list", "safari frequently-visited", "safari history", "safari profiles list", "safari reading-list", "safari tabs"], ["bookmarks", "cloud-tabs", "extensions", "history", "profiles", "tabs"]),
+        provider("safari", "Safari", .beta, .mixed, .high, ["safari bookmarks", "safari cloud-tabs list", "safari cloud-tabs probe", "safari extensions list", "safari frequently-visited", "safari history", "safari profiles list", "safari reading-list", "safari tabs"], ["bookmarks", "cloud-tabs", "consistent-snapshot", "extensions", "history", "profiles", "tabs"]),
         provider("shortcuts", "Shortcuts", .stable, .filesystem, .moderate, ["shortcuts list"], ["inventory", "search"], true),
         provider("stocks", "Stocks", .beta, .sqlite, .moderate, ["stocks groups", "stocks watchlist"], ["groups", "watchlist"]),
         provider("storage", "iCloud Storage", .stable, .preferences, .low, ["storage status"], ["quota", "status"], true),

Sources/ICloudCLICore/SQLiteSnapshotQuery.swift

@@ -0,0 +1,156 @@
+import Darwin
+import Foundation
+
+public struct SQLiteSnapshotQueryEngine: Sendable {
+    public static let productionMinimumTimeout: TimeInterval = 5
+    public static let productionMinimumBusyTimeoutMilliseconds = 500
+    public let source: URL
+    public let timeout: TimeInterval
+    public let busyTimeoutMilliseconds: Int
+    public let temporaryRoot: URL
+    private let reportedStore: String
+
+    public init(
+        source: URL,
+        timeout: TimeInterval = 10,
+        busyTimeoutMilliseconds: Int = 1_000,
+        temporaryRoot: URL = FileManager.default.temporaryDirectory,
+        reportedStore: String? = nil
+    ) {
+        self.source = source
+        self.timeout = max(0.001, timeout)
+        self.busyTimeoutMilliseconds = max(0, busyTimeoutMilliseconds)
+        self.temporaryRoot = temporaryRoot
+        self.reportedStore = reportedStore ?? source.path
+    }
+
+    /// Creates an engine for a live iCloud SQLite store with conservative timeout floors.
+    public static func production(
+        source: URL,
+        timeout: TimeInterval = 10,
+        busyTimeoutMilliseconds: Int = 1_000,
+        temporaryRoot: URL = FileManager.default.temporaryDirectory,
+        reportedStore: String? = nil
+    ) -> Self {
+        Self(
+            source: source,
+            timeout: max(productionMinimumTimeout, timeout),
+            busyTimeoutMilliseconds: max(productionMinimumBusyTimeoutMilliseconds, busyTimeoutMilliseconds),
+            temporaryRoot: temporaryRoot,
+            reportedStore: reportedStore
+        )
+    }
+
+    public func query<T: Decodable>(_ sql: String) throws -> [T] {
+        try withSnapshot { snapshot, directory in
+            try querySnapshot(snapshot, workspace: directory, sql: sql)
+        }
+    }
+
+    func querySnapshot<T: Decodable>(_ snapshot: URL, workspace: URL, sql: String) throws -> [T] {
+        let identifier = UUID().uuidString
+        let output = workspace.appendingPathComponent("query-\(identifier).json")
+        let errors = workspace.appendingPathComponent("query-\(identifier).err")
+        FileManager.default.createFile(atPath: output.path, contents: nil, attributes: [.posixPermissions: 0o600])
+        FileManager.default.createFile(atPath: errors.path, contents: nil, attributes: [.posixPermissions: 0o600])
+        defer {
+            try? FileManager.default.removeItem(at: output)
+            try? FileManager.default.removeItem(at: errors)
+        }
+        let outputHandle = try FileHandle(forWritingTo: output)
+        let errorHandle = try FileHandle(forWritingTo: errors)
+        defer {
+            try? outputHandle.close()
+            try? errorHandle.close()
+        }
+
+        let process = Process()
+        process.executableURL = URL(fileURLWithPath: "/usr/bin/sqlite3")
+        process.arguments = [
+            "-readonly", "-json",
+            "-cmd", ".timeout \(busyTimeoutMilliseconds)",
+            "-cmd", "PRAGMA query_only=ON;",
+            snapshot.path,
+            sql,
+        ]
+        process.standardOutput = outputHandle
+        process.standardError = errorHandle
+
+        let completed = DispatchSemaphore(value: 0)
+        process.terminationHandler = { _ in completed.signal() }
+        do {
+            try process.run()
+        } catch {
+            throw LocalInventoryError.sqliteFailure(error.localizedDescription)
+        }
+        guard completed.wait(timeout: .now() + timeout) == .success else {
+            if process.isRunning {
+                process.terminate()
+            }
+            if completed.wait(timeout: .now() + 1) != .success {
+                if process.isRunning {
+                    Darwin.kill(process.processIdentifier, SIGKILL)
+                    _ = completed.wait(timeout: .now() + 1)
+                }
+            }
+            throw LocalInventoryError.queryTimeout(reportedStore)
+        }
+
+        outputHandle.synchronizeFile()
+        errorHandle.synchronizeFile()
+        let data = try Data(contentsOf: output)
+        let errorData = try Data(contentsOf: errors)
+        guard process.terminationStatus == 0 else {
+            throw sqliteError(from: errorData, store: reportedStore)
+        }
+        if data.isEmpty { return [] }
+        do {
+            return try JSONDecoder().decode([T].self, from: data)
+        } catch {
+            throw LocalInventoryError.sqliteFailure("SQLite returned an invalid JSON result")
+        }
+    }
+
+    func withSnapshot<Result>(_ operation: (URL, URL) throws -> Result) throws -> Result {
+        let resolvedSource = source.resolvingSymlinksInPath()
+        guard FileManager.default.fileExists(atPath: resolvedSource.path) else {
+            throw LocalInventoryError.missingStore(reportedStore)
+        }
+
+        let directory = temporaryRoot.appendingPathComponent("icloud-cli-sqlite-\(UUID().uuidString)", isDirectory: true)
+        do {
+            try FileManager.default.createDirectory(
+                at: directory,
+                withIntermediateDirectories: true,
+                attributes: [.posixPermissions: 0o700]
+            )
+        } catch {
+            throw LocalInventoryError.sqliteFailure("Unable to create private SQLite snapshot directory")
+        }
+        defer { try? FileManager.default.removeItem(at: directory) }
+
+        let snapshot = directory.appendingPathComponent("snapshot.sqlite")
+        do {
+            try copy(resolvedSource, to: snapshot)
+            for suffix in ["-wal", "-shm"] {
+                let companion = URL(fileURLWithPath: resolvedSource.path + suffix)
+                guard FileManager.default.fileExists(atPath: companion.path) else { continue }
+                try copy(companion, to: URL(fileURLWithPath: snapshot.path + suffix))
+            }
+        } catch {
+            if isPermissionError(error) { throw LocalInventoryError.permissionDenied(reportedStore) }
+            throw LocalInventoryError.sqliteFailure("Unable to create SQLite snapshot")
+        }
+        return try operation(snapshot, directory)
+    }
+
+    private func copy(_ source: URL, to destination: URL) throws {
+        try FileManager.default.copyItem(at: source, to: destination)
+        try FileManager.default.setAttributes([.posixPermissions: 0o600], ofItemAtPath: destination.path)
+    }
+
+    private func isPermissionError(_ error: Error) -> Bool {
+        let cocoa = error as NSError
+        return cocoa.domain == NSCocoaErrorDomain && [NSFileReadNoPermissionError, NSFileWriteNoPermissionError].contains(cocoa.code)
+    }
+}

Tests/ICloudCLICoreTests/ProviderManifestTests.swift

@@ -21,6 +21,8 @@ import Testing
     ])
     #expect(manifest.providers.allSatisfy { !$0.commands.isEmpty && !$0.capabilities.isEmpty })
     #expect(manifest.providers.allSatisfy { $0.accessMode == .readOnly })
+    #expect(manifest.providers.first { $0.id == "messages" }?.capabilities.contains("consistent-snapshot") == true)
+    #expect(manifest.providers.first { $0.id == "safari" }?.capabilities.contains("consistent-snapshot") == true)
 }
 
 @Test func providerManifestJSONContainsMetadataOnly() throws {