diff --git a/Sources/ICloudCLICore/LocalInventories.swift b/Sources/ICloudCLICore/LocalInventories.swift index ccb41c9..d99f578 100644 --- a/Sources/ICloudCLICore/LocalInventories.swift +++ b/Sources/ICloudCLICore/LocalInventories.swift @@ -4,6 +4,8 @@ public enum LocalInventoryError: Error, LocalizedError, Equatable { case missingRoot(String) case missingStore(String) case permissionDenied(String) + case lockedStore(String) + case queryTimeout(String) case sensitiveConfirmationRequired(String) case unsupportedSchema(store: String, detail: String) case sqliteFailure(String) @@ -14,6 +16,8 @@ public enum LocalInventoryError: Error, LocalizedError, Equatable { case .missingStore(let path): return "Inventory store not available: \(path)" case .permissionDenied(let path): return "Permission denied reading local inventory store: \(path). Grant Full Disk Access to the calling terminal or agent process, then retry." + case .lockedStore(let path): return "Local inventory store is locked or busy: \(path)" + case .queryTimeout(let path): return "Timed out querying local inventory store: \(path)" case .sensitiveConfirmationRequired(let command): return "\(command) reads high-sensitivity local data; rerun with --confirm-sensitive" case .unsupportedSchema(let store, let detail): @@ -247,9 +251,22 @@ public struct NewsTopicEntry: Codable, Equatable, Sendable { public struct LocalSQLiteInventoryReader: Sendable { public let database: URL + private let snapshotsLiveStores: Bool + private let reportedStore: String + private let snapshotWorkspace: URL? public init(database: URL) { self.database = database + self.snapshotsLiveStores = true + self.reportedStore = database.path + self.snapshotWorkspace = nil + } + + private init(database: URL, snapshotsLiveStores: Bool, reportedStore: String, snapshotWorkspace: URL?) { + self.database = database + self.snapshotsLiveStores = snapshotsLiveStores + self.reportedStore = reportedStore + self.snapshotWorkspace = snapshotWorkspace } public func notes(folder: String?, modifiedSince: String?, includeBody: Bool) throws -> [NoteEntry] { @@ -375,6 +392,11 @@ public struct LocalSQLiteInventoryReader: Sendable { public func safariHistory(confirmSensitive: Bool, since: String?, until: String?, limit: Int, redactURLs: Bool) throws -> [SafariHistoryEntry] { guard confirmSensitive else { throw LocalInventoryError.sensitiveConfirmationRequired("icloud-cli safari history") } + if snapshotsLiveStores { + return try withSnapshot { reader in + try reader.safariHistory(confirmSensitive: true, since: since, until: until, limit: limit, redactURLs: redactURLs) + } + } if try tableExists("history_items"), try tableExists("history_visits") { return try appleSafariHistory(since: since, until: until, limit: limit, redactURLs: redactURLs) } @@ -414,11 +436,14 @@ public struct LocalSQLiteInventoryReader: Sendable { } public func messageConversations(limit: Int = 50) throws -> [MessageConversation] { + if snapshotsLiveStores { + return try withSnapshot { try $0.messageConversations(limit: limit) } + } if try tableExists("message_conversations") { return try query("SELECT chatIdentifier, displayName, participantCount, lastMessageAt, messageCount FROM message_conversations ORDER BY lastMessageAt DESC LIMIT \(bounded(limit, defaultValue: 50, max: 1000));") } guard try tableExists("chat"), try tableExists("message"), try tableExists("chat_message_join") else { - throw LocalInventoryError.unsupportedSchema(store: database.path, detail: "missing message_conversations or chat/message/chat_message_join tables") + throw LocalInventoryError.unsupportedSchema(store: reportedStore, detail: "missing message_conversations or chat/message/chat_message_join tables") } return try query(""" SELECT @@ -439,13 +464,18 @@ public struct LocalSQLiteInventoryReader: Sendable { public func recentMessages(confirmSensitive: Bool, includeBody: Bool, since: String?, limit: Int) throws -> [MessageRecentEntry] { guard confirmSensitive else { throw LocalInventoryError.sensitiveConfirmationRequired("icloud-cli messages recent") } + if snapshotsLiveStores { + return try withSnapshot { reader in + try reader.recentMessages(confirmSensitive: true, includeBody: includeBody, since: since, limit: limit) + } + } let floor = since ?? ISO8601DateFormatter().string(from: Date(timeIntervalSinceNow: -86_400)) let bodyColumn = includeBody ? "body" : "NULL AS body" if try tableExists("recent_messages") { return try query("SELECT chatIdentifier, sender, sentAt, isFromMe, \(bodyColumn) FROM recent_messages WHERE sentAt >= '\(sqlEscape(floor))' ORDER BY sentAt DESC LIMIT \(bounded(limit, defaultValue: 20, max: 1000));") } guard try tableExists("message"), try tableExists("chat"), try tableExists("chat_message_join") else { - throw LocalInventoryError.unsupportedSchema(store: database.path, detail: "missing recent_messages or message/chat/chat_message_join tables") + throw LocalInventoryError.unsupportedSchema(store: reportedStore, detail: "missing recent_messages or message/chat/chat_message_join tables") } let appleBodyColumn = includeBody ? "m.text" : "NULL AS body" let appleFloor = since.flatMap { Int64($0) } @@ -612,7 +642,12 @@ public struct LocalSQLiteInventoryReader: Sendable { } private func query(_ sql: String) throws -> [T] { - guard FileManager.default.fileExists(atPath: database.path) else { throw LocalInventoryError.missingStore(database.path) } + if !snapshotsLiveStores { + guard let snapshotWorkspace else { throw LocalInventoryError.sqliteFailure("SQLite snapshot workspace is unavailable") } + return try SQLiteSnapshotQueryEngine.production(source: database, reportedStore: reportedStore) + .querySnapshot(database, workspace: snapshotWorkspace, sql: sql) + } + guard FileManager.default.fileExists(atPath: database.path) else { throw LocalInventoryError.missingStore(reportedStore) } let process = Process() process.executableURL = URL(fileURLWithPath: "/usr/bin/sqlite3") process.arguments = ["-readonly", "-json", database.path, sql] @@ -627,7 +662,7 @@ public struct LocalSQLiteInventoryReader: Sendable { let data = outputPipe.fileHandleForReading.readDataToEndOfFile() let errorData = errorPipe.fileHandleForReading.readDataToEndOfFile() guard process.terminationStatus == 0 else { - throw sqliteError(from: errorData, store: database.path) + throw sqliteError(from: errorData, store: reportedStore) } if data.isEmpty { return [] } return try JSONDecoder().decode([T].self, from: data) @@ -643,6 +678,12 @@ public struct LocalSQLiteInventoryReader: Sendable { return rows.contains { $0.name == column } } + private func withSnapshot(_ operation: (LocalSQLiteInventoryReader) throws -> Result) throws -> Result { + try SQLiteSnapshotQueryEngine.production(source: database).withSnapshot { snapshot, workspace in + try operation(LocalSQLiteInventoryReader(database: snapshot, snapshotsLiveStores: false, reportedStore: reportedStore, snapshotWorkspace: workspace)) + } + } + private func contactColumnReference(_ candidates: [String]) -> String { for column in candidates { if (try? columnExists("ZABCDRECORD", column)) == true { @@ -796,6 +837,9 @@ func sqliteError(from errorData: Data, store: String) -> LocalInventoryError { if lowercased.contains("no such table") || lowercased.contains("no such column") { return .unsupportedSchema(store: store, detail: message) } + if lowercased.contains("database is locked") || lowercased.contains("database is busy") { + return .lockedStore(store) + } if lowercased.contains("file is not a database") || lowercased.contains("file is not in a database") { return .unsupportedSchema(store: store, detail: "not a SQLite database") } diff --git a/Sources/ICloudCLICore/ProviderManifest.swift b/Sources/ICloudCLICore/ProviderManifest.swift index 6cb28b3..d31d2dd 100644 --- a/Sources/ICloudCLICore/ProviderManifest.swift +++ b/Sources/ICloudCLICore/ProviderManifest.swift @@ -48,13 +48,13 @@ public enum ProviderRegistry { provider("home", "Home", .beta, .sqlite, .high, ["home accessories", "home homes", "home rooms", "home scenes"], ["accessories", "homes", "rooms", "scenes"]), provider("mail", "Mail", .beta, .sqlite, .high, ["mail accounts", "mail mailboxes", "mail recent"], ["accounts", "headers", "mailboxes"]), provider("maps", "Maps", .beta, .sqlite, .high, ["maps favorites", "maps recents"], ["favorites", "recents", "location"]), - provider("messages", "Messages", .beta, .sqlite, .high, ["messages conversations", "messages recent"], ["conversations", "messages", "date-filtering"]), + provider("messages", "Messages", .beta, .sqlite, .high, ["messages conversations", "messages recent"], ["consistent-snapshot", "conversations", "messages", "date-filtering"]), provider("music", "Music", .beta, .sqlite, .moderate, ["music playlists", "music status", "music tracks"], ["inventory", "playlists", "status"]), provider("news", "News", .beta, .sqlite, .moderate, ["news history", "news topics"], ["history", "topics", "date-filtering"]), provider("notes", "Notes", .beta, .sqlite, .high, ["notes accounts", "notes folders", "notes list", "notes shared", "notes tags"], ["accounts", "folders", "inventory", "sharing", "tags"]), provider("photos", "Photos", .beta, .mixed, .high, ["photos list", "photos screenshots", "photos shared-albums", "photos shared-library"], ["assets", "screenshots", "sharing"]), provider("reminders", "Reminders", .beta, .sqlite, .high, ["reminders assigned", "reminders flagged", "reminders list", "reminders lists", "reminders scheduled", "reminders today"], ["inventory", "lists", "smart-views"]), - provider("safari", "Safari", .beta, .mixed, .high, ["safari bookmarks", "safari cloud-tabs list", "safari cloud-tabs probe", "safari extensions list", "safari frequently-visited", "safari history", "safari profiles list", "safari reading-list", "safari tabs"], ["bookmarks", "cloud-tabs", "extensions", "history", "profiles", "tabs"]), + provider("safari", "Safari", .beta, .mixed, .high, ["safari bookmarks", "safari cloud-tabs list", "safari cloud-tabs probe", "safari extensions list", "safari frequently-visited", "safari history", "safari profiles list", "safari reading-list", "safari tabs"], ["bookmarks", "cloud-tabs", "consistent-snapshot", "extensions", "history", "profiles", "tabs"]), provider("shortcuts", "Shortcuts", .stable, .filesystem, .moderate, ["shortcuts list"], ["inventory", "search"], true), provider("stocks", "Stocks", .beta, .sqlite, .moderate, ["stocks groups", "stocks watchlist"], ["groups", "watchlist"]), provider("storage", "iCloud Storage", .stable, .preferences, .low, ["storage status"], ["quota", "status"], true), diff --git a/Sources/ICloudCLICore/SQLiteSnapshotQuery.swift b/Sources/ICloudCLICore/SQLiteSnapshotQuery.swift new file mode 100644 index 0000000..b73e55f --- /dev/null +++ b/Sources/ICloudCLICore/SQLiteSnapshotQuery.swift @@ -0,0 +1,156 @@ +import Darwin +import Foundation + +public struct SQLiteSnapshotQueryEngine: Sendable { + public static let productionMinimumTimeout: TimeInterval = 5 + public static let productionMinimumBusyTimeoutMilliseconds = 500 + public let source: URL + public let timeout: TimeInterval + public let busyTimeoutMilliseconds: Int + public let temporaryRoot: URL + private let reportedStore: String + + public init( + source: URL, + timeout: TimeInterval = 10, + busyTimeoutMilliseconds: Int = 1_000, + temporaryRoot: URL = FileManager.default.temporaryDirectory, + reportedStore: String? = nil + ) { + self.source = source + self.timeout = max(0.001, timeout) + self.busyTimeoutMilliseconds = max(0, busyTimeoutMilliseconds) + self.temporaryRoot = temporaryRoot + self.reportedStore = reportedStore ?? source.path + } + + /// Creates an engine for a live iCloud SQLite store with conservative timeout floors. + public static func production( + source: URL, + timeout: TimeInterval = 10, + busyTimeoutMilliseconds: Int = 1_000, + temporaryRoot: URL = FileManager.default.temporaryDirectory, + reportedStore: String? = nil + ) -> Self { + Self( + source: source, + timeout: max(productionMinimumTimeout, timeout), + busyTimeoutMilliseconds: max(productionMinimumBusyTimeoutMilliseconds, busyTimeoutMilliseconds), + temporaryRoot: temporaryRoot, + reportedStore: reportedStore + ) + } + + public func query(_ sql: String) throws -> [T] { + try withSnapshot { snapshot, directory in + try querySnapshot(snapshot, workspace: directory, sql: sql) + } + } + + func querySnapshot(_ snapshot: URL, workspace: URL, sql: String) throws -> [T] { + let identifier = UUID().uuidString + let output = workspace.appendingPathComponent("query-\(identifier).json") + let errors = workspace.appendingPathComponent("query-\(identifier).err") + FileManager.default.createFile(atPath: output.path, contents: nil, attributes: [.posixPermissions: 0o600]) + FileManager.default.createFile(atPath: errors.path, contents: nil, attributes: [.posixPermissions: 0o600]) + defer { + try? FileManager.default.removeItem(at: output) + try? FileManager.default.removeItem(at: errors) + } + let outputHandle = try FileHandle(forWritingTo: output) + let errorHandle = try FileHandle(forWritingTo: errors) + defer { + try? outputHandle.close() + try? errorHandle.close() + } + + let process = Process() + process.executableURL = URL(fileURLWithPath: "/usr/bin/sqlite3") + process.arguments = [ + "-readonly", "-json", + "-cmd", ".timeout \(busyTimeoutMilliseconds)", + "-cmd", "PRAGMA query_only=ON;", + snapshot.path, + sql, + ] + process.standardOutput = outputHandle + process.standardError = errorHandle + + let completed = DispatchSemaphore(value: 0) + process.terminationHandler = { _ in completed.signal() } + do { + try process.run() + } catch { + throw LocalInventoryError.sqliteFailure(error.localizedDescription) + } + guard completed.wait(timeout: .now() + timeout) == .success else { + if process.isRunning { + process.terminate() + } + if completed.wait(timeout: .now() + 1) != .success { + if process.isRunning { + Darwin.kill(process.processIdentifier, SIGKILL) + _ = completed.wait(timeout: .now() + 1) + } + } + throw LocalInventoryError.queryTimeout(reportedStore) + } + + outputHandle.synchronizeFile() + errorHandle.synchronizeFile() + let data = try Data(contentsOf: output) + let errorData = try Data(contentsOf: errors) + guard process.terminationStatus == 0 else { + throw sqliteError(from: errorData, store: reportedStore) + } + if data.isEmpty { return [] } + do { + return try JSONDecoder().decode([T].self, from: data) + } catch { + throw LocalInventoryError.sqliteFailure("SQLite returned an invalid JSON result") + } + } + + func withSnapshot(_ operation: (URL, URL) throws -> Result) throws -> Result { + let resolvedSource = source.resolvingSymlinksInPath() + guard FileManager.default.fileExists(atPath: resolvedSource.path) else { + throw LocalInventoryError.missingStore(reportedStore) + } + + let directory = temporaryRoot.appendingPathComponent("icloud-cli-sqlite-\(UUID().uuidString)", isDirectory: true) + do { + try FileManager.default.createDirectory( + at: directory, + withIntermediateDirectories: true, + attributes: [.posixPermissions: 0o700] + ) + } catch { + throw LocalInventoryError.sqliteFailure("Unable to create private SQLite snapshot directory") + } + defer { try? FileManager.default.removeItem(at: directory) } + + let snapshot = directory.appendingPathComponent("snapshot.sqlite") + do { + try copy(resolvedSource, to: snapshot) + for suffix in ["-wal", "-shm"] { + let companion = URL(fileURLWithPath: resolvedSource.path + suffix) + guard FileManager.default.fileExists(atPath: companion.path) else { continue } + try copy(companion, to: URL(fileURLWithPath: snapshot.path + suffix)) + } + } catch { + if isPermissionError(error) { throw LocalInventoryError.permissionDenied(reportedStore) } + throw LocalInventoryError.sqliteFailure("Unable to create SQLite snapshot") + } + return try operation(snapshot, directory) + } + + private func copy(_ source: URL, to destination: URL) throws { + try FileManager.default.copyItem(at: source, to: destination) + try FileManager.default.setAttributes([.posixPermissions: 0o600], ofItemAtPath: destination.path) + } + + private func isPermissionError(_ error: Error) -> Bool { + let cocoa = error as NSError + return cocoa.domain == NSCocoaErrorDomain && [NSFileReadNoPermissionError, NSFileWriteNoPermissionError].contains(cocoa.code) + } +} diff --git a/Tests/ICloudCLICoreTests/ProviderManifestTests.swift b/Tests/ICloudCLICoreTests/ProviderManifestTests.swift index 74f6665..f480ded 100644 --- a/Tests/ICloudCLICoreTests/ProviderManifestTests.swift +++ b/Tests/ICloudCLICoreTests/ProviderManifestTests.swift @@ -21,6 +21,8 @@ import Testing ]) #expect(manifest.providers.allSatisfy { !$0.commands.isEmpty && !$0.capabilities.isEmpty }) #expect(manifest.providers.allSatisfy { $0.accessMode == .readOnly }) + #expect(manifest.providers.first { $0.id == "messages" }?.capabilities.contains("consistent-snapshot") == true) + #expect(manifest.providers.first { $0.id == "safari" }?.capabilities.contains("consistent-snapshot") == true) } @Test func providerManifestJSONContainsMetadataOnly() throws { diff --git a/Tests/ICloudCLICoreTests/SQLiteSnapshotQueryTests.swift b/Tests/ICloudCLICoreTests/SQLiteSnapshotQueryTests.swift new file mode 100644 index 0000000..a27d6d1 --- /dev/null +++ b/Tests/ICloudCLICoreTests/SQLiteSnapshotQueryTests.swift @@ -0,0 +1,209 @@ +import Foundation +import Testing +@testable import ICloudCLICore + +private struct SnapshotValueRow: Decodable, Equatable { let value: String } + +@Test func snapshotQueryReadsDatabaseWithoutCompanionFilesAndCleansUp() throws { + let root = try temporarySQLiteSnapshotDirectory(named: "no-companions") + defer { try? FileManager.default.removeItem(at: root) } + let database = root.appendingPathComponent("source.db") + try runSnapshotFixtureSQL(database: database, sql: "CREATE TABLE values_table (value TEXT); INSERT INTO values_table VALUES ('one');") + let snapshots = root.appendingPathComponent("snapshots") + try FileManager.default.createDirectory(at: snapshots, withIntermediateDirectories: true) + + let rows: [SnapshotValueRow] = try SQLiteSnapshotQueryEngine(source: database, temporaryRoot: snapshots) + .query("SELECT value FROM values_table;") + + #expect(rows == [SnapshotValueRow(value: "one")]) + #expect(try FileManager.default.contentsOfDirectory(atPath: snapshots.path).isEmpty) +} + +@Test func snapshotQueryIncludesCommittedWALRows() throws { + let root = try temporarySQLiteSnapshotDirectory(named: "wal") + defer { try? FileManager.default.removeItem(at: root) } + let database = root.appendingPathComponent("source.db") + let writer = try openWALFixture(database: database) + defer { writer.stop() } + + #expect(FileManager.default.fileExists(atPath: database.path + "-wal")) + let rows: [SnapshotValueRow] = try SQLiteSnapshotQueryEngine(source: database) + .query("SELECT value FROM values_table ORDER BY value;") + #expect(rows == [SnapshotValueRow(value: "wal-row")]) +} + +@Test func snapshotQueryFollowsSymlinkedStoreAndCopiesWALCompanions() throws { + let root = try temporarySQLiteSnapshotDirectory(named: "symlink-wal") + defer { try? FileManager.default.removeItem(at: root) } + let liveStore = root.appendingPathComponent("live/source.db") + try FileManager.default.createDirectory(at: liveStore.deletingLastPathComponent(), withIntermediateDirectories: true) + let writer = try openWALFixture(database: liveStore) + defer { writer.stop() } + let linkedStore = root.appendingPathComponent("linked-source.db") + try FileManager.default.createSymbolicLink(atPath: linkedStore.path, withDestinationPath: liveStore.path) + + let engine = SQLiteSnapshotQueryEngine(source: linkedStore) + try engine.withSnapshot { snapshot, workspace in + #expect((try? FileManager.default.destinationOfSymbolicLink(atPath: snapshot.path)) == nil) + #expect(FileManager.default.fileExists(atPath: snapshot.path + "-wal")) + let rows: [SnapshotValueRow] = try engine.querySnapshot(snapshot, workspace: workspace, sql: "SELECT value FROM values_table ORDER BY value;") + #expect(rows == [SnapshotValueRow(value: "wal-row")]) + } +} + +@Test func snapshotQueryMapsSchemaDriftAndBusyFailures() { + let schema = sqliteError(from: Data("Error: no such table: missing".utf8), store: "/private/source.db") + #expect(schema == .unsupportedSchema(store: "/private/source.db", detail: "Error: no such table: missing")) + + let locked = sqliteError(from: Data("Error: database is locked".utf8), store: "/private/source.db") + #expect(locked == .lockedStore("/private/source.db")) +} + +@Test func snapshotQueryRejectsWritesAndLeavesSourceUnchanged() throws { + let root = try temporarySQLiteSnapshotDirectory(named: "query-only") + defer { try? FileManager.default.removeItem(at: root) } + let database = root.appendingPathComponent("source.db") + try runSnapshotFixtureSQL(database: database, sql: "CREATE TABLE values_table (value TEXT); INSERT INTO values_table VALUES ('one');") + let engine = SQLiteSnapshotQueryEngine(source: database) + + #expect(throws: LocalInventoryError.self) { + let _: [SnapshotValueRow] = try engine.query("INSERT INTO values_table VALUES ('two') RETURNING value;") + } + #expect(try snapshotFixtureScalar(database: database, sql: "SELECT COUNT(*) FROM values_table;") == "1") +} + +@Test func snapshotQueryReportsLockedStore() throws { + let root = try temporarySQLiteSnapshotDirectory(named: "locked") + defer { try? FileManager.default.removeItem(at: root) } + let database = root.appendingPathComponent("source.db") + try runSnapshotFixtureSQL(database: database, sql: "CREATE TABLE values_table (value TEXT); INSERT INTO values_table VALUES ('one');") + let engine = SQLiteSnapshotQueryEngine(source: database, busyTimeoutMilliseconds: 20) + + #expect(throws: LocalInventoryError.lockedStore(database.path)) { + try engine.withSnapshot { snapshot, workspace in + let locker = try lockSnapshotDatabase(snapshot) + defer { locker.stop() } + Thread.sleep(forTimeInterval: 0.05) + let _: [SnapshotValueRow] = try engine.querySnapshot(snapshot, workspace: workspace, sql: "SELECT value FROM values_table;") + } + } +} + +@Test func snapshotQueryTimesOutAndCleansUp() throws { + let root = try temporarySQLiteSnapshotDirectory(named: "timeout") + defer { try? FileManager.default.removeItem(at: root) } + let database = root.appendingPathComponent("source.db") + try runSnapshotFixtureSQL(database: database, sql: "CREATE TABLE values_table (value TEXT);") + let snapshots = root.appendingPathComponent("snapshots") + try FileManager.default.createDirectory(at: snapshots, withIntermediateDirectories: true) + let engine = SQLiteSnapshotQueryEngine(source: database, timeout: 0.01, temporaryRoot: snapshots) + + #expect(throws: LocalInventoryError.queryTimeout(database.path)) { + let _: [SnapshotValueRow] = try engine.query("WITH RECURSIVE loop(x) AS (VALUES(0) UNION ALL SELECT x + 1 FROM loop LIMIT 100000000) SELECT CAST(MAX(x) AS TEXT) AS value FROM loop;") + } + #expect(try FileManager.default.contentsOfDirectory(atPath: snapshots.path).isEmpty) +} + +@Test func productionSnapshotQueryEngineEnforcesConservativeTimeoutFloors() { + let engine = SQLiteSnapshotQueryEngine.production( + source: URL(fileURLWithPath: "/private/source.db"), + timeout: 0.001, + busyTimeoutMilliseconds: 0 + ) + + #expect(engine.timeout == SQLiteSnapshotQueryEngine.productionMinimumTimeout) + #expect(engine.busyTimeoutMilliseconds == SQLiteSnapshotQueryEngine.productionMinimumBusyTimeoutMilliseconds) +} + +@Test func snapshotQueryMapsCopyPermissionFailures() throws { + let root = try temporarySQLiteSnapshotDirectory(named: "permission") + defer { try? FileManager.default.removeItem(at: root) } + let database = root.appendingPathComponent("source.db") + try runSnapshotFixtureSQL(database: database, sql: "CREATE TABLE values_table (value TEXT);") + try FileManager.default.setAttributes([.posixPermissions: 0], ofItemAtPath: database.path) + defer { try? FileManager.default.setAttributes([.posixPermissions: 0o600], ofItemAtPath: database.path) } + + #expect(throws: LocalInventoryError.permissionDenied(database.path)) { + let _: [SnapshotValueRow] = try SQLiteSnapshotQueryEngine(source: database).query("SELECT value FROM values_table;") + } +} + +private final class WALFixtureWriter { + let process: Process + let input: FileHandle + + init(process: Process, input: FileHandle) { self.process = process; self.input = input } + + func stop() { + try? input.write(contentsOf: Data(".quit\n".utf8)) + try? input.close() + process.waitUntilExit() + } +} + +private func openWALFixture(database: URL) throws -> WALFixtureWriter { + let process = Process() + let input = Pipe() + process.executableURL = URL(fileURLWithPath: "/usr/bin/sqlite3") + process.arguments = [database.path] + process.standardInput = input + process.standardOutput = FileHandle.nullDevice + process.standardError = FileHandle.nullDevice + try process.run() + let commands = "PRAGMA journal_mode=WAL; PRAGMA wal_autocheckpoint=0; CREATE TABLE values_table (value TEXT); INSERT INTO values_table VALUES ('wal-row');\n" + try input.fileHandleForWriting.write(contentsOf: Data(commands.utf8)) + + let deadline = Date().addingTimeInterval(2) + while Date() < deadline { + if FileManager.default.fileExists(atPath: database.path + "-wal"), + (try? snapshotFixtureScalar(database: database, sql: "SELECT value FROM values_table;")) == "wal-row" { + break + } + Thread.sleep(forTimeInterval: 0.01) + } + return WALFixtureWriter(process: process, input: input.fileHandleForWriting) +} + +private func lockSnapshotDatabase(_ database: URL) throws -> WALFixtureWriter { + let process = Process() + let input = Pipe() + process.executableURL = URL(fileURLWithPath: "/usr/bin/sqlite3") + process.arguments = [database.path] + process.standardInput = input + process.standardOutput = FileHandle.nullDevice + process.standardError = FileHandle.nullDevice + try process.run() + try input.fileHandleForWriting.write(contentsOf: Data("PRAGMA locking_mode=EXCLUSIVE; BEGIN EXCLUSIVE; SELECT 1;\n".utf8)) + return WALFixtureWriter(process: process, input: input.fileHandleForWriting) +} + +private func snapshotFixtureScalar(database: URL, sql: String) throws -> String { + let process = Process() + let output = Pipe() + process.executableURL = URL(fileURLWithPath: "/usr/bin/sqlite3") + process.arguments = ["-readonly", database.path, sql] + process.standardOutput = output + process.standardError = FileHandle.nullDevice + try process.run() + process.waitUntilExit() + guard process.terminationStatus == 0 else { throw CocoaError(.fileReadUnknown) } + return String(decoding: output.fileHandleForReading.readDataToEndOfFile(), as: UTF8.self) + .trimmingCharacters(in: .whitespacesAndNewlines) +} + +private func temporarySQLiteSnapshotDirectory(named name: String) throws -> URL { + let url = FileManager.default.temporaryDirectory.appendingPathComponent("icloud-cli-snapshot-\(name)-\(UUID().uuidString)") + try FileManager.default.createDirectory(at: url, withIntermediateDirectories: true) + return url +} + +private func runSnapshotFixtureSQL(database: URL, sql: String) throws { + let process = Process() + process.executableURL = URL(fileURLWithPath: "/usr/bin/sqlite3") + process.arguments = [database.path, sql] + process.standardOutput = FileHandle.nullDevice + process.standardError = FileHandle.nullDevice + try process.run() + process.waitUntilExit() + guard process.terminationStatus == 0 else { throw CocoaError(.fileWriteUnknown) } +} diff --git a/docs/privacy.md b/docs/privacy.md index a98a721..f2c6dac 100644 --- a/docs/privacy.md +++ b/docs/privacy.md @@ -116,6 +116,8 @@ The second inventory wave adds local-only command trees for `account status`, `b These commands use preference/plist readers for account, backup, Family Sharing, permissions, and snapshot status, plus best-effort SQLite metadata readers for Apple private cache stores when a stable local table shape is known or supplied by fixtures. All adapters are read-only and local-only. They do not refresh iCloud, contact Apple services, control apps, trigger HomeKit accessories, execute shortcuts, read audio/photo/book/media payloads, or emit auth tokens. Private Apple schemas can vary across macOS releases, so commands fail closed when expected local tables are absent. +Messages and Safari history queries first copy the database and present WAL/SHM companions into a private temporary directory, query that copy in read-only/query-only mode with bounded busy and process timeouts, and delete it deterministically. See [sqlite-snapshots.md](sqlite-snapshots.md) for the migration and cleanup contract. + High-sensitivity gates: - `icloud-cli safari cloud-tabs list`, `icloud-cli mail recent`, and `icloud-cli health summary` require `--confirm-sensitive`. diff --git a/docs/sqlite-snapshots.md b/docs/sqlite-snapshots.md new file mode 100644 index 0000000..83af1ad --- /dev/null +++ b/docs/sqlite-snapshots.md @@ -0,0 +1,23 @@ +# Safe SQLite snapshots + +Apple applications often keep committed SQLite state across a database file and its write-ahead log. Querying the live path directly can produce inconsistent reads, wait indefinitely on a busy store, or interact with an Apple-owned database in ways the CLI does not intend. + +`SQLiteSnapshotQueryEngine` creates a mode-`0700` temporary directory, copies the database and any present `-wal` and `-shm` companions as mode-`0600` files, and queries only that private copy. The `sqlite3` process runs with `-readonly`, `PRAGMA query_only=ON`, a bounded busy timeout, and a bounded process timeout. Production callers use `SQLiteSnapshotQueryEngine.production`, which enforces minimums of 500ms busy timeout and 5s process timeout. The snapshot, query output, and error output are deleted before the call returns or throws. Copied contents are never logged. + +Errors retain the original store path and distinguish missing stores, permission denial, unsupported schemas, locked or busy stores, and timeouts. The implementation does not checkpoint, lock, vacuum, or write to the Apple-owned source. + +## SQLite and temporary-directory requirements + +WAL snapshots require SQLite 3.14 or later for reliable `-readonly` access; supported macOS releases meet this requirement. Snapshot directories are unique and private, but callers that invoke external `sqlite3` processes against a shared temporary root must coordinate their own process lifecycle and cleanup. + +## Migration status + +Messages conversations/recent messages and Safari history use the snapshot engine first because they read high-sensitivity, frequently changing databases. The remaining SQLite-backed providers still use the older read-only query path and should migrate incrementally with their provider-specific work: + +- Notes and Reminders +- Calendar, Contacts, Mail, and Maps +- Photos sharing and Find My metadata +- Books, Voice Memos, Home, Health, Freeform, Music, Stocks, Weather, and News +- Safari Cloud Tabs, profiles, and extensions where their selected store is SQLite + +Provider migrations should retain existing confirmation and redaction gates and add WAL-backed fixtures before switching paths.