diff --git a/scripts/lavafuncs.py b/scripts/lavafuncs.py index b8926f37..09a8668c 100644 --- a/scripts/lavafuncs.py +++ b/scripts/lavafuncs.py @@ -36,6 +36,7 @@ from scripts.version_info import rleapp_version from scripts.context import Context +from scripts.storage_safety import configure_lava_journal_mode, finalize_lava_journal_mode # Global variables lava_data = None @@ -44,6 +45,15 @@ lava_json_name = '_lava_data.lava' +def _get_logfunc(): + """Import ilapfuncs.logfunc lazily to avoid a circular import at load time.""" + try: + from scripts.ilapfuncs import logfunc + return logfunc + except ImportError: + return None + + def sanitize_sql_name(name): """ Sanitizes a given name by removing invalid characters and formatting it. @@ -118,6 +128,22 @@ def initialize_lava(input_path, output_path, input_type): db_path = os.path.join(output_path, lava_db_name) lava_db = sqlite3.connect(db_path) + # Performance: the media insert helpers commit() per row, so a media-heavy + # artifact (e.g. a device backup with tens of thousands of files) triggers + # tens of thousands of fsync'd commits. Default rollback-journal + + # synchronous=FULL makes each commit fsync the whole DB, which can take many + # minutes. WAL + synchronous=NORMAL keeps the same durability for a tool run + # (only a power-loss in the final checkpoint window could lose the tail) and + # is dramatically faster. + # + # WAL is NOT safe over a network filesystem (https://www.sqlite.org/wal.html), + # and examiners commonly write LEAPP output straight to NAS/mapped drives. + # configure_lava_journal_mode enables WAL only when output_path is confirmed + # local, stays on the network-safe rollback journal otherwise, and verifies + # WAL actually took effect before tuning synchronous mode. + # lava_finalize_output() switches back to the delete journal so the + # delivered .db is standalone (no -wal/-shm) and readable from read-only media. + configure_lava_journal_mode(lava_db, output_path, logfunc=_get_logfunc()) cursor = lava_db.cursor() cursor.execute('''CREATE TABLE _artifact_search_patterns ( @@ -662,5 +688,10 @@ def lava_finalize_output(output_path): with open(os.path.join(output_path, lava_json_name), 'w', encoding='utf-8') as f: json.dump(lava_data, f, indent=4) + # If WAL was enabled for the run, checkpoint it and return to the delete + # journal so the delivered .db file is complete, standalone (no -wal/-shm + # sidecars) and openable from read-only media by the LAVA viewer. + finalize_lava_journal_mode(lava_db, logfunc=_get_logfunc()) + # Close the SQLite database lava_db.close() diff --git a/scripts/storage_safety.py b/scripts/storage_safety.py new file mode 100644 index 00000000..6148fc64 --- /dev/null +++ b/scripts/storage_safety.py @@ -0,0 +1,320 @@ +""" +storage_safety.py — Network-aware SQLite journal mode selection. + +WAL (Write-Ahead Logging) mode significantly improves SQLite write performance, +but per the SQLite documentation it is NOT safe over network filesystems: + + https://www.sqlite.org/wal.html + "All processes using a database must be on the same host computer; WAL does + not work over a network filesystem." + +Forensic examiners commonly write LEAPP output directly to network storage +(NAS/SAN, mapped drives, UNC paths). Enabling WAL on such a path risks database +corruption or a crashed run — unacceptable when processing evidence. + +Strategy (safe by default): + 1. Only enable WAL when the output path can be AFFIRMATIVELY confirmed local. + 2. If the path is network-backed, or if we cannot determine its nature with + confidence, stay on SQLite's default rollback journal (works everywhere). + 3. After attempting to set WAL, verify the journal mode actually took effect + before tuning synchronous mode (WAL can silently fall back on some + filesystems). This is the backstop suggested by @JamesHabben. + 4. Before the report is finalized, switch the database back to the delete + (rollback) journal mode. journal_mode=WAL is persisted in the database + header, and a WAL-mode file cannot be opened from read-only media (SQLite + needs to create the -shm file), which would break read-only consumers such + as the LAVA viewer (sqlite3.OPEN_READONLY) when the report is archived or + write-blocked. Leaving WAL mode also checkpoints and removes the -wal/-shm + sidecars, guaranteeing the .db file is complete and standalone. +""" + +import os +import re +import subprocess +import sys + + +# Filesystem types that indicate network-backed storage on Unix-like systems. +_NETWORK_FSTYPES = frozenset({ + 'nfs', 'nfs4', 'cifs', 'smbfs', 'smb3', 'fuse.sshfs', 'afpfs', + 'ncpfs', 'fuse.s3fs', 'fuse.rclone', '9p', 'fuse.glusterfs', + 'lustre', 'ceph', 'gfs', 'gfs2', 'ocfs2', +}) + + +# Windows extended-length path prefixes. These are NOT network indicators: +# \\?\C:\dir -> a LOCAL drive-letter path (the \\?\ just lifts MAX_PATH) +# \\?\UNC\srv\share -> a UNC (network) path in extended-length form +# LEAPP prepends \\?\ to every drive-letter output path (rleapp.py), so the path +# handed to us is routinely extended-length and must be normalized before the +# leading-\\ UNC test, or a local \\?\X:\... is misread as a network share. +_EXT_PREFIX = '\\\\?\\' # \\?\ +_EXT_UNC_PREFIX = '\\\\?\\UNC\\' # \\?\UNC\ + + +def _strip_extended_prefix(path): + r"""Normalize a Windows extended-length path to its plain form. + + \\?\UNC\server\share -> \\server\share (still UNC/network) + \\?\C:\dir -> C:\dir (plain drive-letter) + Anything else is returned unchanged. + """ + p = str(path) + if p.startswith(_EXT_UNC_PREFIX): + return '\\\\' + p[len(_EXT_UNC_PREFIX):] + if p.startswith(_EXT_PREFIX): + return p[len(_EXT_PREFIX):] + return p + + +def _is_unc_path(path): + r"""True if path is a UNC network path (\\server\share or //server/share). + + The Windows extended-length prefix is stripped first so that a local + \\?\: path is not mistaken for UNC, while \\?\UNC\... is preserved + as UNC. + """ + p = _strip_extended_prefix(path) + return p.startswith('\\\\') or p.startswith('//') + + +def _windows_path_is_network(path): + """ + On Windows, determine whether path resides on a network drive. + + Returns True (network), False (local), or None (undetermined). + Uses the Win32 API GetDriveType against the path's drive root. + UNC paths are treated as network without an API call. + """ + path = _strip_extended_prefix(path) + if _is_unc_path(path): + return True + + try: + import ctypes + + abs_path = os.path.abspath(path) + drive, _ = os.path.splitdrive(abs_path) + if not drive: + return None # no drive letter and not UNC — undetermined + drive_root = drive + '\\' + + # GetDriveTypeW returns: + # 0 DRIVE_UNKNOWN, 1 DRIVE_NO_ROOT_DIR, 2 DRIVE_REMOVABLE, + # 3 DRIVE_FIXED, 4 DRIVE_REMOTE, 5 DRIVE_CDROM, 6 DRIVE_RAMDISK + DRIVE_REMOTE = 4 + drive_type = ctypes.windll.kernel32.GetDriveTypeW(ctypes.c_wchar_p(drive_root)) + if drive_type == DRIVE_REMOTE: + return True + if drive_type in (3, 2, 5, 6): # FIXED, REMOVABLE, CDROM, RAMDISK = local + return False + return None # UNKNOWN / NO_ROOT_DIR — undetermined + except (ImportError, AttributeError, OSError, ValueError): + return None # API unavailable or error — undetermined + + +def _unix_path_is_network(path, mounts_text=None): + """ + On Unix-like systems, determine whether path resides on a network mount by + finding the longest matching mount point in /proc/mounts and checking its + filesystem type. + + Returns True (network), False (local), or None (undetermined). + mounts_text is injectable for testing. + """ + try: + if mounts_text is None: + # /proc/mounts is Linux. macOS/BSD lack it; fall back to undetermined. + if not os.path.exists('/proc/mounts'): + return None + with open('/proc/mounts', 'r', encoding='utf-8', errors='replace') as f: + mounts_text = f.read() + + abs_path = os.path.abspath(path) + best_mount = '' + best_fstype = '' + for line in mounts_text.strip().splitlines(): + parts = line.split() + if len(parts) < 3: + continue + mount_point, fstype = parts[1], parts[2] + if abs_path == mount_point or abs_path.startswith(mount_point.rstrip('/') + '/'): + if len(mount_point) >= len(best_mount): + best_mount = mount_point + best_fstype = fstype + + if not best_fstype: + return None + return best_fstype.lower() in _NETWORK_FSTYPES + except OSError: + return None + + +# Matches macOS/BSD `mount` output lines: on () +# The first comma-separated token in is the filesystem type, and +# local filesystems carry the 'local' option (the MNT_LOCAL statfs flag). +_MOUNT_LINE_RE = re.compile(r'^.+ on (.+) \(([^)]*)\)$') + + +def _macos_path_is_network(path, mount_text=None): + """ + On macOS (and BSDs with the same `mount` output format), determine whether + path resides on a network mount by finding the longest matching mount point + in `mount` output and checking its options. + + A local filesystem advertises the 'local' option (the MNT_LOCAL flag), e.g. + /dev/disk3s5 on /System/Volumes/Data (apfs, local, journaled, nobrowse) + while network mounts do not, e.g. + //user@server/share on /Volumes/share (smbfs, nodev, nosuid) + + Returns True (network), False (local), or None (undetermined). + mount_text is injectable for testing. + """ + try: + if mount_text is None: + mount_text = subprocess.run( + ['/sbin/mount'], capture_output=True, text=True, timeout=10, + check=True).stdout + + abs_path = os.path.abspath(path) + best_mount = None + best_opts = () + for line in mount_text.strip().splitlines(): + match = _MOUNT_LINE_RE.match(line.strip()) + if not match: + continue + mount_point, opts = match.group(1), match.group(2) + if abs_path == mount_point or abs_path.startswith(mount_point.rstrip('/') + '/'): + if best_mount is None or len(mount_point) >= len(best_mount): + best_mount = mount_point + best_opts = tuple(opt.strip() for opt in opts.split(',')) + + if best_mount is None: + return None + if 'local' in best_opts: + return False # MNT_LOCAL flag — affirmatively local + fstype = best_opts[0].lower() if best_opts else '' + if fstype in _NETWORK_FSTYPES: + return True + return None # not flagged local, type unrecognized — undetermined + except (OSError, subprocess.SubprocessError): + return None + + +def path_is_network(path): + """ + Best-effort determination of whether path is on network-backed storage. + + Returns: + True — confirmed network storage + False — confirmed local storage + None — could not determine (caller should treat as unsafe for WAL) + """ + # Normalize away the Windows extended-length prefix first so a local + # \\?\: path is not misread as UNC. + path = _strip_extended_prefix(path) + + # UNC is network on any platform that understands it + if _is_unc_path(path): + return True + + if sys.platform == 'win32': + return _windows_path_is_network(path) + if sys.platform == 'darwin': + return _macos_path_is_network(path) + return _unix_path_is_network(path) + + +def configure_lava_journal_mode(lava_db, output_path, logfunc=None): + """ + Configure the SQLite journal mode for the LAVA database safely. + + Enables WAL only when output_path is AFFIRMATIVELY local. On network or + undetermined paths, leaves SQLite on its default rollback journal, which is + safe over network filesystems. + + After attempting WAL, verifies the mode actually took effect before setting + synchronous=NORMAL (WAL can silently fall back on some filesystems). + + Args: + lava_db: an open sqlite3 Connection (or cursor-capable object). + output_path: the path where the LAVA database is being written. + logfunc: optional logging callable (e.g. ilapfuncs.logfunc). + + Returns: + The effective journal mode string (e.g. 'wal', 'delete'). + """ + def _log(msg): + if logfunc: + logfunc(msg) + + network = path_is_network(output_path) + + if network is True: + _log('LAVA DB: output path is network-backed storage; ' + 'keeping default journal mode (WAL unsafe over network per SQLite docs).') + # Ensure a safe, network-compatible journal mode explicitly. + effective = lava_db.execute("PRAGMA journal_mode=DELETE").fetchone()[0] + return effective.lower() + + if network is None: + _log('LAVA DB: could not determine if output path is local or network; ' + 'defaulting to safe journal mode (no WAL).') + effective = lava_db.execute("PRAGMA journal_mode=DELETE").fetchone()[0] + return effective.lower() + + # network is False — confirmed local storage; safe to attempt WAL. + effective = lava_db.execute("PRAGMA journal_mode=WAL").fetchone()[0] + if effective.lower() == 'wal': + # WAL took effect — NORMAL sync is safe and faster under WAL. + lava_db.execute("PRAGMA synchronous=NORMAL") + _log('LAVA DB: local storage confirmed; WAL journaling enabled ' + '(synchronous=NORMAL).') + else: + # WAL did not take effect (filesystem refused it). Leave sync at default. + _log(f'LAVA DB: WAL requested but filesystem returned journal_mode=' + f'{effective!r}; leaving synchronous at default for safety.') + return effective.lower() + + +def finalize_lava_journal_mode(lava_db, logfunc=None): + """ + Return the LAVA database to the delete (rollback) journal mode before the + report is finalized. + + journal_mode=WAL is persisted in the database header. If the delivered + .db file kept it, the report database could not be opened from read-only + media (write-blocked storage, archived/burned reports) by read-only + consumers such as the LAVA viewer, because SQLite must create a -shm file + to read a WAL database. Switching back to DELETE also checkpoints the WAL + and removes the -wal/-shm sidecar files, so the .db is complete and + standalone no matter how the report folder is copied or zipped. + + Safe to call regardless of the current journal mode. + + Args: + lava_db: an open sqlite3 Connection. + logfunc: optional logging callable (e.g. ilapfuncs.logfunc). + + Returns: + The effective journal mode string (expected 'delete'). + """ + def _log(msg): + if logfunc: + logfunc(msg) + + current = lava_db.execute("PRAGMA journal_mode").fetchone()[0].lower() + if current != 'wal': + return current + + # Flush the write-ahead log into the main database file, then leave WAL + # mode (which resets the header and deletes the -wal/-shm sidecars). + lava_db.execute("PRAGMA wal_checkpoint(TRUNCATE)") + effective = lava_db.execute("PRAGMA journal_mode=DELETE").fetchone()[0].lower() + if effective == 'delete': + _log('LAVA DB: WAL checkpointed and journal mode reset to DELETE; ' + 'database file is complete and standalone.') + else: + _log(f'LAVA DB: could not leave WAL mode (journal_mode={effective!r}); ' + f'keep the -wal/-shm files with the database if copying the report.') + return effective