Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 28 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -382,6 +382,34 @@ Tags are applied (or updated) every time the library is rescanned. Tags set via

---

## Ignoring Files with .grimoireignore

Add a `.grimoireignore` file to keep files on disk but out of Grimoire. It uses the same syntax as `.gitignore` / `.dockerignore`, so anything matched by a rule is skipped during scanning and never appears in the UI — useful when a book ships extra print variants (black-and-white single pages, zine-sized layouts) you want kept next to the book but hidden.

Place it at your **library root** to apply everywhere, or in any subfolder to add rules for just that subtree. Rules are cumulative and nested, like git.

```
library/
├── .grimoireignore ← applies to the whole library
└── books/
└── Example TTRPG/
├── core/
│ └── Players Handbook.pdf
└── ignore/ ← whole folder skipped
└── Players Handbook BW Single Pages.pdf
```

```
# .grimoireignore
ignore/ # skip an entire folder
*BW Single Pages*.pdf # skip print variants anywhere
!keep-this.pdf # re-include a file an earlier rule excluded
```

The full gitignore dialect is supported (`!` negation, `**` for arbitrary depth, anchoring with `/`), and rules apply to every collection: `books/`, `maps/`, `tokens/`, and `audio/`. Changes take effect on the next scan. Adding a rule that matches an already-indexed file hides it (marked missing) on the next rescan; remove the rule and rescan to bring it back.

---

## Adding Files to Your Library

Grimoire mounts your library folder **read-only** and never modifies your files. To upload, organize, or remove content, use a companion tool that mounts the same library folder with write access.
Expand Down
84 changes: 67 additions & 17 deletions backend/indexer.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
from sqlalchemy.orm import Session

from . import config, ocr
from .library_ignore import IgnoreMatcher
from .models import (
GameSystem,
Book,
Expand Down Expand Up @@ -802,14 +803,38 @@ def _commit(session: Session, label: str) -> None:
session.rollback()


def _count_eligible_files(directory: Path, extensions: set) -> int:
"""Count non-hidden files with matching extensions under directory."""
def _prune_dirs(root: str, dirs: list[str], ignore: Optional[IgnoreMatcher]) -> list[str]:
"""Return the walk subdirectories to descend into.

Drops hidden dirs (``.``-prefixed) and, when an ``ignore`` matcher is given,
any directory excluded by a ``.grimoireignore`` rule — pruning the whole
subtree so ignored folders are never walked.
"""
return [
d
for d in dirs
if not d.startswith(".")
and not (ignore and ignore.is_ignored(os.path.join(root, d), is_dir=True))
]


def _count_eligible_files(
directory: Path, extensions: set, ignore: Optional[IgnoreMatcher] = None
) -> int:
"""Count non-hidden files with matching extensions under directory.

When an ``ignore`` matcher is supplied, directories and files excluded by a
``.grimoireignore`` rule are skipped so the count matches what the scan will
actually process (keeping progress totals accurate).
"""
count = 0
for root, dirs, files in os.walk(directory):
dirs[:] = [d for d in dirs if not d.startswith(".")]
dirs[:] = _prune_dirs(root, dirs, ignore)
for f in files:
if f.startswith("."):
continue
if ignore and ignore.is_ignored(os.path.join(root, f), is_dir=False):
continue
if Path(f).suffix.lower() in extensions or archive_ext(f) in extensions:
count += 1
return count
Expand Down Expand Up @@ -1130,6 +1155,10 @@ def scan_library(
scope_section, scope_dir = resolve_scope(library_path, scope_path)
logger.debug(f"Scoped scan: section={scope_section}, dir={scope_dir}, mode={metadata_mode}")

# Matcher for .grimoireignore rules across the whole library tree (issue
# #224). Built once from the library root; queried per path in each walk.
ignore = IgnoreMatcher(library_path)

scan_books = scope_section in (None, "books")
scan_maps = scope_section in (None, "maps")
scan_tokens = scope_section in (None, "tokens")
Expand All @@ -1142,22 +1171,22 @@ def scan_library(
audio_walk_dir = scope_dir if scope_section == "audio" else audio_dir

total_books = (
_count_eligible_files(books_walk_dir, DOC_EXTS | IMAGE_EXTS | ARCHIVE_EXTS)
_count_eligible_files(books_walk_dir, DOC_EXTS | IMAGE_EXTS | ARCHIVE_EXTS, ignore)
if scan_books and books_walk_dir.exists()
else 0
)
total_maps = (
_count_eligible_files(maps_walk_dir, MAP_IMAGE_EXTS)
_count_eligible_files(maps_walk_dir, MAP_IMAGE_EXTS, ignore)
if scan_maps and maps_walk_dir.exists()
else 0
)
total_tokens = (
_count_eligible_files(tokens_walk_dir, IMAGE_EXTS)
_count_eligible_files(tokens_walk_dir, IMAGE_EXTS, ignore)
if scan_tokens and tokens_walk_dir.exists()
else 0
)
total_audio = (
_count_eligible_files(audio_walk_dir, AUDIO_EXTS)
_count_eligible_files(audio_walk_dir, AUDIO_EXTS, ignore)
if scan_audio and audio_walk_dir.exists()
else 0
)
Expand Down Expand Up @@ -1236,7 +1265,7 @@ def scan_library(
scope_dir if (scope_section == "books" and len(scope_parts) > 1) else system_dir
)
for root, dirs, files in os.walk(walk_root):
dirs[:] = [d for d in dirs if not d.startswith(".")]
dirs[:] = _prune_dirs(root, dirs, ignore)

# Collect cover image filenames declared in any OPF files in this
# directory so we can skip them — Calibre exports a cover JPG that
Expand All @@ -1260,6 +1289,10 @@ def scan_library(
if ext not in DOC_EXTS and ext not in IMAGE_EXTS and not arc_ext:
continue

if ignore.is_ignored(filepath, is_dir=False):
logger.debug(f"Ignored by .grimoireignore: {filepath}")
continue

if filename in opf_cover_filenames:
logger.debug(f"Skipping OPF cover image: {filepath}")
continue
Expand Down Expand Up @@ -1475,7 +1508,7 @@ def scan_library(

if scan_maps and maps_walk_dir.exists():
for root, dirs, files in os.walk(maps_walk_dir):
dirs[:] = [d for d in dirs if not d.startswith(".")]
dirs[:] = _prune_dirs(root, dirs, ignore)

for filename in sorted(files):
if filename.startswith("."):
Expand All @@ -1487,6 +1520,10 @@ def scan_library(
if ext not in MAP_IMAGE_EXTS:
continue

if ignore.is_ignored(filepath, is_dir=False):
logger.debug(f"Ignored by .grimoireignore: {filepath}")
continue

scanned_maps += 1
if on_progress:
on_progress(
Expand Down Expand Up @@ -1563,7 +1600,7 @@ def scan_library(

if scan_tokens and tokens_walk_dir.exists():
for root, dirs, files in os.walk(tokens_walk_dir):
dirs[:] = [d for d in dirs if not d.startswith(".")]
dirs[:] = _prune_dirs(root, dirs, ignore)

for filename in sorted(files):
if filename.startswith("."):
Expand All @@ -1575,6 +1612,10 @@ def scan_library(
if ext not in IMAGE_EXTS:
continue

if ignore.is_ignored(filepath, is_dir=False):
logger.debug(f"Ignored by .grimoireignore: {filepath}")
continue

scanned_tokens += 1
if on_progress:
on_progress(
Expand Down Expand Up @@ -1651,7 +1692,7 @@ def scan_library(

if scan_audio and audio_walk_dir.exists():
for root, dirs, files in os.walk(audio_walk_dir):
dirs[:] = [d for d in dirs if not d.startswith(".")]
dirs[:] = _prune_dirs(root, dirs, ignore)

for filename in sorted(files):
if filename.startswith("."):
Expand All @@ -1663,6 +1704,10 @@ def scan_library(
if ext not in AUDIO_EXTS:
continue

if ignore.is_ignored(filepath, is_dir=False):
logger.debug(f"Ignored by .grimoireignore: {filepath}")
continue

scanned_audio += 1
if on_progress:
on_progress(
Expand Down Expand Up @@ -1737,8 +1782,10 @@ def scan_library(
# --- Mark / unmark missing files ---
# After walking the filesystem, any DB record whose file is gone gets
# is_missing=True; records that exist on disk have is_missing cleared.
# When scoped, only reconcile records under the scope subtree so unrelated
# corners of the library are left untouched.
# A file newly matched by a ``.grimoireignore`` rule (still on disk but now
# excluded) is treated as gone too, so it disappears from the UI; clearing
# the rule brings it back on the next scan. When scoped, only reconcile
# records under the scope subtree so unrelated corners are left untouched.
if should_stop and should_stop():
return stats

Expand All @@ -1747,34 +1794,37 @@ def _scoped(query: Any, model: Any) -> Any:
return query.filter(model.filepath.like(f"{scope_dir}{os.sep}%"))
return query

def _gone(filepath: str) -> bool:
return not os.path.exists(filepath) or ignore.is_ignored(filepath, is_dir=False)

missing_books = missing_maps = missing_tokens = missing_audio = 0
if scan_books:
for book in _scoped(session.query(Book), Book).all():
gone = not os.path.exists(book.filepath)
gone = _gone(book.filepath)
if gone != bool(book.is_missing):
book.is_missing = gone
if gone:
missing_books += 1
logger.warning(f"Missing book: '{book.title}' ({book.filepath})")
if scan_maps:
for m in _scoped(session.query(GenericMap), GenericMap).all():
gone = not os.path.exists(m.filepath)
gone = _gone(m.filepath)
if gone != bool(m.is_missing):
m.is_missing = gone
if gone:
missing_maps += 1
logger.warning(f"Missing map: '{m.filename}' ({m.filepath})")
if scan_tokens:
for t in _scoped(session.query(Token), Token).all():
gone = not os.path.exists(t.filepath)
gone = _gone(t.filepath)
if gone != bool(t.is_missing):
t.is_missing = gone
if gone:
missing_tokens += 1
logger.warning(f"Missing token: '{t.filename}' ({t.filepath})")
if scan_audio:
for a in _scoped(session.query(Audio), Audio).all():
gone = not os.path.exists(a.filepath)
gone = _gone(a.filepath)
if gone != bool(a.is_missing):
a.is_missing = gone
if gone:
Expand Down
109 changes: 109 additions & 0 deletions backend/library_ignore.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
"""``.grimoireignore`` support for the library scanner (issue #224).

A ``.grimoireignore`` file uses the same syntax as ``.gitignore`` /
``.dockerignore`` (git's *gitignore* dialect): one pattern per line, blank
lines and ``#`` comments ignored, ``!`` negation to re-include, ``/`` to anchor
or mark a directory, and ``**`` for arbitrary-depth matching.

Files and directories matched by an ignore rule are skipped by the scanner, so
they never appear in the library UI. Rules are **cumulative and nested** like
git: a ``.grimoireignore`` at the library root applies everywhere, and one in a
subfolder adds rules for its own subtree. Each file's patterns are matched
relative to the directory that file lives in.

The public entry point is :class:`IgnoreMatcher`, built once per scan from the
library root and queried per path with :meth:`IgnoreMatcher.is_ignored`.
"""

import logging
import os
from pathlib import Path
from typing import Optional

from pathspec import PathSpec

logger = logging.getLogger("grimoire.indexer")

# Name of the per-directory ignore file, mirroring ``.gitignore``.
IGNORE_FILENAME = ".grimoireignore"


class IgnoreMatcher:
"""Resolves ``.grimoireignore`` rules for paths under a library root.

Ignore files are loaded lazily and cached per directory, so a full scan
reads each ``.grimoireignore`` at most once. Matching walks from the
library root down to the path's own directory, applying each directory's
patterns relative to that directory (git semantics), with deeper and
later rules overriding shallower ones — including ``!`` re-inclusion.
"""

def __init__(self, library_root: str) -> None:
# Resolve so ancestor computation is stable regardless of how callers
# spell the root (trailing slash, symlinks in the walked paths, …).
self._root = Path(library_root).resolve()
# dir (resolved) -> compiled PathSpec, or None when that dir has no
# ignore file. None is cached too, so a missing file is stat'd once.
self._cache: dict[Path, Optional[PathSpec]] = {}

def _spec_for_dir(self, directory: Path) -> Optional[PathSpec]:
"""Load and cache the PathSpec for a single directory's ignore file."""
if directory in self._cache:
return self._cache[directory]
spec: Optional[PathSpec] = None
ignore_file = directory / IGNORE_FILENAME
try:
if ignore_file.is_file():
lines = ignore_file.read_text(encoding="utf-8").splitlines()
spec = PathSpec.from_lines("gitignore", lines)
except OSError as exc:
logger.warning(f"Could not read {ignore_file}: {exc}")
self._cache[directory] = spec
return spec

def _ancestors(self, directory: Path) -> list[Path]:
"""Return [root, …, directory], shallowest first, or [] if outside root."""
try:
directory = directory.resolve()
except OSError:
return []
if directory != self._root and self._root not in directory.parents:
return []
chain = [directory]
while chain[-1] != self._root:
chain.append(chain[-1].parent)
chain.reverse()
return chain

def is_ignored(self, path: str, is_dir: bool) -> bool:
"""Return True if `path` is excluded by any applicable ignore rule.

`path` is an absolute filesystem path. `is_dir` must be True for
directories so that directory-only patterns (``foo/``) match correctly.
Deeper directories' rules are evaluated after shallower ones, so a
nested ``.grimoireignore`` can override (including re-include via ``!``)
what an ancestor ignored; the last matching rule wins, as in git.
"""
# Resolve so the path is spelled the same way as the ancestor chain
# (which is derived from the resolved library root); otherwise a symlink
# in the root (e.g. macOS ``/var`` -> ``/private/var``) makes relpath
# produce a bogus ``../..`` prefix that never matches. resolve() is
# non-strict, so it works even for the removed-file reconciliation case
# where the path no longer exists on disk.
try:
p = Path(path).resolve()
except OSError:
p = Path(path)
result = False
for ancestor in self._ancestors(p.parent):
spec = self._spec_for_dir(ancestor)
if spec is None:
continue
# Patterns in `ancestor`'s ignore file are relative to `ancestor`.
rel = os.path.relpath(str(p), str(ancestor)).replace(os.sep, "/")
if is_dir:
rel += "/"
match = spec.check_file(rel)
if match.include is not None:
result = match.include
return result
1 change: 1 addition & 0 deletions backend/requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -20,4 +20,5 @@ mutagen==1.47.0
rarfile==4.2
py7zr==1.1.3
pytesseract==0.3.13
pathspec==1.1.1
tzdata==2025.2
Loading
Loading