fix: exFAT scan truncation and ScanStageProperties migration

pkg/database/mediadb/mediadb_integration_test.go

@@ -187,6 +187,11 @@ func TestMediaTagCompositeLookupAndDelete(t *testing.T) {
 	assert.ErrorIs(t, err, sql.ErrNoRows)
 }
 
+// browseSortCollationPrevVersion is the migration immediately before
+// 20260830140000_browse_sort_collation.sql, the point a downgrade past the
+// collation must reach.
+const browseSortCollationPrevVersion int64 = 20260825120000
+
 // The browse sort collation must carry a schema version bump: without one, a
 // build that lacks ZAPAROO_TITLE_V1 opens the database happily and then fails
 // on the first prepare against Media, and the rebuild that exists for an
@@ -230,7 +235,11 @@ func TestMigrations_BrowseSortCollationBumpsVersionWithoutTableWork(t *testing.T
 	// version but cannot prepare statements against Media.
 	goose.SetBaseFS(migrationFiles)
 	require.NoError(t, goose.SetDialect("sqlite"))
-	require.NoError(t, goose.Down(sqlDB, "migrations"))
+	// DownTo the version before the collation migration, not a bare Down:
+	// Down reverts whatever migration happens to be last, so any migration
+	// added afterwards would silently stop this from testing the downgrade it
+	// is named for.
+	require.NoError(t, goose.DownTo(sqlDB, "migrations", browseSortCollationPrevVersion))
 
 	require.NoError(t, sqlDB.QueryRowContext(ctx,
 		"SELECT sql FROM sqlite_master WHERE type = 'index' AND name = ?", browseSortIndexName,

pkg/database/mediadb/migrations/20260901120000_scan_stage_properties.sql

@@ -0,0 +1,21 @@
+-- +goose Up
+-- ScanStageProperties was added to the scanner staging set after the
+-- 20260703080000 migration shipped, and only ever existed because
+-- sqlClearScanStage failed on the missing table and called
+-- sqlEnsureScanStagingTables to repair the scratch schema. Every fresh
+-- media.db took that error path on its first system. Definitions match
+-- sqlEnsureScanStagingTables so the repair path stays a no-op.
+
+CREATE TABLE IF NOT EXISTS ScanStageProperties (
+    Path         TEXT NOT NULL,
+    PropertyType TEXT NOT NULL,
+    Property     TEXT NOT NULL,
+    Text         TEXT NOT NULL,
+    PRIMARY KEY (Path, PropertyType, Property)
+) WITHOUT ROWID;
+CREATE INDEX IF NOT EXISTS scanstageproperties_property_idx
+    ON ScanStageProperties(PropertyType, Property, Text);
+
+-- +goose Down
+DROP INDEX IF EXISTS scanstageproperties_property_idx;
+DROP TABLE IF EXISTS ScanStageProperties;

pkg/database/mediascanner/mediascanner.go

@@ -737,6 +737,27 @@ func entryIsSymlink(
 	return err == nil && info.Mode()&os.ModeSymlink != 0
 }
 
+// skipEntry returns the sentinel that actually skips this entry, and nothing
+// more. fastwalk only honours filepath.SkipDir for an entry it typed as a
+// directory (enqueued, so the callback runs from walker.walk, which swallows
+// the sentinel) or as a symlink (walker.onDirEnt has an explicit escape
+// hatch). Returned for anything else, the sentinel escapes readDir and aborts
+// iteration of the whole containing directory, dropping every entry after this
+// one.
+//
+// That is not hypothetical on exFAT and FAT: the dirent carries no symlink
+// bit there, so direntTypesReportSymlinks is false and entryIsSymlink finds
+// links by lstat while fastwalk still sees a regular file. A MiSTer log
+// carried 1450 truncated directories under _Arcade/_alternatives from exactly
+// this. Returning nil is the correct skip in that case, because fastwalk never
+// descends into an entry it typed as a regular file.
+func skipEntry(d fs.DirEntry) error {
+	if d.IsDir() || d.Type()&os.ModeSymlink != 0 {
+		return filepath.SkipDir
+	}
+	return nil
+}
+
 // shouldSkipSymlinkAlias reports whether a symlink must be kept out of the
 // walk because it aliases media already scanned under its target path. A
 // timeout while reading the link leaves its target unknown, but letting
@@ -842,7 +863,7 @@ func GetFiles(
 					Str("system", systemID).
 					Str("path", p).
 					Msg("skipping launcher-excluded scan directory")
-				return filepath.SkipDir
+				return skipEntry(d)
 			}
 		}
 
@@ -861,7 +882,7 @@ func GetFiles(
 					Str("system", systemID).
 					Str("path", p).
 					Msg("skipping symlink alias of scanned media")
-				return filepath.SkipDir
+				return skipEntry(d)
 			}
 		}
 

pkg/database/mediascanner/walk_skipdir_test.go

@@ -94,3 +94,136 @@ func TestWalkSkipDirFromFileKeepsSiblings(t *testing.T) {
 		})
 	}
 }
+
+// TestSkipEntry covers the three shapes an entry reaches the walk callback as.
+// The regular-file case is the one that matters: on exFAT and FAT a symlink
+// arrives typed as a regular file, and only nil skips it without taking the
+// rest of its directory with it.
+func TestSkipEntry(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		want error
+		name string
+		mode fs.FileMode
+	}{
+		{name: "directory", mode: fs.ModeDir, want: filepath.SkipDir},
+		{name: "symlink", mode: fs.ModeSymlink, want: filepath.SkipDir},
+		{name: "regular file", mode: 0, want: nil},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+			assert.Equal(t, tt.want, skipEntry(fakeDirEntry{name: "entry", mode: tt.mode}))
+		})
+	}
+}
+
+// walkReturningForAlias walks root, calling decide for the entry named
+// aliasName and returning nil for everything else. It reports the entries the
+// callback saw and any error handed back to the callback. SortLexical makes
+// readdir order deterministic; the sentinel propagates identically under
+// SortNone, which is what GetFiles uses.
+func walkReturningForAlias(
+	t *testing.T,
+	root, aliasName string,
+	workers int,
+	decide func(fs.DirEntry) error,
+) ([]string, []error) {
+	t.Helper()
+
+	var mu syncutil.Mutex
+	var seen []string
+	var walkErrs []error
+
+	conf := &fastwalk.Config{Follow: true, NumWorkers: workers, Sort: fastwalk.SortLexical}
+	err := fastwalk.Walk(conf, root, func(p string, d fs.DirEntry, walkErr error) error {
+		if walkErr != nil {
+			mu.Lock()
+			walkErrs = append(walkErrs, walkErr)
+			mu.Unlock()
+			return nil
+		}
+		rel, relErr := filepath.Rel(root, p)
+		require.NoError(t, relErr)
+		if rel == "." {
+			return nil
+		}
+		mu.Lock()
+		seen = append(seen, rel)
+		mu.Unlock()
+		if rel == aliasName {
+			return decide(d)
+		}
+		return nil
+	})
+	require.NoError(t, err)
+
+	mu.Lock()
+	defer mu.Unlock()
+	sort.Strings(seen)
+	return append([]string(nil), seen...), append([]error(nil), walkErrs...)
+}
+
+// aliasDirFixture builds a directory holding an entry that sorts first,
+// followed by real media. On exFAT the alias would be a symlink the dirent
+// reports as a regular file; a plain file reproduces the type fastwalk sees.
+func aliasDirFixture(t *testing.T) (root, alias string, realNames []string) {
+	t.Helper()
+
+	root = t.TempDir()
+	alias = "00-alias.rom"
+	require.NoError(t, os.WriteFile(filepath.Join(root, alias), []byte("x"), 0o600))
+
+	realNames = []string{"a-real.rom", "b-real.rom", "c-real.rom", "d-real.rom"}
+	for _, n := range realNames {
+		require.NoError(t, os.WriteFile(filepath.Join(root, n), []byte("x"), 0o600))
+	}
+	return root, alias, realNames
+}
+
+// TestWalkSkipDirFromUntypedEntryTruncatesDirectory pins the fastwalk
+// behaviour skipEntry exists to work around. When the callback returns
+// filepath.SkipDir for an entry fastwalk typed as a regular file, the sentinel
+// escapes readDir, the rest of the directory is never visited, and it comes
+// back to the callback as a walk error. If fastwalk ever handles this the way
+// it handles typed symlinks, this test fails and skipEntry can go.
+func TestWalkSkipDirFromUntypedEntryTruncatesDirectory(t *testing.T) {
+	t.Parallel()
+
+	root, alias, realNames := aliasDirFixture(t)
+
+	seen, walkErrs := walkReturningForAlias(t, root, alias, 1, func(fs.DirEntry) error {
+		return filepath.SkipDir
+	})
+
+	for _, n := range realNames {
+		assert.NotContains(t, seen, n,
+			"raw SkipDir on an untyped entry is expected to drop the rest of the directory")
+	}
+	require.Len(t, walkErrs, 1)
+	assert.ErrorIs(t, walkErrs[0], filepath.SkipDir)
+}
+
+// TestSkipEntryKeepsSiblingsWhenDirentUntyped is the exFAT regression: the
+// symlink-alias branch skips one entry and every sibling after it still gets
+// scanned.
+func TestSkipEntryKeepsSiblingsWhenDirentUntyped(t *testing.T) {
+	t.Parallel()
+
+	for _, workers := range []int{1, 4} {
+		t.Run("workers", func(t *testing.T) {
+			t.Parallel()
+
+			root, alias, realNames := aliasDirFixture(t)
+			seen, walkErrs := walkReturningForAlias(t, root, alias, workers, skipEntry)
+
+			for _, n := range realNames {
+				assert.Contains(t, seen, n,
+					"skipping an alias must not drop its siblings")
+			}
+			assert.Empty(t, walkErrs)
+		})
+	}
+}