From 92800626ad8cf509be8d67080d9f3cd5c65cb30a Mon Sep 17 00:00:00 2001 From: Hoseong Liam Hwang <449334+thefron@users.noreply.github.com> Date: Fri, 13 Nov 2026 23:20:19 -0800 Subject: [PATCH 1/5] fix(secrets): re-read findings when the file or the rule set changes Co-Authored-By: Claude Opus 5.5 --- architecture/cli-sync.md | 2 +- cmd/bdrive/hooksync_test.go | 30 ++++------- internal/secrets/secrets.go | 21 +++++++- internal/secrets/secrets_test.go | 8 ++- internal/store/secrets.go | 93 +++++++++++++++++++++++++++----- internal/store/secrets_test.go | 40 ++++++++++++++ internal/syncer/secrets.go | 56 ++++++++++++++++--- internal/syncer/syncer.go | 9 ++-- internal/syncer/syncer_test.go | 64 ++++++++++++++++++++++ 9 files changed, 276 insertions(+), 47 deletions(-) diff --git a/architecture/cli-sync.md b/architecture/cli-sync.md index 5072c7bd..fe61eaac 100644 --- a/architecture/cli-sync.md +++ b/architecture/cli-sync.md @@ -190,7 +190,7 @@ classDiagram +LoadSecrets / SaveSecrets mountID +Lock() / TryLock() flock } - note for Store "secrets-mount-id.json is the credential-finding record: what `bdrive status` prints and what the agent hook appends a sentence about. Like the inbound spool it outlives the cycle that wrote it — the daemon scans the write seconds before the turn starts — but unlike it, it is STATE and is never drained: a finding stands until the file changes without it" + note for Store "secrets-mount-id.json is the credential-finding record: what `bdrive status` prints and what the agent hook appends a sentence about. Like the inbound spool it outlives the cycle that wrote it — the daemon scans the write seconds before the turn starts — but unlike it, it is STATE and is never drained: a finding stands until the file changes without it. Each path's entry records the blob it was read from and the record carries the rule-set version (secrets.RulesVersion); finish re-reads any entry whose file now holds another blob (a peer's edit, which the scan never reads) or whose rules differ (a detector fix), and drops entries for untracked paths" note for Store "internal/store: ~/.bdrive/volumes/volume-id holds shared content-addressed blobs, per-device journal copies, and sync.json; state-mount-id.json, paused-mount-id, note-mount-id.json, secrets-mount-id.json, and inbound-mount-id.jsonl are per-mount (free funcs Paused/SetPaused and PausedMount/SetPausedMount, no flock)" note for Store "inbound-mount-id.jsonl (with legacy inbound.jsonl fallback) is the read spool's twin, running the other way: materialize appends every path it wrote or removed for a peer, and `sync --hook` drains it into the turn's context (re-read before editing). Appends and the drain share an flock on the spool's .lock sidecar, since the drain runs outside the volume lock beside a daemon that may be appending. A spool and not a Result field because the daemon usually materializes the change seconds before the turn starts, so the hook's own cycle sees nothing. Capped, best-effort, never fails a cycle. Result.Inbound now carries the SAME events for the post_sync hook and is not a duplicate to delete: that consumer fires from the cycle itself, and a second drainer would silently empty the agent hook's context" note for Store "The fold-cache keeps the replayed target, op count, lamport high-water mark, and ignore winner together; ReplayFold reuses unchanged journal folds instead of re-reading and sorting every device on every cycle. LoadCacheReadOnly supplies the immutable map used by echo suppression without cloning large mount caches." diff --git a/cmd/bdrive/hooksync_test.go b/cmd/bdrive/hooksync_test.go index 6e945f5c..1fd76649 100644 --- a/cmd/bdrive/hooksync_test.go +++ b/cmd/bdrive/hooksync_test.go @@ -10,7 +10,6 @@ import ( "testing" "github.com/runbear-io/beardrive/internal/config" - "github.com/runbear-io/beardrive/internal/secrets" "github.com/runbear-io/beardrive/internal/store" ) @@ -445,17 +444,12 @@ func TestSyncHookModeInboundSpoolUnreadable(t *testing.T) { } } -func seedSecrets(t *testing.T, proj config.Project, found map[string][]secrets.Finding) { +// writeSecretFile puts a credential-shaped string on the given line of rel, so +// the hook's cycle records the finding the way a daemon scan would. +func writeSecretFile(t *testing.T, dir, rel, secret string, line int) { t.Helper() - vdir, err := config.VolumeDir(proj.ID) - if err != nil { - t.Fatal(err) - } - st, err := store.Open(vdir) - if err != nil { - t.Fatal(err) - } - if err := st.SaveSecrets(proj.ID, found); err != nil { + body := strings.Repeat("text\n", line-1) + secret + "\n" + if err := os.WriteFile(filepath.Join(dir, rel), []byte(body), 0o644); err != nil { t.Fatal(err) } } @@ -468,10 +462,8 @@ func TestSyncHookModeReportsSecrets(t *testing.T) { t.Setenv("BDRIVE_HOME", t.TempDir()) root := t.TempDir() root, _ = filepath.EvalSymlinks(root) - proj := mountAt(t, root, "wiki", "https://hub.example.com/p/p-12345678") - seedSecrets(t, proj, map[string][]secrets.Finding{ - "deploy.md": {{Rule: "aws_access_key_id", Line: 12}}, - }) + mountAt(t, root, "wiki", "https://hub.example.com/p/p-12345678") + writeSecretFile(t, filepath.Join(root, "wiki"), "deploy.md", "export AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE", 12) got := runHook(t, filepath.Join(root, "wiki")) for _, want := range []string{ @@ -496,10 +488,10 @@ func TestSyncHookModeSecretsMultipleMounts(t *testing.T) { t.Setenv("BDRIVE_HOME", t.TempDir()) root := t.TempDir() root, _ = filepath.EvalSymlinks(root) - a := mountAt(t, root, "projA", "https://hub.example.com/p/p-aaaaaaaa") - b := mountAt(t, root, "projB", "https://hub.example.com/p/p-bbbbbbbb") - seedSecrets(t, a, map[string][]secrets.Finding{"a.md": {{Rule: "private_key", Line: 1}}}) - seedSecrets(t, b, map[string][]secrets.Finding{"b.md": {{Rule: "slack_token", Line: 2}}}) + mountAt(t, root, "projA", "https://hub.example.com/p/p-aaaaaaaa") + mountAt(t, root, "projB", "https://hub.example.com/p/p-bbbbbbbb") + writeSecretFile(t, filepath.Join(root, "projA"), "a.md", "-----BEGIN PRIVATE KEY-----\nMIIEowIBAAKCAQEAexampleexampleexampleexample0123456789", 1) + writeSecretFile(t, filepath.Join(root, "projB"), "b.md", "token: xoxb-1234567890-abcdefghij", 2) got := runHook(t, root) for _, want := range []string{"`projA/a.md` (a private key, line 1)", "`projB/b.md` (a Slack token, line 2)"} { diff --git a/internal/secrets/secrets.go b/internal/secrets/secrets.go index cd7db862..970b655c 100644 --- a/internal/secrets/secrets.go +++ b/internal/secrets/secrets.go @@ -14,6 +14,8 @@ package secrets import ( "bytes" + "crypto/sha256" + "encoding/hex" "regexp" "sort" ) @@ -47,10 +49,27 @@ var secretRules = []struct { {"openai_api_key", regexp.MustCompile(`(?:^|[^A-Za-z0-9_/-])(sk-[A-Za-z0-9_-]{20,})`)}, {"github_pat", regexp.MustCompile(`ghp_[A-Za-z0-9]{36}`)}, {"slack_token", regexp.MustCompile(`xox[baprs]-[A-Za-z0-9-]{10,}`)}, - {"private_key", regexp.MustCompile(`-----BEGIN [A-Z ]*PRIVATE KEY-----`)}, + // A header alone is not a key: docs quote it as a placeholder + // ("-----BEGIN PRIVATE KEY-----\n..."). The body must follow, after a real + // or JSON-escaped line break and any RFC 1421 headers (Proc-Type:, + // DEK-Info:) an encrypted PEM carries. + {"private_key", regexp.MustCompile(`-----BEGIN [A-Z ]*PRIVATE KEY-----(?:\s|\\[rn])+(?:[A-Za-z-]+:[^\n\\]*(?:\s|\\[rn])+)*[A-Za-z0-9+/]{32,}`)}, {"gitlab_pat", regexp.MustCompile(`glpat-[A-Za-z0-9_-]{20,}`)}, } +// rulesVersion names this rule set; see RulesVersion. +var rulesVersion = func() string { + h := sha256.New() + for _, r := range secretRules { + h.Write([]byte(r.id + "\x00" + r.re.String() + "\x00")) + } + return hex.EncodeToString(h.Sum(nil))[:12] +}() + +// RulesVersion changes whenever a rule does. A finding recorded under another +// version describes a rule set no longer in force, so its file is read again. +func RulesVersion() string { return rulesVersion } + // labels are the rule ids in words. They existed only in the frontend // (SECRET_LABELS in Browser.tsx) while `bdrive share` printed the bare id, so // the same finding read as "an AWS access key" in the browser and diff --git a/internal/secrets/secrets_test.go b/internal/secrets/secrets_test.go index eb6c9e74..215c42dc 100644 --- a/internal/secrets/secrets_test.go +++ b/internal/secrets/secrets_test.go @@ -14,6 +14,7 @@ const ( fakeSlackTok = "xoxb-1234567890-abcdefghij" fakeGitLabPAT = "glpat-abcdefghij0123456789XY" fakePrivKey = "-----BEGIN RSA PRIVATE KEY-----" + fakeKeyBody = "MIIEowIBAAKCAQEAexampleexampleexampleexample0123456789" ) func TestScanSecrets(t *testing.T) { @@ -39,7 +40,12 @@ func TestScanSecrets(t *testing.T) { {"github", "\n\n" + fakeGitHubPAT, []Finding{{"github_pat", 3}}}, {"slack", "token: " + fakeSlackTok, []Finding{{"slack_token", 1}}}, {"gitlab", "x\n" + fakeGitLabPAT, []Finding{{"gitlab_pat", 2}}}, - {"private key", "a\nb\nc\n" + fakePrivKey + "\nMIIE...\n", []Finding{{"private_key", 4}}}, + {"private key", "a\nb\nc\n" + fakePrivKey + "\n" + fakeKeyBody + "\n", []Finding{{"private_key", 4}}}, + {"private key, JSON-escaped", `{"key": "` + fakePrivKey + `\n` + fakeKeyBody + `\n"}`, []Finding{{"private_key", 1}}}, + {"private key, encrypted PEM headers", fakePrivKey + "\nProc-Type: 4,ENCRYPTED\nDEK-Info: AES-128-CBC,0123456789ABCDEF\n\n" + fakeKeyBody + "\n", []Finding{{"private_key", 1}}}, + // A doc quoting the header as a placeholder holds no key. + {"private key header as a placeholder", "E.g. `-----BEGIN PRIVATE KEY-----\\n...` in the field", nil}, + {"private key header alone", fakePrivKey + "\n...\n", nil}, { // One line, one rule, three keys: one finding, not three. "multi key line deduped", diff --git a/internal/store/secrets.go b/internal/store/secrets.go index bfadf53e..e71fc289 100644 --- a/internal/store/secrets.go +++ b/internal/store/secrets.go @@ -1,6 +1,10 @@ package store import ( + "bytes" + "encoding/json" + "os" + "github.com/runbear-io/beardrive/internal/secrets" ) @@ -27,31 +31,96 @@ func (s *Store) secretsPath(mountID string) (string, error) { return s.mountStatePath("secrets-", mountID) } -// LoadSecrets returns the findings recorded for each mount-relative path. -func (s *Store) LoadSecrets(mountID string) (map[string][]secrets.Finding, error) { +// SecretRecord is the findings file: what the rule set named by Rules found +// in each path, as of the blob that path held when it was read. +type SecretRecord struct { + Rules string `json:"rules"` + Paths map[string]SecretEntry `json:"paths"` +} + +// SecretEntry is one path's findings and the content they describe. A path +// whose file now holds another blob, or a record written under other rules, +// is read again rather than trusted. +type SecretEntry struct { + Blob string `json:"blob,omitempty"` + Found []secrets.Finding `json:"found"` +} + +// LoadSecretRecord reads the findings file. A file in the bare per-path shape +// ({"path": [findings]}) loads with no rules and no blobs, so every path in it +// is read again. +func (s *Store) LoadSecretRecord(mountID string) (SecretRecord, error) { + rec := SecretRecord{Paths: map[string]SecretEntry{}} p, err := s.secretsPath(mountID) if err != nil { - return nil, err + return rec, err } - out := map[string][]secrets.Finding{} - if err := readJSON(p, &out); err != nil { - return nil, err + data, err := os.ReadFile(p) + if os.IsNotExist(err) { + return rec, nil + } + if err != nil { + return rec, err + } + var top map[string]json.RawMessage + if err := json.Unmarshal(data, &top); err != nil { + return rec, err + } + // Bare-shape values are arrays, so an object under "paths" is the record. + if raw := bytes.TrimSpace(top["paths"]); len(raw) > 0 && raw[0] == '{' { + if err := json.Unmarshal(data, &rec); err != nil { + return SecretRecord{Paths: map[string]SecretEntry{}}, err + } + if rec.Paths == nil { + rec.Paths = map[string]SecretEntry{} + } + } else { + for rel, raw := range top { + var f []secrets.Finding + if err := json.Unmarshal(raw, &f); err != nil { + return SecretRecord{Paths: map[string]SecretEntry{}}, err + } + rec.Paths[rel] = SecretEntry{Found: f} + } } // Same treatment as the cache's keys: plain JSON in $BDRIVE_HOME that // anything running as the user can write, rendered into a terminal and // joined against cache keys. An out-of-volume path is dropped, not shown. - for rel, f := range out { - if !cleanRel(rel) || len(f) == 0 { - delete(out, rel) + for rel, e := range rec.Paths { + if !cleanRel(rel) || len(e.Found) == 0 { + delete(rec.Paths, rel) } } - return out, nil + return rec, nil } -func (s *Store) SaveSecrets(mountID string, f map[string][]secrets.Finding) error { +func (s *Store) SaveSecretRecord(mountID string, rec SecretRecord) error { p, err := s.secretsPath(mountID) if err != nil { return err } - return WriteJSONAtomic(p, f) + return WriteJSONAtomic(p, rec) +} + +// LoadSecrets returns the findings recorded for each mount-relative path. +func (s *Store) LoadSecrets(mountID string) (map[string][]secrets.Finding, error) { + rec, err := s.LoadSecretRecord(mountID) + if err != nil { + return nil, err + } + out := make(map[string][]secrets.Finding, len(rec.Paths)) + for rel, e := range rec.Paths { + out[rel] = e.Found + } + return out, nil +} + +// SaveSecrets records findings with no blob or rules, so a cycle reads each +// path again before trusting them. +func (s *Store) SaveSecrets(mountID string, f map[string][]secrets.Finding) error { + rec := SecretRecord{Paths: make(map[string]SecretEntry, len(f))} + for rel, found := range f { + rec.Paths[rel] = SecretEntry{Found: found} + } + return s.SaveSecretRecord(mountID, rec) } diff --git a/internal/store/secrets_test.go b/internal/store/secrets_test.go index 6ccf3061..b45b93c8 100644 --- a/internal/store/secrets_test.go +++ b/internal/store/secrets_test.go @@ -82,3 +82,43 @@ func TestLoadSecretsDropsForeignKeys(t *testing.T) { t.Fatalf("mode = %v, want 0600", fi.Mode().Perm()) } } + +// A record in the bare per-path shape loads with no rules and no blobs, which +// is what makes the next cycle read every path in it again. +func TestLoadSecretRecordReadsTheBarePerPathShape(t *testing.T) { + s, err := Open(t.TempDir()) + if err != nil { + t.Fatal(err) + } + p, err := s.secretsPath("m1") + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(`{"notes.md": [{"rule":"openai_api_key","line":49}]}`), 0o600); err != nil { + t.Fatal(err) + } + rec, err := s.LoadSecretRecord("m1") + if err != nil { + t.Fatal(err) + } + e, ok := rec.Paths["notes.md"] + if rec.Rules != "" || !ok || e.Blob != "" || len(e.Found) != 1 { + t.Fatalf("record = %+v, want notes.md with no rules and no blob", rec) + } + + rec.Rules = "r1" + rec.Paths["notes.md"] = SecretEntry{Blob: "abc", Found: e.Found} + if err := s.SaveSecretRecord("m1", rec); err != nil { + t.Fatal(err) + } + got, err := s.LoadSecretRecord("m1") + if err != nil { + t.Fatal(err) + } + if got.Rules != "r1" || got.Paths["notes.md"].Blob != "abc" { + t.Fatalf("round trip = %+v", got) + } + if found, _ := s.LoadSecrets("m1"); len(found["notes.md"]) != 1 { + t.Fatalf("LoadSecrets = %+v, want notes.md", found) + } +} diff --git a/internal/syncer/secrets.go b/internal/syncer/secrets.go index ebc2c172..59a59576 100644 --- a/internal/syncer/secrets.go +++ b/internal/syncer/secrets.go @@ -32,10 +32,18 @@ const secretsPerFile = 8 // is the point — nearly every cycle scans zero changed files, and a whole-set // rewrite would erase the warning seconds after it appeared. type secretLog struct { - found map[string][]secrets.Finding + rec store.SecretRecord dirty bool } +func newSecretLog(st *store.Store, mountID string) *secretLog { + l := &secretLog{rec: store.SecretRecord{Paths: map[string]store.SecretEntry{}}} + if rec, err := st.LoadSecretRecord(mountID); err == nil { + l.rec = rec + } + return l +} + // scanBlob records what the blob just written for rel contains. Called only on // the branches that ran PutBlobFile, so an unchanged file is never re-read. func (l *secretLog) scanBlob(st *store.Store, rel, sum string) { @@ -55,29 +63,63 @@ func (l *secretLog) scanBlob(st *store.Store, rel, sum string) { if len(found) > secretsPerFile { found = found[:secretsPerFile] } - l.set(rel, found) + l.set(rel, sum, found) } // set records rel's current findings, dropping the entry when there are none — // which is how fixing the file clears the warning with no command and no flag. -func (l *secretLog) set(rel string, found []secrets.Finding) { +func (l *secretLog) set(rel, sum string, found []secrets.Finding) { if len(found) == 0 { l.drop(rel) return } - if !sameFindings(l.found[rel], found) { - l.found[rel] = found + if e := l.rec.Paths[rel]; e.Blob != sum || !sameFindings(e.Found, found) { + l.rec.Paths[rel] = store.SecretEntry{Blob: sum, Found: found} l.dirty = true } } +// revalidate reads again every flagged path whose file no longer holds the +// blob its findings came from, or whose findings came from other rules. The +// scan reads only files this device changed, so without this a file a peer +// fixed, or one a detector fix stops flagging, keeps its warning here for as +// long as nobody on this device edits it. It costs one blob read per stale +// finding and nothing on a quiet cycle. +func (l *secretLog) revalidate(st *store.Store, cache map[string]store.CachedFile) { + if l == nil { + return + } + current := secrets.RulesVersion() + rescanAll := l.rec.Rules != current + if rescanAll { + l.rec.Rules = current + l.dirty = true + } + rels := make([]string, 0, len(l.rec.Paths)) + for rel := range l.rec.Paths { + rels = append(rels, rel) + } + for _, rel := range rels { + c, tracked := cache[rel] + if !tracked || c.Link != "" || c.Blob == "" { + l.drop(rel) + continue + } + if !rescanAll && l.rec.Paths[rel].Blob == c.Blob { + continue + } + l.drop(rel) + l.scanBlob(st, rel, c.Blob) + } +} + // drop forgets a path: its credential is gone, or the path is. func (l *secretLog) drop(rel string) { if l == nil { return } - if _, ok := l.found[rel]; ok { - delete(l.found, rel) + if _, ok := l.rec.Paths[rel]; ok { + delete(l.rec.Paths, rel) l.dirty = true } } diff --git a/internal/syncer/syncer.go b/internal/syncer/syncer.go index 2c667724..5148414f 100644 --- a/internal/syncer/syncer.go +++ b/internal/syncer/syncer.go @@ -48,7 +48,6 @@ import ( "github.com/runbear-io/beardrive/internal/config" "github.com/runbear-io/beardrive/internal/journal" "github.com/runbear-io/beardrive/internal/remote" - "github.com/runbear-io/beardrive/internal/secrets" "github.com/runbear-io/beardrive/internal/store" ) @@ -499,10 +498,7 @@ func (s *Session) cycleLocked(ctx context.Context, hint *ScanHint) (*Result, err loadedSync := store.CloneSyncState(st) // Advisory telemetry about files this cycle will journal and push anyway, // so an unreadable record starts the cycle empty rather than failing it. - sec := &secretLog{found: map[string][]secrets.Finding{}} - if found, err := s.Store.LoadSecrets(s.mountID()); err == nil { - sec.found = found - } + sec := newSecretLog(s.Store, s.mountID()) myOpCount, err := s.Store.OwnOpCount(s.Device.ID) if err != nil { return nil, fmt.Errorf("read own journal: %w", err) @@ -1102,8 +1098,9 @@ func (s *Session) finish(cache map[string]store.CachedFile, st store.SyncState, // advisory telemetry about ops that are already journaled and pushed, and // giving it a veto over convergence is the one invariant this feature is // close enough to break. + sec.revalidate(s.Store, cache) if sec != nil && sec.dirty { - if err := s.Store.SaveSecrets(s.mountID(), sec.found); err != nil { + if err := s.Store.SaveSecretRecord(s.mountID(), sec.rec); err != nil { log.Printf("beardrive: could not record credential findings: %v", err) } } diff --git a/internal/syncer/syncer_test.go b/internal/syncer/syncer_test.go index 13b9c967..ddc2c70d 100644 --- a/internal/syncer/syncer_test.go +++ b/internal/syncer/syncer_test.go @@ -1636,6 +1636,70 @@ func TestSecretScanSkipsUnchangedFiles(t *testing.T) { } } +// A peer that edits the key out clears the warning here too: the scan reads +// only what this device changed, so the record has to follow the file to the +// blob the peer wrote. +func TestSecretClearsWhenAPeerFixesTheFile(t *testing.T) { + be := sharedRemote(t) + a := newDevice(t, "deva", be) + b := newDevice(t, "devb", be) + + write(t, a.Folder, "deploy.md", "key = "+testAWSKey+"\n") + cycle(t, a) + cycle(t, b) + write(t, b.Folder, "deploy.md", "key = read it from the environment\n") + cycle(t, b) + + cycle(t, a) + if got := read(t, a.Folder, "deploy.md"); strings.Contains(got, testAWSKey) { + t.Fatalf("control: the peer's fix did not arrive: %q", got) + } + if found, _ := a.Store.LoadSecrets(a.mountID()); len(found) != 0 { + t.Fatalf("findings = %+v, want empty once the peer removed the key", found) + } +} + +// Findings recorded under another rule set are judged again by the rules in +// force, on the next cycle and with nothing on disk changed: a detector fix +// clears its false positives without anyone editing the files. +func TestSecretFindingsFromOtherRulesAreReadAgain(t *testing.T) { + a := newDevice(t, "deva", sharedRemote(t)) + + write(t, a.Folder, "notes.md", "see https://example.com/helpdesk-agents-for-slack-and-teams\n") + write(t, a.Folder, "deploy.md", "key = "+testAWSKey+"\n") + cycle(t, a) + rec, err := a.Store.LoadSecretRecord(a.mountID()) + if err != nil { + t.Fatal(err) + } + cache, err := a.Store.LoadCache(a.mountID()) + if err != nil { + t.Fatal(err) + } + // What an older detector left behind: a false positive on notes.md, under + // another rule set, for the blob the file still holds. + rec.Rules = "older-rules" + rec.Paths["notes.md"] = store.SecretEntry{Blob: cache["notes.md"].Blob, Found: []secrets.Finding{{Rule: "openai_api_key", Line: 1}}} + if err := a.Store.SaveSecretRecord(a.mountID(), rec); err != nil { + t.Fatal(err) + } + + cycle(t, a) // nothing changed on disk + found, err := a.Store.LoadSecrets(a.mountID()) + if err != nil { + t.Fatal(err) + } + if _, ok := found["notes.md"]; ok { + t.Errorf("findings = %+v, want the false positive gone under the current rules", found) + } + if len(found["deploy.md"]) == 0 { + t.Errorf("findings = %+v, want the real key still reported", found) + } + if rec, _ := a.Store.LoadSecretRecord(a.mountID()); rec.Rules != secrets.RulesVersion() { + t.Errorf("record rules = %q, want %q", rec.Rules, secrets.RulesVersion()) + } +} + func TestPullRejectsSymlinkTargetOnlySlotRedefinition(t *testing.T) { be := sharedRemote(t) s := newDevice(t, "local", be) From 0c26f17a9f44801de60301ba967cb95c214d5123 Mon Sep 17 00:00:00 2001 From: Hoseong Liam Hwang <449334+thefron@users.noreply.github.com> Date: Sat, 14 Nov 2026 00:14:51 -0800 Subject: [PATCH 2/5] fix(secrets): count a key body across line breaks and keep findings on an unreadable blob Co-Authored-By: Claude Opus 5.5 --- internal/secrets/secrets.go | 5 ++-- internal/secrets/secrets_test.go | 11 ++++++++ internal/syncer/secrets.go | 35 ++++++++++++++++++-------- internal/syncer/syncer_test.go | 43 ++++++++++++++++++++++++++++++++ internal/webapp/shares_test.go | 25 +++++++++++++++++++ 5 files changed, 107 insertions(+), 12 deletions(-) diff --git a/internal/secrets/secrets.go b/internal/secrets/secrets.go index 970b655c..f5bae475 100644 --- a/internal/secrets/secrets.go +++ b/internal/secrets/secrets.go @@ -52,8 +52,9 @@ var secretRules = []struct { // A header alone is not a key: docs quote it as a placeholder // ("-----BEGIN PRIVATE KEY-----\n..."). The body must follow, after a real // or JSON-escaped line break and any RFC 1421 headers (Proc-Type:, - // DEK-Info:) an encrypted PEM carries. - {"private_key", regexp.MustCompile(`-----BEGIN [A-Z ]*PRIVATE KEY-----(?:\s|\\[rn])+(?:[A-Za-z-]+:[^\n\\]*(?:\s|\\[rn])+)*[A-Za-z0-9+/]{32,}`)}, + // DEK-Info:) an encrypted PEM carries: 32 base64 characters, counted + // across the line breaks a PEM may wrap its body at, at any width. + {"private_key", regexp.MustCompile(`-----BEGIN [A-Z ]*PRIVATE KEY-----(?:\s|\\[rn])+(?:[A-Za-z-]+:[^\n\\]*(?:\s|\\[rn])+)*(?:[A-Za-z0-9+/=](?:\r?\n[ \t]*|\\[rn])*){32}`)}, {"gitlab_pat", regexp.MustCompile(`glpat-[A-Za-z0-9_-]{20,}`)}, } diff --git a/internal/secrets/secrets_test.go b/internal/secrets/secrets_test.go index 215c42dc..2938bad9 100644 --- a/internal/secrets/secrets_test.go +++ b/internal/secrets/secrets_test.go @@ -17,6 +17,15 @@ const ( fakeKeyBody = "MIIEowIBAAKCAQEAexampleexampleexampleexample0123456789" ) +// wrap breaks s into lines of n characters, as a PEM body may be. +func wrap(s string, n int) string { + var lines []string + for len(s) > n { + lines, s = append(lines, s[:n]), s[n:] + } + return strings.Join(append(lines, s), "\n") +} + func TestScanSecrets(t *testing.T) { for _, tc := range []struct { name string @@ -43,6 +52,8 @@ func TestScanSecrets(t *testing.T) { {"private key", "a\nb\nc\n" + fakePrivKey + "\n" + fakeKeyBody + "\n", []Finding{{"private_key", 4}}}, {"private key, JSON-escaped", `{"key": "` + fakePrivKey + `\n` + fakeKeyBody + `\n"}`, []Finding{{"private_key", 1}}}, {"private key, encrypted PEM headers", fakePrivKey + "\nProc-Type: 4,ENCRYPTED\nDEK-Info: AES-128-CBC,0123456789ABCDEF\n\n" + fakeKeyBody + "\n", []Finding{{"private_key", 1}}}, + {"private key wrapped in short lines", fakePrivKey + "\n" + wrap(fakeKeyBody, 16) + "\n", []Finding{{"private_key", 1}}}, + {"private key wrapped in short indented lines", "key: |\n " + fakePrivKey + "\n " + strings.ReplaceAll(wrap(fakeKeyBody, 8), "\n", "\n ") + "\n", []Finding{{"private_key", 2}}}, // A doc quoting the header as a placeholder holds no key. {"private key header as a placeholder", "E.g. `-----BEGIN PRIVATE KEY-----\\n...` in the field", nil}, {"private key header alone", fakePrivKey + "\n...\n", nil}, diff --git a/internal/syncer/secrets.go b/internal/syncer/secrets.go index 59a59576..88461eed 100644 --- a/internal/syncer/secrets.go +++ b/internal/syncer/secrets.go @@ -50,20 +50,28 @@ func (l *secretLog) scanBlob(st *store.Store, rel, sum string) { if l == nil { return } + if found, ok := readFindings(st, sum); ok { + l.set(rel, sum, found) + } +} + +// readFindings scans a blob; ok is false when it could not be read, which +// leaves whatever was recorded before standing. +func readFindings(st *store.Store, sum string) ([]secrets.Finding, bool) { f, err := st.OpenBlob(sum) if err != nil { - return // unreadable: skipped like every other unreadable-file case + return nil, false // unreadable: skipped like every other unreadable-file case } defer f.Close() buf, err := io.ReadAll(io.LimitReader(f, secrets.ScanLimit)) if err != nil { - return + return nil, false } found := secrets.Scan(buf) if len(found) > secretsPerFile { found = found[:secretsPerFile] } - l.set(rel, sum, found) + return found, true } // set records rel's current findings, dropping the entry when there are none — @@ -84,17 +92,16 @@ func (l *secretLog) set(rel, sum string, found []secrets.Finding) { // scan reads only files this device changed, so without this a file a peer // fixed, or one a detector fix stops flagging, keeps its warning here for as // long as nobody on this device edits it. It costs one blob read per stale -// finding and nothing on a quiet cycle. +// finding and nothing on a quiet cycle. A blob that cannot be read keeps its +// path's findings and, under other rules, keeps the record's rules too, so +// the next cycle tries again. func (l *secretLog) revalidate(st *store.Store, cache map[string]store.CachedFile) { if l == nil { return } current := secrets.RulesVersion() rescanAll := l.rec.Rules != current - if rescanAll { - l.rec.Rules = current - l.dirty = true - } + unread := false rels := make([]string, 0, len(l.rec.Paths)) for rel := range l.rec.Paths { rels = append(rels, rel) @@ -108,8 +115,16 @@ func (l *secretLog) revalidate(st *store.Store, cache map[string]store.CachedFil if !rescanAll && l.rec.Paths[rel].Blob == c.Blob { continue } - l.drop(rel) - l.scanBlob(st, rel, c.Blob) + found, ok := readFindings(st, c.Blob) + if !ok { + unread = true + continue + } + l.set(rel, c.Blob, found) + } + if rescanAll && !unread { + l.rec.Rules = current + l.dirty = true } } diff --git a/internal/syncer/syncer_test.go b/internal/syncer/syncer_test.go index ddc2c70d..da0f93fd 100644 --- a/internal/syncer/syncer_test.go +++ b/internal/syncer/syncer_test.go @@ -1700,6 +1700,49 @@ func TestSecretFindingsFromOtherRulesAreReadAgain(t *testing.T) { } } +// A finding whose blob cannot be read stays, and so do the rules it was +// recorded under: the next cycle reads it again instead of the warning +// vanishing on one failed read. +func TestSecretFindingSurvivesAnUnreadableBlob(t *testing.T) { + a := newDevice(t, "deva", sharedRemote(t)) + + write(t, a.Folder, "deploy.md", "key = "+testAWSKey+"\n") + cycle(t, a) + rec, err := a.Store.LoadSecretRecord(a.mountID()) + if err != nil { + t.Fatal(err) + } + rec.Rules = "older-rules" + if err := a.Store.SaveSecretRecord(a.mountID(), rec); err != nil { + t.Fatal(err) + } + blob := a.Store.BlobPath(rec.Paths["deploy.md"].Blob) + data, err := os.ReadFile(blob) + if err != nil { + t.Fatal(err) + } + if err := os.Remove(blob); err != nil { + t.Fatal(err) + } + + cycle(t, a) // nothing changed on disk; the blob cannot be read + got, _ := a.Store.LoadSecretRecord(a.mountID()) + if len(got.Paths["deploy.md"].Found) == 0 { + t.Fatalf("record = %+v, want the finding kept when its blob cannot be read", got) + } + if got.Rules != "older-rules" { + t.Fatalf("rules = %q, want the old rules kept so the next cycle retries", got.Rules) + } + + if err := os.WriteFile(blob, data, 0o600); err != nil { + t.Fatal(err) + } + cycle(t, a) + if got, _ := a.Store.LoadSecretRecord(a.mountID()); got.Rules != secrets.RulesVersion() || len(got.Paths["deploy.md"].Found) == 0 { + t.Fatalf("record = %+v, want it re-read under the current rules once readable", got) + } +} + func TestPullRejectsSymlinkTargetOnlySlotRedefinition(t *testing.T) { be := sharedRemote(t) s := newDevice(t, "local", be) diff --git a/internal/webapp/shares_test.go b/internal/webapp/shares_test.go index 63ceae5e..995faf3d 100644 --- a/internal/webapp/shares_test.go +++ b/internal/webapp/shares_test.go @@ -895,6 +895,31 @@ func TestShareSecretScan(t *testing.T) { } } +// A private key's body may be wrapped at any width, and the share gate still +// refuses it; only the header quoted as a placeholder passes. +func TestShareRefusesAPrivateKeyWrappedInShortLines(t *testing.T) { + srv, p, _, f, h := shareHub(t) + body := "MIIEowIBAAKCAQEAexampleexampleexampleexample0123456789" + var wrapped []string + for len(body) > 16 { + wrapped, body = append(wrapped, body[:16]), body[16:] + } + wrapped = append(wrapped, body) + f.put("dev1", "deploy/server.pem", "-----BEGIN PRIVATE KEY-----\n"+strings.Join(wrapped, "\n")+"\n-----END PRIVATE KEY-----\n") + f.put("dev1", "docs/fields.md", "E.g. `-----BEGIN PRIVATE KEY-----\\n...` goes in this field\n") + + rec := postShare(t, srv, h, p.ID, map[string]string{"path": "deploy/server.pem"}) + if rec.Code != http.StatusConflict { + t.Fatalf("share of a wrapped private key: %d %s, want 409", rec.Code, rec.Body) + } + if got := decodeFindings(t, rec); !reflect.DeepEqual(got, []secrets.Finding{{Rule: "private_key", Line: 1}}) { + t.Fatalf("findings = %v, want private_key on line 1", got) + } + if rec := postShare(t, srv, h, p.ID, map[string]string{"path": "docs/fields.md"}); rec.Code != 200 { + t.Fatalf("a doc quoting the header as a placeholder: %d %s, want 200", rec.Code, rec.Body) + } +} + // TestShareSecretNeverEchoed is the one rule that cannot bend: the matched // bytes never leave secrets.Scan — not in the body, not in the log. func TestShareSecretNeverEchoed(t *testing.T) { From 1fa1a8cf93a3b090b13780c703d34bc26242eb68 Mon Sep 17 00:00:00 2001 From: Hoseong Liam Hwang <449334+thefron@users.noreply.github.com> Date: Sat, 14 Nov 2026 00:25:04 -0800 Subject: [PATCH 3/5] test(hooks): seed the daemon's secret record for mounts below the session folder Since the hook no longer cycles mounts below the session folder, those mounts' warnings come only from the record their daemon wrote. Co-Authored-By: Claude Opus 5.5 --- cmd/bdrive/hooksync_test.go | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/cmd/bdrive/hooksync_test.go b/cmd/bdrive/hooksync_test.go index 1fd76649..2f800b3b 100644 --- a/cmd/bdrive/hooksync_test.go +++ b/cmd/bdrive/hooksync_test.go @@ -10,6 +10,7 @@ import ( "testing" "github.com/runbear-io/beardrive/internal/config" + "github.com/runbear-io/beardrive/internal/secrets" "github.com/runbear-io/beardrive/internal/store" ) @@ -446,6 +447,21 @@ func TestSyncHookModeInboundSpoolUnreadable(t *testing.T) { // writeSecretFile puts a credential-shaped string on the given line of rel, so // the hook's cycle records the finding the way a daemon scan would. +func seedSecrets(t *testing.T, proj config.Project, found map[string][]secrets.Finding) { + t.Helper() + vdir, err := config.VolumeDir(proj.ID) + if err != nil { + t.Fatal(err) + } + st, err := store.Open(vdir) + if err != nil { + t.Fatal(err) + } + if err := st.SaveSecrets(proj.ID, found); err != nil { + t.Fatal(err) + } +} + func writeSecretFile(t *testing.T, dir, rel, secret string, line int) { t.Helper() body := strings.Repeat("text\n", line-1) + secret + "\n" @@ -488,10 +504,12 @@ func TestSyncHookModeSecretsMultipleMounts(t *testing.T) { t.Setenv("BDRIVE_HOME", t.TempDir()) root := t.TempDir() root, _ = filepath.EvalSymlinks(root) - mountAt(t, root, "projA", "https://hub.example.com/p/p-aaaaaaaa") - mountAt(t, root, "projB", "https://hub.example.com/p/p-bbbbbbbb") - writeSecretFile(t, filepath.Join(root, "projA"), "a.md", "-----BEGIN PRIVATE KEY-----\nMIIEowIBAAKCAQEAexampleexampleexampleexample0123456789", 1) - writeSecretFile(t, filepath.Join(root, "projB"), "b.md", "token: xoxb-1234567890-abcdefghij", 2) + a := mountAt(t, root, "projA", "https://hub.example.com/p/p-aaaaaaaa") + b := mountAt(t, root, "projB", "https://hub.example.com/p/p-bbbbbbbb") + // Mounts below the session folder belong to their daemons and the hook + // never cycles them, so the daemon's record is all the hook reads. + seedSecrets(t, a, map[string][]secrets.Finding{"a.md": {{Rule: "private_key", Line: 1}}}) + seedSecrets(t, b, map[string][]secrets.Finding{"b.md": {{Rule: "slack_token", Line: 2}}}) got := runHook(t, root) for _, want := range []string{"`projA/a.md` (a private key, line 1)", "`projB/b.md` (a Slack token, line 2)"} { From 5cd265a2fa6d2c057df559faa4cbf4473a2984b3 Mon Sep 17 00:00:00 2001 From: Hoseong Liam Hwang <449334+thefron@users.noreply.github.com> Date: Sat, 14 Nov 2026 00:40:47 -0800 Subject: [PATCH 4/5] fix(secrets): allow blanks before the line breaks in a key body Co-Authored-By: Claude Opus 5.5 --- internal/secrets/secrets.go | 5 +++-- internal/secrets/secrets_test.go | 2 ++ internal/webapp/shares_test.go | 5 +++++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/internal/secrets/secrets.go b/internal/secrets/secrets.go index f5bae475..121aec66 100644 --- a/internal/secrets/secrets.go +++ b/internal/secrets/secrets.go @@ -53,8 +53,9 @@ var secretRules = []struct { // ("-----BEGIN PRIVATE KEY-----\n..."). The body must follow, after a real // or JSON-escaped line break and any RFC 1421 headers (Proc-Type:, // DEK-Info:) an encrypted PEM carries: 32 base64 characters, counted - // across the line breaks a PEM may wrap its body at, at any width. - {"private_key", regexp.MustCompile(`-----BEGIN [A-Z ]*PRIVATE KEY-----(?:\s|\\[rn])+(?:[A-Za-z-]+:[^\n\\]*(?:\s|\\[rn])+)*(?:[A-Za-z0-9+/=](?:\r?\n[ \t]*|\\[rn])*){32}`)}, + // across the line breaks a PEM may wrap its body at, at any width, with + // blanks on either side of each break. + {"private_key", regexp.MustCompile(`-----BEGIN [A-Z ]*PRIVATE KEY-----(?:\s|\\[rn])+(?:[A-Za-z-]+:[^\n\\]*(?:\s|\\[rn])+)*(?:[A-Za-z0-9+/=](?:[ \t]*\r?\n[ \t]*|\\[rn])*){32}`)}, {"gitlab_pat", regexp.MustCompile(`glpat-[A-Za-z0-9_-]{20,}`)}, } diff --git a/internal/secrets/secrets_test.go b/internal/secrets/secrets_test.go index 2938bad9..22a29822 100644 --- a/internal/secrets/secrets_test.go +++ b/internal/secrets/secrets_test.go @@ -53,6 +53,8 @@ func TestScanSecrets(t *testing.T) { {"private key, JSON-escaped", `{"key": "` + fakePrivKey + `\n` + fakeKeyBody + `\n"}`, []Finding{{"private_key", 1}}}, {"private key, encrypted PEM headers", fakePrivKey + "\nProc-Type: 4,ENCRYPTED\nDEK-Info: AES-128-CBC,0123456789ABCDEF\n\n" + fakeKeyBody + "\n", []Finding{{"private_key", 1}}}, {"private key wrapped in short lines", fakePrivKey + "\n" + wrap(fakeKeyBody, 16) + "\n", []Finding{{"private_key", 1}}}, + {"private key wrapped in short lines with trailing blanks", fakePrivKey + " \n" + strings.ReplaceAll(wrap(fakeKeyBody, 16), "\n", " \t\n") + " \n", []Finding{{"private_key", 1}}}, + {"private key wrapped in short CRLF lines with trailing blanks", fakePrivKey + "\r\n" + strings.ReplaceAll(wrap(fakeKeyBody, 16), "\n", " \r\n") + "\r\n", []Finding{{"private_key", 1}}}, {"private key wrapped in short indented lines", "key: |\n " + fakePrivKey + "\n " + strings.ReplaceAll(wrap(fakeKeyBody, 8), "\n", "\n ") + "\n", []Finding{{"private_key", 2}}}, // A doc quoting the header as a placeholder holds no key. {"private key header as a placeholder", "E.g. `-----BEGIN PRIVATE KEY-----\\n...` in the field", nil}, diff --git a/internal/webapp/shares_test.go b/internal/webapp/shares_test.go index 995faf3d..77b627c9 100644 --- a/internal/webapp/shares_test.go +++ b/internal/webapp/shares_test.go @@ -918,6 +918,11 @@ func TestShareRefusesAPrivateKeyWrappedInShortLines(t *testing.T) { if rec := postShare(t, srv, h, p.ID, map[string]string{"path": "docs/fields.md"}); rec.Code != 200 { t.Fatalf("a doc quoting the header as a placeholder: %d %s, want 200", rec.Code, rec.Body) } + + f.put("dev1", "deploy/padded.pem", "-----BEGIN PRIVATE KEY-----\n"+strings.Join(wrapped, " \n")+" \n-----END PRIVATE KEY-----\n") + if rec := postShare(t, srv, h, p.ID, map[string]string{"path": "deploy/padded.pem"}); rec.Code != http.StatusConflict { + t.Fatalf("share of a wrapped private key with trailing blanks: %d %s, want 409", rec.Code, rec.Body) + } } // TestShareSecretNeverEchoed is the one rule that cannot bend: the matched From 5034346a2c4cb3b1f06c77fadf8370d8cee3c843 Mon Sep 17 00:00:00 2001 From: Hoseong Liam Hwang <449334+thefron@users.noreply.github.com> Date: Sat, 14 Nov 2026 00:56:29 -0800 Subject: [PATCH 5/5] fix(secrets): allow blanks around escaped line breaks in a key body Co-Authored-By: Claude Opus 5.5 --- internal/secrets/secrets.go | 6 +++--- internal/secrets/secrets_test.go | 2 ++ internal/webapp/shares_test.go | 5 +++++ 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/internal/secrets/secrets.go b/internal/secrets/secrets.go index 121aec66..c2e971bd 100644 --- a/internal/secrets/secrets.go +++ b/internal/secrets/secrets.go @@ -53,9 +53,9 @@ var secretRules = []struct { // ("-----BEGIN PRIVATE KEY-----\n..."). The body must follow, after a real // or JSON-escaped line break and any RFC 1421 headers (Proc-Type:, // DEK-Info:) an encrypted PEM carries: 32 base64 characters, counted - // across the line breaks a PEM may wrap its body at, at any width, with - // blanks on either side of each break. - {"private_key", regexp.MustCompile(`-----BEGIN [A-Z ]*PRIVATE KEY-----(?:\s|\\[rn])+(?:[A-Za-z-]+:[^\n\\]*(?:\s|\\[rn])+)*(?:[A-Za-z0-9+/=](?:[ \t]*\r?\n[ \t]*|\\[rn])*){32}`)}, + // across the line breaks, real or escaped, a PEM may wrap its body at, at + // any width, with blanks on either side of each break. + {"private_key", regexp.MustCompile(`-----BEGIN [A-Z ]*PRIVATE KEY-----(?:\s|\\[rnt])+(?:[A-Za-z-]+:[^\n\\]*(?:\s|\\[rnt])+)*(?:[A-Za-z0-9+/=](?:(?:[ \t]|\\t)*(?:\r?\n|\\[rn])(?:[ \t]|\\t)*)*){32}`)}, {"gitlab_pat", regexp.MustCompile(`glpat-[A-Za-z0-9_-]{20,}`)}, } diff --git a/internal/secrets/secrets_test.go b/internal/secrets/secrets_test.go index 22a29822..32750a8a 100644 --- a/internal/secrets/secrets_test.go +++ b/internal/secrets/secrets_test.go @@ -55,6 +55,8 @@ func TestScanSecrets(t *testing.T) { {"private key wrapped in short lines", fakePrivKey + "\n" + wrap(fakeKeyBody, 16) + "\n", []Finding{{"private_key", 1}}}, {"private key wrapped in short lines with trailing blanks", fakePrivKey + " \n" + strings.ReplaceAll(wrap(fakeKeyBody, 16), "\n", " \t\n") + " \n", []Finding{{"private_key", 1}}}, {"private key wrapped in short CRLF lines with trailing blanks", fakePrivKey + "\r\n" + strings.ReplaceAll(wrap(fakeKeyBody, 16), "\n", " \r\n") + "\r\n", []Finding{{"private_key", 1}}}, + {"JSON-escaped private key wrapped in short lines with trailing blanks", `{"key": "` + fakePrivKey + ` \n` + strings.ReplaceAll(wrap(fakeKeyBody, 16), "\n", ` \n`) + ` \n"}`, []Finding{{"private_key", 1}}}, + {"JSON-escaped private key wrapped in short tab-indented lines", `{"key": "` + fakePrivKey + `\n\t` + strings.ReplaceAll(wrap(fakeKeyBody, 16), "\n", `\t\n\t`) + `\n"}`, []Finding{{"private_key", 1}}}, {"private key wrapped in short indented lines", "key: |\n " + fakePrivKey + "\n " + strings.ReplaceAll(wrap(fakeKeyBody, 8), "\n", "\n ") + "\n", []Finding{{"private_key", 2}}}, // A doc quoting the header as a placeholder holds no key. {"private key header as a placeholder", "E.g. `-----BEGIN PRIVATE KEY-----\\n...` in the field", nil}, diff --git a/internal/webapp/shares_test.go b/internal/webapp/shares_test.go index 77b627c9..040e2c4f 100644 --- a/internal/webapp/shares_test.go +++ b/internal/webapp/shares_test.go @@ -923,6 +923,11 @@ func TestShareRefusesAPrivateKeyWrappedInShortLines(t *testing.T) { if rec := postShare(t, srv, h, p.ID, map[string]string{"path": "deploy/padded.pem"}); rec.Code != http.StatusConflict { t.Fatalf("share of a wrapped private key with trailing blanks: %d %s, want 409", rec.Code, rec.Body) } + + f.put("dev1", "deploy/creds.json", `{"private_key": "-----BEGIN PRIVATE KEY----- \n`+strings.Join(wrapped, ` \n`)+` \n-----END PRIVATE KEY-----\n"}`+"\n") + if rec := postShare(t, srv, h, p.ID, map[string]string{"path": "deploy/creds.json"}); rec.Code != http.StatusConflict { + t.Fatalf("share of a JSON-escaped wrapped private key with trailing blanks: %d %s, want 409", rec.Code, rec.Body) + } } // TestShareSecretNeverEchoed is the one rule that cannot bend: the matched