diff --git a/architecture/cli-sync.md b/architecture/cli-sync.md index b13cf942..ae61c9cc 100644 --- a/architecture/cli-sync.md +++ b/architecture/cli-sync.md @@ -190,7 +190,7 @@ classDiagram +LoadSecrets / SaveSecrets mountID +Lock() / TryLock() flock } - note for Store "secrets-mount-id.json is the credential-finding record: what `bdrive status` prints and what the agent hook appends a sentence about. Like the inbound spool it outlives the cycle that wrote it — the daemon scans the write seconds before the turn starts — but unlike it, it is STATE and is never drained: a finding stands until the file changes without it" + note for Store "secrets-mount-id.json is the credential-finding record: what `bdrive status` prints and what the agent hook appends a sentence about. Like the inbound spool it outlives the cycle that wrote it — the daemon scans the write seconds before the turn starts — but unlike it, it is STATE and is never drained: a finding stands until the file changes without it. Each path's entry records the blob it was read from and the record carries the rule-set version (secrets.RulesVersion); finish re-reads any entry whose file now holds another blob (a peer's edit, which the scan never reads) or whose rules differ (a detector fix), and drops entries for untracked paths" note for Store "internal/store: ~/.bdrive/volumes/volume-id holds shared content-addressed blobs, per-device journal copies, and sync.json; state-mount-id.json, paused-mount-id, note-mount-id.json, secrets-mount-id.json, and inbound-mount-id.jsonl are per-mount (free funcs Paused/SetPaused and PausedMount/SetPausedMount, no flock)" note for Store "inbound-mount-id.jsonl (with legacy inbound.jsonl fallback) is the read spool's twin, running the other way: materialize appends every path it wrote or removed for a peer, and `sync --hook` drains it into the turn's context (re-read before editing). Appends and the drain share an flock on the spool's .lock sidecar, since the drain runs outside the volume lock beside a daemon that may be appending. A spool and not a Result field because the daemon usually materializes the change seconds before the turn starts, so the hook's own cycle sees nothing. Capped, best-effort, never fails a cycle. Result.Inbound now carries the SAME events for the post_sync hook and is not a duplicate to delete: that consumer fires from the cycle itself, and a second drainer would silently empty the agent hook's context" note for Store "The fold-cache keeps the replayed target, op count, lamport high-water mark, and ignore winner together; ReplayFold reuses unchanged journal folds instead of re-reading and sorting every device on every cycle. LoadCacheReadOnly supplies the immutable map used by echo suppression without cloning large mount caches." diff --git a/cmd/bdrive/hooksync_test.go b/cmd/bdrive/hooksync_test.go index 6e945f5c..2f800b3b 100644 --- a/cmd/bdrive/hooksync_test.go +++ b/cmd/bdrive/hooksync_test.go @@ -445,6 +445,8 @@ func TestSyncHookModeInboundSpoolUnreadable(t *testing.T) { } } +// writeSecretFile puts a credential-shaped string on the given line of rel, so +// the hook's cycle records the finding the way a daemon scan would. func seedSecrets(t *testing.T, proj config.Project, found map[string][]secrets.Finding) { t.Helper() vdir, err := config.VolumeDir(proj.ID) @@ -460,6 +462,14 @@ func seedSecrets(t *testing.T, proj config.Project, found map[string][]secrets.F } } +func writeSecretFile(t *testing.T, dir, rel, secret string, line int) { + t.Helper() + body := strings.Repeat("text\n", line-1) + secret + "\n" + if err := os.WriteFile(filepath.Join(dir, rel), []byte(body), 0o644); err != nil { + t.Fatal(err) + } +} + // The credential warning reaches the agent, whose own cycle found nothing: // the daemon scanned that write seconds ago, so — exactly like the inbound // spool — the record is what carries it. Advisory: the sentence says the file @@ -468,10 +478,8 @@ func TestSyncHookModeReportsSecrets(t *testing.T) { t.Setenv("BDRIVE_HOME", t.TempDir()) root := t.TempDir() root, _ = filepath.EvalSymlinks(root) - proj := mountAt(t, root, "wiki", "https://hub.example.com/p/p-12345678") - seedSecrets(t, proj, map[string][]secrets.Finding{ - "deploy.md": {{Rule: "aws_access_key_id", Line: 12}}, - }) + mountAt(t, root, "wiki", "https://hub.example.com/p/p-12345678") + writeSecretFile(t, filepath.Join(root, "wiki"), "deploy.md", "export AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE", 12) got := runHook(t, filepath.Join(root, "wiki")) for _, want := range []string{ @@ -498,6 +506,8 @@ func TestSyncHookModeSecretsMultipleMounts(t *testing.T) { root, _ = filepath.EvalSymlinks(root) a := mountAt(t, root, "projA", "https://hub.example.com/p/p-aaaaaaaa") b := mountAt(t, root, "projB", "https://hub.example.com/p/p-bbbbbbbb") + // Mounts below the session folder belong to their daemons and the hook + // never cycles them, so the daemon's record is all the hook reads. seedSecrets(t, a, map[string][]secrets.Finding{"a.md": {{Rule: "private_key", Line: 1}}}) seedSecrets(t, b, map[string][]secrets.Finding{"b.md": {{Rule: "slack_token", Line: 2}}}) diff --git a/internal/secrets/secrets.go b/internal/secrets/secrets.go index cd7db862..c2e971bd 100644 --- a/internal/secrets/secrets.go +++ b/internal/secrets/secrets.go @@ -14,6 +14,8 @@ package secrets import ( "bytes" + "crypto/sha256" + "encoding/hex" "regexp" "sort" ) @@ -47,10 +49,29 @@ var secretRules = []struct { {"openai_api_key", regexp.MustCompile(`(?:^|[^A-Za-z0-9_/-])(sk-[A-Za-z0-9_-]{20,})`)}, {"github_pat", regexp.MustCompile(`ghp_[A-Za-z0-9]{36}`)}, {"slack_token", regexp.MustCompile(`xox[baprs]-[A-Za-z0-9-]{10,}`)}, - {"private_key", regexp.MustCompile(`-----BEGIN [A-Z ]*PRIVATE KEY-----`)}, + // A header alone is not a key: docs quote it as a placeholder + // ("-----BEGIN PRIVATE KEY-----\n..."). The body must follow, after a real + // or JSON-escaped line break and any RFC 1421 headers (Proc-Type:, + // DEK-Info:) an encrypted PEM carries: 32 base64 characters, counted + // across the line breaks, real or escaped, a PEM may wrap its body at, at + // any width, with blanks on either side of each break. + {"private_key", regexp.MustCompile(`-----BEGIN [A-Z ]*PRIVATE KEY-----(?:\s|\\[rnt])+(?:[A-Za-z-]+:[^\n\\]*(?:\s|\\[rnt])+)*(?:[A-Za-z0-9+/=](?:(?:[ \t]|\\t)*(?:\r?\n|\\[rn])(?:[ \t]|\\t)*)*){32}`)}, {"gitlab_pat", regexp.MustCompile(`glpat-[A-Za-z0-9_-]{20,}`)}, } +// rulesVersion names this rule set; see RulesVersion. +var rulesVersion = func() string { + h := sha256.New() + for _, r := range secretRules { + h.Write([]byte(r.id + "\x00" + r.re.String() + "\x00")) + } + return hex.EncodeToString(h.Sum(nil))[:12] +}() + +// RulesVersion changes whenever a rule does. A finding recorded under another +// version describes a rule set no longer in force, so its file is read again. +func RulesVersion() string { return rulesVersion } + // labels are the rule ids in words. They existed only in the frontend // (SECRET_LABELS in Browser.tsx) while `bdrive share` printed the bare id, so // the same finding read as "an AWS access key" in the browser and diff --git a/internal/secrets/secrets_test.go b/internal/secrets/secrets_test.go index eb6c9e74..32750a8a 100644 --- a/internal/secrets/secrets_test.go +++ b/internal/secrets/secrets_test.go @@ -14,8 +14,18 @@ const ( fakeSlackTok = "xoxb-1234567890-abcdefghij" fakeGitLabPAT = "glpat-abcdefghij0123456789XY" fakePrivKey = "-----BEGIN RSA PRIVATE KEY-----" + fakeKeyBody = "MIIEowIBAAKCAQEAexampleexampleexampleexample0123456789" ) +// wrap breaks s into lines of n characters, as a PEM body may be. +func wrap(s string, n int) string { + var lines []string + for len(s) > n { + lines, s = append(lines, s[:n]), s[n:] + } + return strings.Join(append(lines, s), "\n") +} + func TestScanSecrets(t *testing.T) { for _, tc := range []struct { name string @@ -39,7 +49,18 @@ func TestScanSecrets(t *testing.T) { {"github", "\n\n" + fakeGitHubPAT, []Finding{{"github_pat", 3}}}, {"slack", "token: " + fakeSlackTok, []Finding{{"slack_token", 1}}}, {"gitlab", "x\n" + fakeGitLabPAT, []Finding{{"gitlab_pat", 2}}}, - {"private key", "a\nb\nc\n" + fakePrivKey + "\nMIIE...\n", []Finding{{"private_key", 4}}}, + {"private key", "a\nb\nc\n" + fakePrivKey + "\n" + fakeKeyBody + "\n", []Finding{{"private_key", 4}}}, + {"private key, JSON-escaped", `{"key": "` + fakePrivKey + `\n` + fakeKeyBody + `\n"}`, []Finding{{"private_key", 1}}}, + {"private key, encrypted PEM headers", fakePrivKey + "\nProc-Type: 4,ENCRYPTED\nDEK-Info: AES-128-CBC,0123456789ABCDEF\n\n" + fakeKeyBody + "\n", []Finding{{"private_key", 1}}}, + {"private key wrapped in short lines", fakePrivKey + "\n" + wrap(fakeKeyBody, 16) + "\n", []Finding{{"private_key", 1}}}, + {"private key wrapped in short lines with trailing blanks", fakePrivKey + " \n" + strings.ReplaceAll(wrap(fakeKeyBody, 16), "\n", " \t\n") + " \n", []Finding{{"private_key", 1}}}, + {"private key wrapped in short CRLF lines with trailing blanks", fakePrivKey + "\r\n" + strings.ReplaceAll(wrap(fakeKeyBody, 16), "\n", " \r\n") + "\r\n", []Finding{{"private_key", 1}}}, + {"JSON-escaped private key wrapped in short lines with trailing blanks", `{"key": "` + fakePrivKey + ` \n` + strings.ReplaceAll(wrap(fakeKeyBody, 16), "\n", ` \n`) + ` \n"}`, []Finding{{"private_key", 1}}}, + {"JSON-escaped private key wrapped in short tab-indented lines", `{"key": "` + fakePrivKey + `\n\t` + strings.ReplaceAll(wrap(fakeKeyBody, 16), "\n", `\t\n\t`) + `\n"}`, []Finding{{"private_key", 1}}}, + {"private key wrapped in short indented lines", "key: |\n " + fakePrivKey + "\n " + strings.ReplaceAll(wrap(fakeKeyBody, 8), "\n", "\n ") + "\n", []Finding{{"private_key", 2}}}, + // A doc quoting the header as a placeholder holds no key. + {"private key header as a placeholder", "E.g. `-----BEGIN PRIVATE KEY-----\\n...` in the field", nil}, + {"private key header alone", fakePrivKey + "\n...\n", nil}, { // One line, one rule, three keys: one finding, not three. "multi key line deduped", diff --git a/internal/store/secrets.go b/internal/store/secrets.go index bfadf53e..e71fc289 100644 --- a/internal/store/secrets.go +++ b/internal/store/secrets.go @@ -1,6 +1,10 @@ package store import ( + "bytes" + "encoding/json" + "os" + "github.com/runbear-io/beardrive/internal/secrets" ) @@ -27,31 +31,96 @@ func (s *Store) secretsPath(mountID string) (string, error) { return s.mountStatePath("secrets-", mountID) } -// LoadSecrets returns the findings recorded for each mount-relative path. -func (s *Store) LoadSecrets(mountID string) (map[string][]secrets.Finding, error) { +// SecretRecord is the findings file: what the rule set named by Rules found +// in each path, as of the blob that path held when it was read. +type SecretRecord struct { + Rules string `json:"rules"` + Paths map[string]SecretEntry `json:"paths"` +} + +// SecretEntry is one path's findings and the content they describe. A path +// whose file now holds another blob, or a record written under other rules, +// is read again rather than trusted. +type SecretEntry struct { + Blob string `json:"blob,omitempty"` + Found []secrets.Finding `json:"found"` +} + +// LoadSecretRecord reads the findings file. A file in the bare per-path shape +// ({"path": [findings]}) loads with no rules and no blobs, so every path in it +// is read again. +func (s *Store) LoadSecretRecord(mountID string) (SecretRecord, error) { + rec := SecretRecord{Paths: map[string]SecretEntry{}} p, err := s.secretsPath(mountID) if err != nil { - return nil, err + return rec, err } - out := map[string][]secrets.Finding{} - if err := readJSON(p, &out); err != nil { - return nil, err + data, err := os.ReadFile(p) + if os.IsNotExist(err) { + return rec, nil + } + if err != nil { + return rec, err + } + var top map[string]json.RawMessage + if err := json.Unmarshal(data, &top); err != nil { + return rec, err + } + // Bare-shape values are arrays, so an object under "paths" is the record. + if raw := bytes.TrimSpace(top["paths"]); len(raw) > 0 && raw[0] == '{' { + if err := json.Unmarshal(data, &rec); err != nil { + return SecretRecord{Paths: map[string]SecretEntry{}}, err + } + if rec.Paths == nil { + rec.Paths = map[string]SecretEntry{} + } + } else { + for rel, raw := range top { + var f []secrets.Finding + if err := json.Unmarshal(raw, &f); err != nil { + return SecretRecord{Paths: map[string]SecretEntry{}}, err + } + rec.Paths[rel] = SecretEntry{Found: f} + } } // Same treatment as the cache's keys: plain JSON in $BDRIVE_HOME that // anything running as the user can write, rendered into a terminal and // joined against cache keys. An out-of-volume path is dropped, not shown. - for rel, f := range out { - if !cleanRel(rel) || len(f) == 0 { - delete(out, rel) + for rel, e := range rec.Paths { + if !cleanRel(rel) || len(e.Found) == 0 { + delete(rec.Paths, rel) } } - return out, nil + return rec, nil } -func (s *Store) SaveSecrets(mountID string, f map[string][]secrets.Finding) error { +func (s *Store) SaveSecretRecord(mountID string, rec SecretRecord) error { p, err := s.secretsPath(mountID) if err != nil { return err } - return WriteJSONAtomic(p, f) + return WriteJSONAtomic(p, rec) +} + +// LoadSecrets returns the findings recorded for each mount-relative path. +func (s *Store) LoadSecrets(mountID string) (map[string][]secrets.Finding, error) { + rec, err := s.LoadSecretRecord(mountID) + if err != nil { + return nil, err + } + out := make(map[string][]secrets.Finding, len(rec.Paths)) + for rel, e := range rec.Paths { + out[rel] = e.Found + } + return out, nil +} + +// SaveSecrets records findings with no blob or rules, so a cycle reads each +// path again before trusting them. +func (s *Store) SaveSecrets(mountID string, f map[string][]secrets.Finding) error { + rec := SecretRecord{Paths: make(map[string]SecretEntry, len(f))} + for rel, found := range f { + rec.Paths[rel] = SecretEntry{Found: found} + } + return s.SaveSecretRecord(mountID, rec) } diff --git a/internal/store/secrets_test.go b/internal/store/secrets_test.go index 6ccf3061..b45b93c8 100644 --- a/internal/store/secrets_test.go +++ b/internal/store/secrets_test.go @@ -82,3 +82,43 @@ func TestLoadSecretsDropsForeignKeys(t *testing.T) { t.Fatalf("mode = %v, want 0600", fi.Mode().Perm()) } } + +// A record in the bare per-path shape loads with no rules and no blobs, which +// is what makes the next cycle read every path in it again. +func TestLoadSecretRecordReadsTheBarePerPathShape(t *testing.T) { + s, err := Open(t.TempDir()) + if err != nil { + t.Fatal(err) + } + p, err := s.secretsPath("m1") + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(`{"notes.md": [{"rule":"openai_api_key","line":49}]}`), 0o600); err != nil { + t.Fatal(err) + } + rec, err := s.LoadSecretRecord("m1") + if err != nil { + t.Fatal(err) + } + e, ok := rec.Paths["notes.md"] + if rec.Rules != "" || !ok || e.Blob != "" || len(e.Found) != 1 { + t.Fatalf("record = %+v, want notes.md with no rules and no blob", rec) + } + + rec.Rules = "r1" + rec.Paths["notes.md"] = SecretEntry{Blob: "abc", Found: e.Found} + if err := s.SaveSecretRecord("m1", rec); err != nil { + t.Fatal(err) + } + got, err := s.LoadSecretRecord("m1") + if err != nil { + t.Fatal(err) + } + if got.Rules != "r1" || got.Paths["notes.md"].Blob != "abc" { + t.Fatalf("round trip = %+v", got) + } + if found, _ := s.LoadSecrets("m1"); len(found["notes.md"]) != 1 { + t.Fatalf("LoadSecrets = %+v, want notes.md", found) + } +} diff --git a/internal/syncer/secrets.go b/internal/syncer/secrets.go index ebc2c172..88461eed 100644 --- a/internal/syncer/secrets.go +++ b/internal/syncer/secrets.go @@ -32,41 +32,98 @@ const secretsPerFile = 8 // is the point — nearly every cycle scans zero changed files, and a whole-set // rewrite would erase the warning seconds after it appeared. type secretLog struct { - found map[string][]secrets.Finding + rec store.SecretRecord dirty bool } +func newSecretLog(st *store.Store, mountID string) *secretLog { + l := &secretLog{rec: store.SecretRecord{Paths: map[string]store.SecretEntry{}}} + if rec, err := st.LoadSecretRecord(mountID); err == nil { + l.rec = rec + } + return l +} + // scanBlob records what the blob just written for rel contains. Called only on // the branches that ran PutBlobFile, so an unchanged file is never re-read. func (l *secretLog) scanBlob(st *store.Store, rel, sum string) { if l == nil { return } + if found, ok := readFindings(st, sum); ok { + l.set(rel, sum, found) + } +} + +// readFindings scans a blob; ok is false when it could not be read, which +// leaves whatever was recorded before standing. +func readFindings(st *store.Store, sum string) ([]secrets.Finding, bool) { f, err := st.OpenBlob(sum) if err != nil { - return // unreadable: skipped like every other unreadable-file case + return nil, false // unreadable: skipped like every other unreadable-file case } defer f.Close() buf, err := io.ReadAll(io.LimitReader(f, secrets.ScanLimit)) if err != nil { - return + return nil, false } found := secrets.Scan(buf) if len(found) > secretsPerFile { found = found[:secretsPerFile] } - l.set(rel, found) + return found, true } // set records rel's current findings, dropping the entry when there are none — // which is how fixing the file clears the warning with no command and no flag. -func (l *secretLog) set(rel string, found []secrets.Finding) { +func (l *secretLog) set(rel, sum string, found []secrets.Finding) { if len(found) == 0 { l.drop(rel) return } - if !sameFindings(l.found[rel], found) { - l.found[rel] = found + if e := l.rec.Paths[rel]; e.Blob != sum || !sameFindings(e.Found, found) { + l.rec.Paths[rel] = store.SecretEntry{Blob: sum, Found: found} + l.dirty = true + } +} + +// revalidate reads again every flagged path whose file no longer holds the +// blob its findings came from, or whose findings came from other rules. The +// scan reads only files this device changed, so without this a file a peer +// fixed, or one a detector fix stops flagging, keeps its warning here for as +// long as nobody on this device edits it. It costs one blob read per stale +// finding and nothing on a quiet cycle. A blob that cannot be read keeps its +// path's findings and, under other rules, keeps the record's rules too, so +// the next cycle tries again. +func (l *secretLog) revalidate(st *store.Store, cache map[string]store.CachedFile) { + if l == nil { + return + } + current := secrets.RulesVersion() + rescanAll := l.rec.Rules != current + unread := false + rels := make([]string, 0, len(l.rec.Paths)) + for rel := range l.rec.Paths { + rels = append(rels, rel) + } + for _, rel := range rels { + c, tracked := cache[rel] + if !tracked || c.Link != "" || c.Blob == "" { + l.drop(rel) + continue + } + if !rescanAll && l.rec.Paths[rel].Blob == c.Blob { + continue + } + found, ok := readFindings(st, c.Blob) + if !ok { + unread = true + continue + } + l.set(rel, c.Blob, found) + } + if rescanAll && !unread { + l.rec.Rules = current l.dirty = true } } @@ -76,8 +133,8 @@ func (l *secretLog) drop(rel string) { if l == nil { return } - if _, ok := l.found[rel]; ok { - delete(l.found, rel) + if _, ok := l.rec.Paths[rel]; ok { + delete(l.rec.Paths, rel) l.dirty = true } } diff --git a/internal/syncer/syncer.go b/internal/syncer/syncer.go index 2c667724..5148414f 100644 --- a/internal/syncer/syncer.go +++ b/internal/syncer/syncer.go @@ -48,7 +48,6 @@ import ( "github.com/runbear-io/beardrive/internal/config" "github.com/runbear-io/beardrive/internal/journal" "github.com/runbear-io/beardrive/internal/remote" - "github.com/runbear-io/beardrive/internal/secrets" "github.com/runbear-io/beardrive/internal/store" ) @@ -499,10 +498,7 @@ func (s *Session) cycleLocked(ctx context.Context, hint *ScanHint) (*Result, err loadedSync := store.CloneSyncState(st) // Advisory telemetry about files this cycle will journal and push anyway, // so an unreadable record starts the cycle empty rather than failing it. - sec := &secretLog{found: map[string][]secrets.Finding{}} - if found, err := s.Store.LoadSecrets(s.mountID()); err == nil { - sec.found = found - } + sec := newSecretLog(s.Store, s.mountID()) myOpCount, err := s.Store.OwnOpCount(s.Device.ID) if err != nil { return nil, fmt.Errorf("read own journal: %w", err) @@ -1102,8 +1098,9 @@ func (s *Session) finish(cache map[string]store.CachedFile, st store.SyncState, // advisory telemetry about ops that are already journaled and pushed, and // giving it a veto over convergence is the one invariant this feature is // close enough to break. + sec.revalidate(s.Store, cache) if sec != nil && sec.dirty { - if err := s.Store.SaveSecrets(s.mountID(), sec.found); err != nil { + if err := s.Store.SaveSecretRecord(s.mountID(), sec.rec); err != nil { log.Printf("beardrive: could not record credential findings: %v", err) } } diff --git a/internal/syncer/syncer_test.go b/internal/syncer/syncer_test.go index 13b9c967..da0f93fd 100644 --- a/internal/syncer/syncer_test.go +++ b/internal/syncer/syncer_test.go @@ -1636,6 +1636,113 @@ func TestSecretScanSkipsUnchangedFiles(t *testing.T) { } } +// A peer that edits the key out clears the warning here too: the scan reads +// only what this device changed, so the record has to follow the file to the +// blob the peer wrote. +func TestSecretClearsWhenAPeerFixesTheFile(t *testing.T) { + be := sharedRemote(t) + a := newDevice(t, "deva", be) + b := newDevice(t, "devb", be) + + write(t, a.Folder, "deploy.md", "key = "+testAWSKey+"\n") + cycle(t, a) + cycle(t, b) + write(t, b.Folder, "deploy.md", "key = read it from the environment\n") + cycle(t, b) + + cycle(t, a) + if got := read(t, a.Folder, "deploy.md"); strings.Contains(got, testAWSKey) { + t.Fatalf("control: the peer's fix did not arrive: %q", got) + } + if found, _ := a.Store.LoadSecrets(a.mountID()); len(found) != 0 { + t.Fatalf("findings = %+v, want empty once the peer removed the key", found) + } +} + +// Findings recorded under another rule set are judged again by the rules in +// force, on the next cycle and with nothing on disk changed: a detector fix +// clears its false positives without anyone editing the files. +func TestSecretFindingsFromOtherRulesAreReadAgain(t *testing.T) { + a := newDevice(t, "deva", sharedRemote(t)) + + write(t, a.Folder, "notes.md", "see https://example.com/helpdesk-agents-for-slack-and-teams\n") + write(t, a.Folder, "deploy.md", "key = "+testAWSKey+"\n") + cycle(t, a) + rec, err := a.Store.LoadSecretRecord(a.mountID()) + if err != nil { + t.Fatal(err) + } + cache, err := a.Store.LoadCache(a.mountID()) + if err != nil { + t.Fatal(err) + } + // What an older detector left behind: a false positive on notes.md, under + // another rule set, for the blob the file still holds. + rec.Rules = "older-rules" + rec.Paths["notes.md"] = store.SecretEntry{Blob: cache["notes.md"].Blob, Found: []secrets.Finding{{Rule: "openai_api_key", Line: 1}}} + if err := a.Store.SaveSecretRecord(a.mountID(), rec); err != nil { + t.Fatal(err) + } + + cycle(t, a) // nothing changed on disk + found, err := a.Store.LoadSecrets(a.mountID()) + if err != nil { + t.Fatal(err) + } + if _, ok := found["notes.md"]; ok { + t.Errorf("findings = %+v, want the false positive gone under the current rules", found) + } + if len(found["deploy.md"]) == 0 { + t.Errorf("findings = %+v, want the real key still reported", found) + } + if rec, _ := a.Store.LoadSecretRecord(a.mountID()); rec.Rules != secrets.RulesVersion() { + t.Errorf("record rules = %q, want %q", rec.Rules, secrets.RulesVersion()) + } +} + +// A finding whose blob cannot be read stays, and so do the rules it was +// recorded under: the next cycle reads it again instead of the warning +// vanishing on one failed read. +func TestSecretFindingSurvivesAnUnreadableBlob(t *testing.T) { + a := newDevice(t, "deva", sharedRemote(t)) + + write(t, a.Folder, "deploy.md", "key = "+testAWSKey+"\n") + cycle(t, a) + rec, err := a.Store.LoadSecretRecord(a.mountID()) + if err != nil { + t.Fatal(err) + } + rec.Rules = "older-rules" + if err := a.Store.SaveSecretRecord(a.mountID(), rec); err != nil { + t.Fatal(err) + } + blob := a.Store.BlobPath(rec.Paths["deploy.md"].Blob) + data, err := os.ReadFile(blob) + if err != nil { + t.Fatal(err) + } + if err := os.Remove(blob); err != nil { + t.Fatal(err) + } + + cycle(t, a) // nothing changed on disk; the blob cannot be read + got, _ := a.Store.LoadSecretRecord(a.mountID()) + if len(got.Paths["deploy.md"].Found) == 0 { + t.Fatalf("record = %+v, want the finding kept when its blob cannot be read", got) + } + if got.Rules != "older-rules" { + t.Fatalf("rules = %q, want the old rules kept so the next cycle retries", got.Rules) + } + + if err := os.WriteFile(blob, data, 0o600); err != nil { + t.Fatal(err) + } + cycle(t, a) + if got, _ := a.Store.LoadSecretRecord(a.mountID()); got.Rules != secrets.RulesVersion() || len(got.Paths["deploy.md"].Found) == 0 { + t.Fatalf("record = %+v, want it re-read under the current rules once readable", got) + } +} + func TestPullRejectsSymlinkTargetOnlySlotRedefinition(t *testing.T) { be := sharedRemote(t) s := newDevice(t, "local", be) diff --git a/internal/webapp/shares_test.go b/internal/webapp/shares_test.go index 63ceae5e..040e2c4f 100644 --- a/internal/webapp/shares_test.go +++ b/internal/webapp/shares_test.go @@ -895,6 +895,41 @@ func TestShareSecretScan(t *testing.T) { } } +// A private key's body may be wrapped at any width, and the share gate still +// refuses it; only the header quoted as a placeholder passes. +func TestShareRefusesAPrivateKeyWrappedInShortLines(t *testing.T) { + srv, p, _, f, h := shareHub(t) + body := "MIIEowIBAAKCAQEAexampleexampleexampleexample0123456789" + var wrapped []string + for len(body) > 16 { + wrapped, body = append(wrapped, body[:16]), body[16:] + } + wrapped = append(wrapped, body) + f.put("dev1", "deploy/server.pem", "-----BEGIN PRIVATE KEY-----\n"+strings.Join(wrapped, "\n")+"\n-----END PRIVATE KEY-----\n") + f.put("dev1", "docs/fields.md", "E.g. `-----BEGIN PRIVATE KEY-----\\n...` goes in this field\n") + + rec := postShare(t, srv, h, p.ID, map[string]string{"path": "deploy/server.pem"}) + if rec.Code != http.StatusConflict { + t.Fatalf("share of a wrapped private key: %d %s, want 409", rec.Code, rec.Body) + } + if got := decodeFindings(t, rec); !reflect.DeepEqual(got, []secrets.Finding{{Rule: "private_key", Line: 1}}) { + t.Fatalf("findings = %v, want private_key on line 1", got) + } + if rec := postShare(t, srv, h, p.ID, map[string]string{"path": "docs/fields.md"}); rec.Code != 200 { + t.Fatalf("a doc quoting the header as a placeholder: %d %s, want 200", rec.Code, rec.Body) + } + + f.put("dev1", "deploy/padded.pem", "-----BEGIN PRIVATE KEY-----\n"+strings.Join(wrapped, " \n")+" \n-----END PRIVATE KEY-----\n") + if rec := postShare(t, srv, h, p.ID, map[string]string{"path": "deploy/padded.pem"}); rec.Code != http.StatusConflict { + t.Fatalf("share of a wrapped private key with trailing blanks: %d %s, want 409", rec.Code, rec.Body) + } + + f.put("dev1", "deploy/creds.json", `{"private_key": "-----BEGIN PRIVATE KEY----- \n`+strings.Join(wrapped, ` \n`)+` \n-----END PRIVATE KEY-----\n"}`+"\n") + if rec := postShare(t, srv, h, p.ID, map[string]string{"path": "deploy/creds.json"}); rec.Code != http.StatusConflict { + t.Fatalf("share of a JSON-escaped wrapped private key with trailing blanks: %d %s, want 409", rec.Code, rec.Body) + } +} + // TestShareSecretNeverEchoed is the one rule that cannot bend: the matched // bytes never leave secrets.Scan — not in the body, not in the log. func TestShareSecretNeverEchoed(t *testing.T) {