Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion architecture/cli-sync.md
Original file line number Diff line number Diff line change
Expand Up @@ -190,7 +190,7 @@ classDiagram
+LoadSecrets / SaveSecrets mountID
+Lock() / TryLock() flock
}
note for Store "secrets-mount-id.json is the credential-finding record: what `bdrive status` prints and what the agent hook appends a sentence about. Like the inbound spool it outlives the cycle that wrote it — the daemon scans the write seconds before the turn starts — but unlike it, it is STATE and is never drained: a finding stands until the file changes without it"
note for Store "secrets-mount-id.json is the credential-finding record: what `bdrive status` prints and what the agent hook appends a sentence about. Like the inbound spool it outlives the cycle that wrote it — the daemon scans the write seconds before the turn starts — but unlike it, it is STATE and is never drained: a finding stands until the file changes without it. Each path's entry records the blob it was read from and the record carries the rule-set version (secrets.RulesVersion); finish re-reads any entry whose file now holds another blob (a peer's edit, which the scan never reads) or whose rules differ (a detector fix), and drops entries for untracked paths"
note for Store "internal/store: ~/.bdrive/volumes/volume-id holds shared content-addressed blobs, per-device journal copies, and sync.json; state-mount-id.json, paused-mount-id, note-mount-id.json, secrets-mount-id.json, and inbound-mount-id.jsonl are per-mount (free funcs Paused/SetPaused and PausedMount/SetPausedMount, no flock)"
note for Store "inbound-mount-id.jsonl (with legacy inbound.jsonl fallback) is the read spool's twin, running the other way: materialize appends every path it wrote or removed for a peer, and `sync --hook` drains it into the turn's context (re-read before editing). Appends and the drain share an flock on the spool's .lock sidecar, since the drain runs outside the volume lock beside a daemon that may be appending. A spool and not a Result field because the daemon usually materializes the change seconds before the turn starts, so the hook's own cycle sees nothing. Capped, best-effort, never fails a cycle. Result.Inbound now carries the SAME events for the post_sync hook and is not a duplicate to delete: that consumer fires from the cycle itself, and a second drainer would silently empty the agent hook's context"
note for Store "The fold-cache keeps the replayed target, op count, lamport high-water mark, and ignore winner together; ReplayFold reuses unchanged journal folds instead of re-reading and sorting every device on every cycle. LoadCacheReadOnly supplies the immutable map used by echo suppression without cloning large mount caches."
Expand Down
18 changes: 14 additions & 4 deletions cmd/bdrive/hooksync_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -445,6 +445,8 @@ func TestSyncHookModeInboundSpoolUnreadable(t *testing.T) {
}
}

// writeSecretFile puts a credential-shaped string on the given line of rel, so
// the hook's cycle records the finding the way a daemon scan would.
func seedSecrets(t *testing.T, proj config.Project, found map[string][]secrets.Finding) {
t.Helper()
vdir, err := config.VolumeDir(proj.ID)
Expand All @@ -460,6 +462,14 @@ func seedSecrets(t *testing.T, proj config.Project, found map[string][]secrets.F
}
}

func writeSecretFile(t *testing.T, dir, rel, secret string, line int) {
t.Helper()
body := strings.Repeat("text\n", line-1) + secret + "\n"
if err := os.WriteFile(filepath.Join(dir, rel), []byte(body), 0o644); err != nil {
t.Fatal(err)
}
}

// The credential warning reaches the agent, whose own cycle found nothing:
// the daemon scanned that write seconds ago, so — exactly like the inbound
// spool — the record is what carries it. Advisory: the sentence says the file
Expand All @@ -468,10 +478,8 @@ func TestSyncHookModeReportsSecrets(t *testing.T) {
t.Setenv("BDRIVE_HOME", t.TempDir())
root := t.TempDir()
root, _ = filepath.EvalSymlinks(root)
proj := mountAt(t, root, "wiki", "https://hub.example.com/p/p-12345678")
seedSecrets(t, proj, map[string][]secrets.Finding{
"deploy.md": {{Rule: "aws_access_key_id", Line: 12}},
})
mountAt(t, root, "wiki", "https://hub.example.com/p/p-12345678")
writeSecretFile(t, filepath.Join(root, "wiki"), "deploy.md", "export AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE", 12)

got := runHook(t, filepath.Join(root, "wiki"))
for _, want := range []string{
Expand All @@ -498,6 +506,8 @@ func TestSyncHookModeSecretsMultipleMounts(t *testing.T) {
root, _ = filepath.EvalSymlinks(root)
a := mountAt(t, root, "projA", "https://hub.example.com/p/p-aaaaaaaa")
b := mountAt(t, root, "projB", "https://hub.example.com/p/p-bbbbbbbb")
// Mounts below the session folder belong to their daemons and the hook
// never cycles them, so the daemon's record is all the hook reads.
seedSecrets(t, a, map[string][]secrets.Finding{"a.md": {{Rule: "private_key", Line: 1}}})
seedSecrets(t, b, map[string][]secrets.Finding{"b.md": {{Rule: "slack_token", Line: 2}}})

Expand Down
23 changes: 22 additions & 1 deletion internal/secrets/secrets.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@ package secrets

import (
"bytes"
"crypto/sha256"
"encoding/hex"
"regexp"
"sort"
)
Expand Down Expand Up @@ -47,10 +49,29 @@ var secretRules = []struct {
{"openai_api_key", regexp.MustCompile(`(?:^|[^A-Za-z0-9_/-])(sk-[A-Za-z0-9_-]{20,})`)},
{"github_pat", regexp.MustCompile(`ghp_[A-Za-z0-9]{36}`)},
{"slack_token", regexp.MustCompile(`xox[baprs]-[A-Za-z0-9-]{10,}`)},
{"private_key", regexp.MustCompile(`-----BEGIN [A-Z ]*PRIVATE KEY-----`)},
// A header alone is not a key: docs quote it as a placeholder
// ("-----BEGIN PRIVATE KEY-----\n..."). The body must follow, after a real
// or JSON-escaped line break and any RFC 1421 headers (Proc-Type:,
// DEK-Info:) an encrypted PEM carries: 32 base64 characters, counted
// across the line breaks, real or escaped, a PEM may wrap its body at, at
// any width, with blanks on either side of each break.
{"private_key", regexp.MustCompile(`-----BEGIN [A-Z ]*PRIVATE KEY-----(?:\s|\\[rnt])+(?:[A-Za-z-]+:[^\n\\]*(?:\s|\\[rnt])+)*(?:[A-Za-z0-9+/=](?:(?:[ \t]|\\t)*(?:\r?\n|\\[rn])(?:[ \t]|\\t)*)*){32}`)},
{"gitlab_pat", regexp.MustCompile(`glpat-[A-Za-z0-9_-]{20,}`)},
}

// rulesVersion names this rule set; see RulesVersion.
var rulesVersion = func() string {
h := sha256.New()
for _, r := range secretRules {
h.Write([]byte(r.id + "\x00" + r.re.String() + "\x00"))
}
return hex.EncodeToString(h.Sum(nil))[:12]
}()

// RulesVersion changes whenever a rule does. A finding recorded under another
// version describes a rule set no longer in force, so its file is read again.
func RulesVersion() string { return rulesVersion }

// labels are the rule ids in words. They existed only in the frontend
// (SECRET_LABELS in Browser.tsx) while `bdrive share` printed the bare id, so
// the same finding read as "an AWS access key" in the browser and
Expand Down
23 changes: 22 additions & 1 deletion internal/secrets/secrets_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,18 @@ const (
fakeSlackTok = "xoxb-1234567890-abcdefghij"
fakeGitLabPAT = "glpat-abcdefghij0123456789XY"
fakePrivKey = "-----BEGIN RSA PRIVATE KEY-----"
fakeKeyBody = "MIIEowIBAAKCAQEAexampleexampleexampleexample0123456789"
)

// wrap breaks s into lines of n characters, as a PEM body may be.
func wrap(s string, n int) string {
var lines []string
for len(s) > n {
lines, s = append(lines, s[:n]), s[n:]
}
return strings.Join(append(lines, s), "\n")
}

func TestScanSecrets(t *testing.T) {
for _, tc := range []struct {
name string
Expand All @@ -39,7 +49,18 @@ func TestScanSecrets(t *testing.T) {
{"github", "\n\n" + fakeGitHubPAT, []Finding{{"github_pat", 3}}},
{"slack", "token: " + fakeSlackTok, []Finding{{"slack_token", 1}}},
{"gitlab", "x\n" + fakeGitLabPAT, []Finding{{"gitlab_pat", 2}}},
{"private key", "a\nb\nc\n" + fakePrivKey + "\nMIIE...\n", []Finding{{"private_key", 4}}},
{"private key", "a\nb\nc\n" + fakePrivKey + "\n" + fakeKeyBody + "\n", []Finding{{"private_key", 4}}},
{"private key, JSON-escaped", `{"key": "` + fakePrivKey + `\n` + fakeKeyBody + `\n"}`, []Finding{{"private_key", 1}}},
{"private key, encrypted PEM headers", fakePrivKey + "\nProc-Type: 4,ENCRYPTED\nDEK-Info: AES-128-CBC,0123456789ABCDEF\n\n" + fakeKeyBody + "\n", []Finding{{"private_key", 1}}},
{"private key wrapped in short lines", fakePrivKey + "\n" + wrap(fakeKeyBody, 16) + "\n", []Finding{{"private_key", 1}}},
{"private key wrapped in short lines with trailing blanks", fakePrivKey + " \n" + strings.ReplaceAll(wrap(fakeKeyBody, 16), "\n", " \t\n") + " \n", []Finding{{"private_key", 1}}},
{"private key wrapped in short CRLF lines with trailing blanks", fakePrivKey + "\r\n" + strings.ReplaceAll(wrap(fakeKeyBody, 16), "\n", " \r\n") + "\r\n", []Finding{{"private_key", 1}}},
{"JSON-escaped private key wrapped in short lines with trailing blanks", `{"key": "` + fakePrivKey + ` \n` + strings.ReplaceAll(wrap(fakeKeyBody, 16), "\n", ` \n`) + ` \n"}`, []Finding{{"private_key", 1}}},
{"JSON-escaped private key wrapped in short tab-indented lines", `{"key": "` + fakePrivKey + `\n\t` + strings.ReplaceAll(wrap(fakeKeyBody, 16), "\n", `\t\n\t`) + `\n"}`, []Finding{{"private_key", 1}}},
{"private key wrapped in short indented lines", "key: |\n " + fakePrivKey + "\n " + strings.ReplaceAll(wrap(fakeKeyBody, 8), "\n", "\n ") + "\n", []Finding{{"private_key", 2}}},
// A doc quoting the header as a placeholder holds no key.
{"private key header as a placeholder", "E.g. `-----BEGIN PRIVATE KEY-----\\n...` in the field", nil},
{"private key header alone", fakePrivKey + "\n...\n", nil},
{
// One line, one rule, three keys: one finding, not three.
"multi key line deduped",
Expand Down
93 changes: 81 additions & 12 deletions internal/store/secrets.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,10 @@
package store

import (
"bytes"
"encoding/json"
"os"

"github.com/runbear-io/beardrive/internal/secrets"
)

Expand All @@ -27,31 +31,96 @@ func (s *Store) secretsPath(mountID string) (string, error) {
return s.mountStatePath("secrets-", mountID)
}

// LoadSecrets returns the findings recorded for each mount-relative path.
func (s *Store) LoadSecrets(mountID string) (map[string][]secrets.Finding, error) {
// SecretRecord is the findings file: what the rule set named by Rules found
// in each path, as of the blob that path held when it was read.
type SecretRecord struct {
Rules string `json:"rules"`
Paths map[string]SecretEntry `json:"paths"`
}

// SecretEntry is one path's findings and the content they describe. A path
// whose file now holds another blob, or a record written under other rules,
// is read again rather than trusted.
type SecretEntry struct {
Blob string `json:"blob,omitempty"`
Found []secrets.Finding `json:"found"`
}

// LoadSecretRecord reads the findings file. A file in the bare per-path shape
// ({"path": [findings]}) loads with no rules and no blobs, so every path in it
// is read again.
func (s *Store) LoadSecretRecord(mountID string) (SecretRecord, error) {
rec := SecretRecord{Paths: map[string]SecretEntry{}}
p, err := s.secretsPath(mountID)
if err != nil {
return nil, err
return rec, err
}
out := map[string][]secrets.Finding{}
if err := readJSON(p, &out); err != nil {
return nil, err
data, err := os.ReadFile(p)
if os.IsNotExist(err) {
return rec, nil
}
if err != nil {
return rec, err
}
var top map[string]json.RawMessage
if err := json.Unmarshal(data, &top); err != nil {
return rec, err
}
// Bare-shape values are arrays, so an object under "paths" is the record.
if raw := bytes.TrimSpace(top["paths"]); len(raw) > 0 && raw[0] == '{' {
if err := json.Unmarshal(data, &rec); err != nil {
return SecretRecord{Paths: map[string]SecretEntry{}}, err
}
if rec.Paths == nil {
rec.Paths = map[string]SecretEntry{}
}
} else {
for rel, raw := range top {
var f []secrets.Finding
if err := json.Unmarshal(raw, &f); err != nil {
return SecretRecord{Paths: map[string]SecretEntry{}}, err
}
rec.Paths[rel] = SecretEntry{Found: f}
}
}
// Same treatment as the cache's keys: plain JSON in $BDRIVE_HOME that
// anything running as the user can write, rendered into a terminal and
// joined against cache keys. An out-of-volume path is dropped, not shown.
for rel, f := range out {
if !cleanRel(rel) || len(f) == 0 {
delete(out, rel)
for rel, e := range rec.Paths {
if !cleanRel(rel) || len(e.Found) == 0 {
delete(rec.Paths, rel)
}
}
return out, nil
return rec, nil
}

func (s *Store) SaveSecrets(mountID string, f map[string][]secrets.Finding) error {
func (s *Store) SaveSecretRecord(mountID string, rec SecretRecord) error {
p, err := s.secretsPath(mountID)
if err != nil {
return err
}
return WriteJSONAtomic(p, f)
return WriteJSONAtomic(p, rec)
}

// LoadSecrets returns the findings recorded for each mount-relative path.
func (s *Store) LoadSecrets(mountID string) (map[string][]secrets.Finding, error) {
rec, err := s.LoadSecretRecord(mountID)
if err != nil {
return nil, err
}
out := make(map[string][]secrets.Finding, len(rec.Paths))
for rel, e := range rec.Paths {
out[rel] = e.Found
}
return out, nil
}

// SaveSecrets records findings with no blob or rules, so a cycle reads each
// path again before trusting them.
func (s *Store) SaveSecrets(mountID string, f map[string][]secrets.Finding) error {
rec := SecretRecord{Paths: make(map[string]SecretEntry, len(f))}
for rel, found := range f {
rec.Paths[rel] = SecretEntry{Found: found}
}
return s.SaveSecretRecord(mountID, rec)
}
40 changes: 40 additions & 0 deletions internal/store/secrets_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -82,3 +82,43 @@ func TestLoadSecretsDropsForeignKeys(t *testing.T) {
t.Fatalf("mode = %v, want 0600", fi.Mode().Perm())
}
}

// A record in the bare per-path shape loads with no rules and no blobs, which
// is what makes the next cycle read every path in it again.
func TestLoadSecretRecordReadsTheBarePerPathShape(t *testing.T) {
s, err := Open(t.TempDir())
if err != nil {
t.Fatal(err)
}
p, err := s.secretsPath("m1")
if err != nil {
t.Fatal(err)
}
if err := os.WriteFile(p, []byte(`{"notes.md": [{"rule":"openai_api_key","line":49}]}`), 0o600); err != nil {
t.Fatal(err)
}
rec, err := s.LoadSecretRecord("m1")
if err != nil {
t.Fatal(err)
}
e, ok := rec.Paths["notes.md"]
if rec.Rules != "" || !ok || e.Blob != "" || len(e.Found) != 1 {
t.Fatalf("record = %+v, want notes.md with no rules and no blob", rec)
}

rec.Rules = "r1"
rec.Paths["notes.md"] = SecretEntry{Blob: "abc", Found: e.Found}
if err := s.SaveSecretRecord("m1", rec); err != nil {
t.Fatal(err)
}
got, err := s.LoadSecretRecord("m1")
if err != nil {
t.Fatal(err)
}
if got.Rules != "r1" || got.Paths["notes.md"].Blob != "abc" {
t.Fatalf("round trip = %+v", got)
}
if found, _ := s.LoadSecrets("m1"); len(found["notes.md"]) != 1 {
t.Fatalf("LoadSecrets = %+v, want notes.md", found)
}
}
Loading
Loading