harness/store: decode the JSONL tail window losslessly

src/harness/store/mod.rs

@@ -350,10 +350,19 @@ impl JsonlAppendStore {
             file.seek(SeekFrom::Start(start)).map_err(|e| {
                 TinyAgentsError::Validation(format!("append store seek error: {e}"))
             })?;
-            let mut buf = String::new();
-            file.read_to_string(&mut buf).map_err(|e| {
+            let mut raw = Vec::new();
+            file.read_to_end(&mut raw).map_err(|e| {
                 TinyAgentsError::Validation(format!("append store read error: {e}"))
             })?;
+            // The window starts at an arbitrary byte offset, so it can begin in
+            // the middle of a multi-byte character. `read_to_string` rejects
+            // that outright ("stream did not contain valid UTF-8"), which fails
+            // the whole append and drops the journal record silently — and
+            // permanently, because the file then stops growing, so every later
+            // append reads the same broken window. Lossy decoding is safe here:
+            // the damaged bytes are by definition in the first line, which is
+            // discarded below anyway (`complete_from`).
+            let buf = String::from_utf8_lossy(&raw);
             // Only lines that are certainly complete: when the window did not
             // reach the start of the file, its first line may be cut in half.
             let complete_from = usize::from(start > 0);

src/harness/store/test.rs

@@ -425,3 +425,47 @@ async fn jsonl_rejects_unsafe_stream_names() {
     // Allowed characters round-trip.
     assert_eq!(store.append("runs-1.events_v2", json!(1)).await.unwrap(), 0);
 }
+
+#[tokio::test]
+async fn jsonl_append_survives_a_tail_window_that_starts_mid_character() {
+    // `next_offset` seeks to `len - 4096` and reads from there. That offset is
+    // an arbitrary byte, so on a file full of non-ASCII text (é, €, —) the
+    // window can begin halfway through a multi-byte character. Decoding it as
+    // UTF-8 then fails, which failed the whole append and dropped the record —
+    // permanently, because the file never grows again and every later append
+    // reads the same broken window.
+    let dir = TempDir::new("jsonl-utf8-window");
+    let path = dir.0.join("evts.jsonl");
+
+    // Pad the first record until byte `len - 4096` is a UTF-8 continuation byte
+    // (`10xxxxxx`), i.e. until the window provably starts inside a character.
+    let second = serde_json::to_string(&StoreRecord {
+        offset: 1,
+        value: json!({"n": 1}),
+        created_at_ms: 0,
+    })
+    .unwrap();
+    let mut pad = 0usize;
+    let first = loop {
+        let first = serde_json::to_string(&StoreRecord {
+            offset: 0,
+            value: json!({"tekst": format!("{}{}", "x".repeat(pad), "—".repeat(2000))}),
+            created_at_ms: 0,
+        })
+        .unwrap();
+        let len = first.len() + 1 + second.len() + 1;
+        if first.as_bytes()[len - 4096] & 0b1100_0000 == 0b1000_0000 {
+            break first;
+        }
+        pad += 1;
+    };
+    std::fs::write(&path, format!("{first}\n{second}\n")).unwrap();
+
+    let store = JsonlAppendStore::new(&dir.0);
+    assert_eq!(
+        store.append("evts", json!({"n": 2})).await.unwrap(),
+        2,
+        "an append must not be lost because the tail window starts mid-character"
+    );
+    assert_eq!(store.len("evts").await.unwrap(), 3);
+}