diff --git a/plugins/lean-jev/.claude-plugin/plugin.json b/plugins/lean-jev/.claude-plugin/plugin.json index 4ed0d69..14e598e 100644 --- a/plugins/lean-jev/.claude-plugin/plugin.json +++ b/plugins/lean-jev/.claude-plugin/plugin.json @@ -3,7 +3,7 @@ "name": "lean-jev", "displayName": "Lean Jev", "description": "Ask Jev, TypeSafe's System One model, for calibrated typed judgments: yes/no probabilities, rubric scores, picks from a fixed set. Ships a stdlib-only Python client; an API key is the only prerequisite.", - "version": "0.2.0", + "version": "0.3.0", "author": { "name": "LeanCode" }, diff --git a/plugins/lean-jev/CHANGELOG.md b/plugins/lean-jev/CHANGELOG.md index fe6d176..7ef19b2 100644 --- a/plugins/lean-jev/CHANGELOG.md +++ b/plugins/lean-jev/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## 0.3.0 + +- Reading the TypeSafe docs is a precondition rather than a step inside the loop. `ask-jev/SKILL.md` opens with the pages to read before the first call: `concepts/state.md` every time, plus the page for each question type in use. It says plainly that the request contract is not a substitute for them, because a request can satisfy the contract, ignore the pages, and come back as a confident number about the wrong thing with nothing in the response to mark it. +- A page that will not load is now a reason to stop and tell the user. The clause it replaces allowed a call on the contract alone provided the agent said so in its report, which asked the agent that skipped the reads to be the one to disclose it. +- `jev.py` refuses a state over 400 characters passed as unnamed text, names `concepts/state.md`, and offers `--state-format text` for a genuine single blob such as a diff or a log. Flattening several items into one string is the framing mistake that costs the most, and it is the one an agent makes while believing the request is well formed. +- The examples in `SKILL.md` and `README.md` pass an object and point a question at one of its fields, so the shape a reader copies is the shape most states want. + ## 0.2.0 First public release, carried over from LeanCode's internal marketplace at the same version. diff --git a/plugins/lean-jev/README.md b/plugins/lean-jev/README.md index d148c26..ac24e7f 100644 --- a/plugins/lean-jev/README.md +++ b/plugins/lean-jev/README.md @@ -91,18 +91,24 @@ To skip the approval prompt on every call, allow the script in your Claude Code ```bash python3 /scripts/jev.py key-status python3 /scripts/jev.py ask \ - --state 'The export button crashes the settings page in Safari.' \ - --questions '{"is_bug": {"type": "noul", "instructions": "Does this report a software defect?"}}' + --state '{"ticket": "The export button crashes the settings page in Safari.", "reported_by": "support"}' \ + --questions '{"is_bug": {"type": "noul", "instructions": "Does `ticket` report a software defect?"}}' ``` `key-status` reports whether a key was found and from where, never the key. A `noul` near 1 from the second command means the whole path works. +State goes in a JSON object whenever it has more than one part, so every part carries a name a +question can point at with a backticked path. A bare string is for a single indivisible blob such +as a diff or a log, and the client asks you to declare that with `--state-format text` once such a +string runs long. + ## Troubleshooting | Symptom | Cause | Fix | | ------- | ----- | --- | | `python3: can't open file .../scripts/jev.py` | Wrong path, not a broken setup | The script sits at `/scripts/jev.py`, what `${CLAUDE_PLUGIN_ROOT}` expands to. `ls` the directory | +| `State is N characters of unnamed text` | Several items were flattened into one string | Put each part in a JSON object under its own key and point questions at them with backticked paths, per [`concepts/state.md`](https://docs.typesafe.ai/concepts/state.md). For a genuine single blob, pass `--state-format text` | | `{"key": "missing"}` from `key-status` | No key file | Write `~/.config/typesafe/env` as above; `ls -l` should show `-rw-------` | | 401 from TypeSafe | A key was found and rejected | Check it is current at console.typesafe.ai, or a stale `TYPESAFE_API_KEY` in the environment is shadowing the file. `key-status` names the source | | 422 from TypeSafe | Malformed request, and the API says how | Usually criteria shapes: `score` takes an ordered array, `choice` takes a map, and a `noul`'s optional criteria are a map too. The script checks those locally | diff --git a/plugins/lean-jev/scripts/jev.py b/plugins/lean-jev/scripts/jev.py index babb2e9..23c98c6 100755 --- a/plugins/lean-jev/scripts/jev.py +++ b/plugins/lean-jev/scripts/jev.py @@ -37,6 +37,8 @@ QUESTION_TYPES = {"noul", "score", "choice"} MAX_CHOICE_OPTIONS = 255 MAX_SCORE_LEVELS = 10 +# Past this many characters, an unnamed blob of text is almost never one thing. +MAX_UNSTRUCTURED_STATE = 400 class JevError(RuntimeError): @@ -104,12 +106,32 @@ def build_state(inline: str | None, path: Path | None, fmt: str) -> Any: raise JevError( "State is not valid JSON, and --state-format json was requested" ) from None + reject_unnamed_blob(raw) return raw if fmt == "json" or isinstance(parsed, (dict, list)): return parsed + reject_unnamed_blob(raw) return raw +def reject_unnamed_blob(raw: str) -> None: + """Refuse a long stretch of text that was never given named parts. + + Several items flattened into one string is the framing mistake that costs + the most: the questions end up referring to them in prose, the answers + degrade, and nothing in the response says why. Passing --state-format text + makes the single-blob case a deliberate answer rather than a default. + """ + if len(raw) <= MAX_UNSTRUCTURED_STATE: + return + raise JevError( + f"State is {len(raw)} characters of unnamed text. A state with more than one part " + "belongs in a JSON object, so each part has a name a question can point at with a " + "backticked path — see https://docs.typesafe.ai/concepts/state.md. If this really " + "is one indivisible blob, such as a diff or a log, pass --state-format text." + ) + + def validate_questions(questions: Any) -> None: """Catch locally what the API would otherwise reject with a 422.""" if not isinstance(questions, dict) or not questions: diff --git a/plugins/lean-jev/skills/ask-jev/SKILL.md b/plugins/lean-jev/skills/ask-jev/SKILL.md index 9298201..cd7447a 100644 --- a/plugins/lean-jev/skills/ask-jev/SKILL.md +++ b/plugins/lean-jev/skills/ask-jev/SKILL.md @@ -18,8 +18,36 @@ a list rather than a sample. This skill is how you hand a judgment over without mangling it. Jev reads a question literally, answers each one in isolation, and explains nothing, so the framing carries the whole weight: one factor per question, the right type for the job, state cut to what the question needs. -What follows is the general discipline, which is enough for a one-off question. A question that -keeps coming back earns a **recipe** — a written decomposition, kept in `references/`. +What follows is the general discipline. A question that keeps coming back earns a **recipe** — +a written decomposition, kept in `references/`. + +## Before the first call + +Read these first, in this session, before sending anything: + +- [`concepts/state.md`](https://docs.typesafe.ai/concepts/state.md) — every time. +- The page for each question type you are about to use. + +| What you need | Page | +| --- | --- | +| Yes/no — probability that a condition holds | [`primitives/noul.md`](https://docs.typesafe.ai/primitives/noul.md) | +| Position on ordered levels you describe | [`primitives/score.md`](https://docs.typesafe.ai/primitives/score.md) | +| One of a fixed set, with the distribution | [`primitives/choice.md`](https://docs.typesafe.ai/primitives/choice.md) | +| Reading confidence and picking thresholds | [`confidence.md`](https://docs.typesafe.ai/confidence.md) | +| Fields, limits, and error codes | [`api.md`](https://docs.typesafe.ai/api.md) | +| Combining several scored dimensions | [`patterns/composite-scoring.md`](https://docs.typesafe.ai/patterns/composite-scoring.md) | + +Start from the [index](https://docs.typesafe.ai/llms.txt) for anything not listed. Every page +serves Markdown by appending `.md` to its path; fetch that form, which returns the page itself +rather than a summary of it, and resolve relative links against `https://docs.typesafe.ai`. + +**The request contract below is not a substitute for these pages.** It says what the API accepts. +The pages say what the numbers mean and how to frame a question so the number is worth having. A +request that satisfies the contract and ignores the pages comes back as a confident number about +the wrong thing, and nothing in the response marks it. + +**If a page will not load, stop and tell the user.** Being unable to read the docs is a reason to +raise the problem, not a reason to call Jev anyway. ## Calling it @@ -28,10 +56,15 @@ keeps coming back earns a **recipe** — a written decomposition, kept in `refer ```bash python3 /scripts/jev.py ask \ - --state 'The export button crashes the settings page in Safari.' \ - --questions '{"is_bug": {"type": "noul", "instructions": "Does this report a software defect?"}}' + --state '{"ticket": "The export button crashes the settings page in Safari.", "reported_by": "support"}' \ + --questions '{"is_bug": {"type": "noul", "instructions": "Does `ticket` report a software defect?"}}' ``` +State is a JSON object whenever it has more than one part, so each part carries a name a question +can point at with a backticked path. A bare string is for a single indivisible blob — a diff, a +log, one document — and `jev.py` makes you say so with `--state-format text` once such a string +runs long, because several items flattened into one blob is the mistake that costs the most. + It reads the API key from `~/.config/typesafe/env` on its own, retries 429s and 529s, and prints the API's JSON. `--state-file ` sends a file instead — reach for it whenever the state is a diff, a log, or anything else large, because a file passed by path never enters this transcript. @@ -53,30 +86,6 @@ The request contract, which the script checks before spending a call: When a call errors, `references/setup.md` covers the failures by symptom. -## Read the page for the type you pick - -TypeSafe's documentation is the source of truth for what each type means and how to write its -criteria. This skill owns the decision to ask and the discipline around reporting; the docs own -the semantics, and they stay current as the model does. A page is one fetch — cheap enough to -read before writing criteria, and worth it, because a misread number fails silently. - -Start from the [index](https://docs.typesafe.ai/llms.txt). Every page serves Markdown by -appending `.md` to its path; fetch that form, which returns the page itself rather than a -summary of it, and resolve relative links against `https://docs.typesafe.ai`. - -| What you need | Page | -| --- | --- | -| Yes/no — probability that a condition holds | [`primitives/noul.md`](https://docs.typesafe.ai/primitives/noul.md) | -| Position on ordered levels you describe | [`primitives/score.md`](https://docs.typesafe.ai/primitives/score.md) | -| One of a fixed set, with the distribution | [`primitives/choice.md`](https://docs.typesafe.ai/primitives/choice.md) | -| What belongs in state, and how to shape it | [`concepts/state.md`](https://docs.typesafe.ai/concepts/state.md) | -| Reading confidence and picking thresholds | [`confidence.md`](https://docs.typesafe.ai/confidence.md) | -| Fields, limits, and error codes | [`api.md`](https://docs.typesafe.ai/api.md) | -| Combining several scored dimensions | [`patterns/composite-scoring.md`](https://docs.typesafe.ai/patterns/composite-scoring.md) | - -When the fetch fails — offline, sandboxed, or blocked — the contract above is enough to make a -well-formed call. Say in your report that you wrote the criteria without the page. - ## Why ask Jev instead of just answering Three reasons, and they are worth knowing so you can tell when none of them apply: @@ -113,9 +122,8 @@ are three Jev questions, and the judgment you build on top of them is yours — 1. **Name the decision.** What will you do differently depending on the answer? If nothing, do not ask. 2. **Decompose it** into atomic questions and pick a type for each: `noul` for a yes/no - probability, `score` for a position on ordered levels, `choice` for one of a fixed set. Read - that type's page — the number means something different in each, and the three are not - interchangeable. + probability, `score` for a position on ordered levels, `choice` for one of a fixed set. The + number means something different in each, and the three are not interchangeable. 3. **Build the state.** Only what the questions need. Accuracy falls as irrelevant material grows, and the state is capped around 32k tokens. Filter first. 4. **One `jev.py ask` call** carrying every question, including speculative ones you may not end diff --git a/plugins/lean-jev/tests/test_jev.py b/plugins/lean-jev/tests/test_jev.py index 34819db..4beb4e5 100644 --- a/plugins/lean-jev/tests/test_jev.py +++ b/plugins/lean-jev/tests/test_jev.py @@ -142,6 +142,30 @@ def test_missing_state_is_an_error(self): with self.assertRaises(jev.JevError): jev.build_state(None, None, "auto") + def test_a_long_unnamed_blob_is_refused(self): + blob = " ".join(f"L{n}. a lesson worth recording." for n in range(60)) + self.assertGreater(len(blob), jev.MAX_UNSTRUCTURED_STATE) + with self.assertRaises(jev.JevError) as caught: + jev.build_state(blob, None, "auto") + self.assertIn("concepts/state.md", str(caught.exception)) + + def test_a_long_blob_is_allowed_when_declared_text(self): + blob = "x" * (jev.MAX_UNSTRUCTURED_STATE + 1) + self.assertEqual(jev.build_state(blob, None, "text"), blob) + + def test_a_long_object_is_untouched(self): + state = {f"lesson_{n}": "a lesson worth recording." for n in range(60)} + raw = json.dumps(state) + self.assertGreater(len(raw), jev.MAX_UNSTRUCTURED_STATE) + self.assertEqual(jev.build_state(raw, None, "auto"), state) + + def test_a_long_blob_from_a_file_is_refused_too(self): + with tempfile.TemporaryDirectory() as tmp: + state_file = Path(tmp) / "state.txt" + state_file.write_text("x" * (jev.MAX_UNSTRUCTURED_STATE + 1)) + with self.assertRaises(jev.JevError): + jev.build_state(None, state_file, "auto") + class RedactTest(unittest.TestCase): def test_key_is_removed_from_text(self):