Merge pull request #183 from bogdanbaciu21/skoc-002-propagate-skill-hint-to-taskrecord

feat(sleep): propagate harvested skill hints into mined tasks

Author: Yif-Yang

skillopt_sleep/dream.py

@@ -22,7 +22,6 @@
 from skillopt_sleep.consolidate import ConsolidationResult, consolidate
 from skillopt_sleep.types import TaskRecord
 
-
 # ── synthetic augmentation ("dream up" variants of today's tasks) ─────────────
 
 _WRAPPERS = [
@@ -49,6 +48,7 @@ def dream_augment(real_tasks: List[TaskRecord], *, factor: int = 1) -> List[Task
                 judge=dict(t.judge), system=t.system,
                 tags=list(t.tags) + ["dream"], split="train",
                 origin="dream", derived_from=t.id,
+                skill_hint=t.skill_hint,
             ))
     return out
 
@@ -90,6 +90,7 @@ def recall_similar(new_tasks: List[TaskRecord], history: List[TaskRecord],
             reference=h.reference, judge=dict(h.judge), system=h.system,
             tags=list(h.tags) + ["recall"], split="train", origin="real",
             derived_from=h.id,
+            skill_hint=h.skill_hint,
         ))
     return out
 

skillopt_sleep/llm_miner.py

@@ -24,6 +24,7 @@
 
 from skillopt_sleep import prompts as prompt_registry
 from skillopt_sleep.backend import Backend, _extract_json
+from skillopt_sleep.mine import session_skill_hint
 from skillopt_sleep.types import SessionDigest, TaskRecord
 
 
@@ -65,13 +66,15 @@ def _mk_task(d: SessionDigest, obj: Dict[str, Any], idx: int) -> TaskRecord | No
             reference_kind="rule", judge={"kind": "rule", "checks": clean_checks},
             outcome="success" if satisfied else "fail",
             tags=["mined:llm"], source_sessions=[d.session_id],
+            skill_hint=session_skill_hint(d),
         )
     if rubric:
         return TaskRecord(
             id=tid, project=d.project, intent=intent,
             reference_kind="rubric", reference=rubric,
             outcome="success" if satisfied else "fail",
             tags=["mined:llm"], source_sessions=[d.session_id],
+            skill_hint=session_skill_hint(d),
         )
     return None  # not checkable -> drop
 

skillopt_sleep/mine.py

@@ -18,7 +18,7 @@
 import os
 import re
 from collections import Counter
-from typing import Any, Callable, List, Optional, Set, Tuple
+from typing import Callable, List, Optional, Set, Tuple
 
 from skillopt_sleep.backend import CursorBackendError
 from skillopt_sleep.types import SessionDigest, TaskRecord
@@ -34,6 +34,18 @@ def _short(text: str, n: int = 600) -> str:
     return text if len(text) <= n else text[:n] + " …"
 
 
+def session_skill_hint(digest: SessionDigest) -> str:
+    """Return the session's unambiguous skill, or "" when there is none.
+
+    A session that invoked exactly one skill names the skill its tasks
+    exercised. Zero skills (unknown) and several skills (ambiguous) both
+    return "", which leaves those tasks in the existing catch-all path.
+    """
+    skills = [s for s in (digest.skills_used or []) if s]
+    unique = list(dict.fromkeys(skills))
+    return unique[0] if len(unique) == 1 else ""
+
+
 def _looks_negative(signals: List[str]) -> bool:
     return any(s.startswith("neg:") for s in signals)
 
@@ -196,6 +208,7 @@ def heuristic_mine(
                 reference="",
                 tags=tags,
                 source_sessions=[d.session_id],
+                skill_hint=session_skill_hint(d),
             )
         )
         if len(tasks) >= max_tasks:
@@ -206,7 +219,10 @@ def heuristic_mine(
 def dedup_tasks(tasks: List[TaskRecord]) -> List[TaskRecord]:
     """Merge tasks sharing an id (same project+intent across sessions)."""
     by_id: dict = {}
+    hints_by_id: dict = {}
     for t in tasks:
+        if t.skill_hint:
+            hints_by_id.setdefault(t.id, set()).add(t.skill_hint)
         if t.id in by_id:
             ex = by_id[t.id]
             ex.source_sessions = list(dict.fromkeys(ex.source_sessions + t.source_sessions))
@@ -216,6 +232,9 @@ def dedup_tasks(tasks: List[TaskRecord]) -> List[TaskRecord]:
                 ex.outcome = t.outcome
         else:
             by_id[t.id] = t
+    for task_id, task in by_id.items():
+        hints = hints_by_id.get(task_id, set())
+        task.skill_hint = next(iter(hints)) if len(hints) == 1 else ""
     return list(by_id.values())
 
 

skillopt_sleep/types.py

@@ -80,6 +80,10 @@ class TaskRecord:
     # allowed into val/test, which is the anti-overfitting guarantee.
     origin: str = "real"
     derived_from: str = ""            # for dream tasks: the real task id it varies
+    # Which skill this task exercised, when the source session invoked exactly
+    # one. Empty means unknown or ambiguous, which keeps the task in the
+    # existing managed-skill catch-all path. Kept last for positional compatibility.
+    skill_hint: str = ""
 
     def to_dict(self) -> Dict[str, Any]:
         return asdict(self)

tests/test_sleep_skill_hint.py

@@ -0,0 +1,200 @@
+"""Tests for propagating harvested skill hints into mined tasks (issue #120).
+
+Pure-stdlib (unittest), deterministic, no API key, no third-party deps.
+Run:  python -m pytest tests/test_sleep_skill_hint.py
+"""
+from __future__ import annotations
+
+import unittest
+from itertools import permutations, product
+
+from skillopt_sleep.dream import dream_augment, recall_similar
+from skillopt_sleep.llm_miner import _mk_task
+from skillopt_sleep.mine import dedup_tasks, heuristic_mine, session_skill_hint
+from skillopt_sleep.types import SessionDigest, TaskRecord
+
+
+def _digest(session_id="s1", skills=(), project="/repo/example", prompt="add a login form"):
+    return SessionDigest(
+        session_id=session_id,
+        project=project,
+        user_prompts=[prompt],
+        assistant_finals=["done"],
+        tools_used=["Skill"] if skills else ["Read"],
+        skills_used=list(skills),
+    )
+
+
+class TestTaskRecordSkillHint(unittest.TestCase):
+    def test_defaults_to_empty_string(self):
+        self.assertEqual(TaskRecord(id="t1", project="/repo/example", intent="x").skill_hint, "")
+
+    def test_serialization_roundtrip(self):
+        task = TaskRecord(
+            id="t1", project="/repo/example", intent="x", skill_hint="example-skill"
+        )
+        self.assertEqual(task.to_dict()["skill_hint"], "example-skill")
+        self.assertEqual(TaskRecord.from_dict(task.to_dict()).skill_hint, "example-skill")
+
+    def test_legacy_payload_without_skill_hint_still_loads(self):
+        legacy = {"id": "t1", "project": "/repo/example", "intent": "x", "split": "val"}
+        task = TaskRecord.from_dict(legacy)
+        self.assertEqual(task.skill_hint, "")
+        self.assertEqual(task.split, "val")
+
+    def test_existing_positional_fields_keep_their_slots(self):
+        task = TaskRecord(
+            "t1", "/repo/example", "x", "context", "system", "attempt",
+            "success", "exact", "answer", {}, ["tag"], ["s1"],
+            "val", "dream", "source-task",
+        )
+        self.assertEqual(task.split, "val")
+        self.assertEqual(task.origin, "dream")
+        self.assertEqual(task.derived_from, "source-task")
+        self.assertEqual(task.skill_hint, "")
+
+
+class TestSessionSkillHint(unittest.TestCase):
+    def test_single_skill_is_unambiguous(self):
+        self.assertEqual(session_skill_hint(_digest(skills=["example-skill"])), "example-skill")
+
+    def test_no_skill_is_empty(self):
+        self.assertEqual(session_skill_hint(_digest()), "")
+
+    def test_several_skills_are_ambiguous(self):
+        digest = _digest(skills=["example-skill", "second-skill"])
+        self.assertEqual(session_skill_hint(digest), "")
+
+    def test_repeated_single_skill_stays_unambiguous(self):
+        digest = _digest(skills=["example-skill", "example-skill"])
+        self.assertEqual(session_skill_hint(digest), "example-skill")
+
+
+class TestHeuristicMineSkillHint(unittest.TestCase):
+    def test_hint_propagates_from_session(self):
+        tasks = heuristic_mine([_digest(skills=["example-skill"])])
+        self.assertEqual([t.skill_hint for t in tasks], ["example-skill"])
+
+    def test_ambiguous_and_absent_hints_stay_in_catch_all(self):
+        tasks = heuristic_mine([
+            _digest(session_id="s1", skills=["example-skill", "second-skill"]),
+            _digest(session_id="s2", prompt="write the release notes"),
+        ])
+        self.assertEqual(len(tasks), 2)
+        self.assertEqual([t.skill_hint for t in tasks], ["", ""])
+
+    def test_dedup_keeps_agreeing_hint_and_clears_conflicting_one(self):
+        agreed = dedup_tasks([
+            TaskRecord(id="t1", project="/p", intent="x", skill_hint="example-skill",
+                       source_sessions=["s1"]),
+            TaskRecord(id="t1", project="/p", intent="x", skill_hint="example-skill",
+                       source_sessions=["s2"]),
+        ])
+        self.assertEqual([t.skill_hint for t in agreed], ["example-skill"])
+
+        filled = dedup_tasks([
+            TaskRecord(id="t1", project="/p", intent="x", source_sessions=["s1"]),
+            TaskRecord(id="t1", project="/p", intent="x", skill_hint="example-skill",
+                       source_sessions=["s2"]),
+        ])
+        self.assertEqual([t.skill_hint for t in filled], ["example-skill"])
+
+        conflicting = dedup_tasks([
+            TaskRecord(id="t1", project="/p", intent="x", skill_hint="example-skill",
+                       source_sessions=["s1"]),
+            TaskRecord(id="t1", project="/p", intent="x", skill_hint="second-skill",
+                       source_sessions=["s2"]),
+        ])
+        self.assertEqual([t.skill_hint for t in conflicting], [""])
+
+    def test_dedup_conflict_is_permutation_independent(self):
+        sequences = set(permutations(["skill-a", "skill-b", "skill-a"]))
+        sequences.update(permutations(["skill-b", "skill-a", "skill-b"]))
+        for hints in sequences:
+            with self.subTest(hints=hints):
+                tasks = [
+                    TaskRecord(id="t1", project="/p", intent="x", skill_hint=hint,
+                               source_sessions=[f"s{idx}"])
+                    for idx, hint in enumerate(hints)
+                ]
+                self.assertEqual(dedup_tasks(tasks)[0].skill_hint, "")
+
+    def test_dedup_matches_final_non_empty_hint_set_exhaustively(self):
+        for length in range(1, 5):
+            for hints in product(("", "skill-a", "skill-b"), repeat=length):
+                with self.subTest(hints=hints):
+                    distinct = {hint for hint in hints if hint}
+                    expected = next(iter(distinct)) if len(distinct) == 1 else ""
+                    tasks = [
+                        TaskRecord(id="t1", project="/p", intent="x", skill_hint=hint)
+                        for hint in hints
+                    ]
+                    self.assertEqual(dedup_tasks(tasks)[0].skill_hint, expected)
+
+    def test_dedup_ignores_empty_hints_when_resolving_final_set(self):
+        cases = [
+            (("", "skill-a", ""), "skill-a"),
+            (("skill-a", "", "skill-a"), "skill-a"),
+            (("", "", ""), ""),
+            (("skill-a", "", "skill-b"), ""),
+        ]
+        for hints, expected in cases:
+            with self.subTest(hints=hints):
+                tasks = [
+                    TaskRecord(id="t1", project="/p", intent="x", skill_hint=hint)
+                    for hint in hints
+                ]
+                self.assertEqual(dedup_tasks(tasks)[0].skill_hint, expected)
+
+
+class TestDreamAndRecallSkillHint(unittest.TestCase):
+    def test_dream_augment_preserves_hint(self):
+        source = TaskRecord(
+            id="t1", project="/p", intent="add validation", skill_hint="example-skill"
+        )
+        dreamed = dream_augment([source], factor=1)
+        self.assertEqual(len(dreamed), 1)
+        self.assertEqual(dreamed[0].skill_hint, "example-skill")
+
+    def test_recall_similar_preserves_hint(self):
+        new = TaskRecord(id="new", project="/p", intent="add form validation")
+        historical = TaskRecord(
+            id="old", project="/p", intent="improve form validation",
+            skill_hint="example-skill",
+        )
+        recalled = recall_similar([new], [historical], 1)
+        self.assertEqual(len(recalled), 1)
+        self.assertEqual(recalled[0].skill_hint, "example-skill")
+
+
+class TestLlmMinerSkillHint(unittest.TestCase):
+    def test_rule_task_carries_the_hint(self):
+        task = _mk_task(
+            _digest(skills=["example-skill"]),
+            {"intent": "add a login form", "checks": [{"op": "contains", "arg": "form"}]},
+            0,
+        )
+        self.assertIsNotNone(task)
+        self.assertEqual(task.reference_kind, "rule")
+        self.assertEqual(task.skill_hint, "example-skill")
+
+    def test_rubric_task_carries_the_hint(self):
+        task = _mk_task(
+            _digest(skills=["example-skill"]),
+            {"intent": "add a login form", "rubric": "mentions validation"},
+            0,
+        )
+        self.assertEqual(task.reference_kind, "rubric")
+        self.assertEqual(task.skill_hint, "example-skill")
+
+    def test_ambiguous_session_leaves_hint_empty(self):
+        task = _mk_task(
+            _digest(skills=["example-skill", "second-skill"]),
+            {"intent": "add a login form", "rubric": "mentions validation"},
+            0,
+        )
+        self.assertEqual(task.skill_hint, "")
+
+
+if __name__ == "__main__":
+    unittest.main()