Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 27 additions & 4 deletions skillopt_sleep/mine.py
Original file line number Diff line number Diff line change
Expand Up @@ -337,13 +337,36 @@ def _promote_one(*, to: str, from_splits: set[str]) -> None:
else:
t.split = "train"

import logging

# Guarantee val (the gate) is non-empty when we have >=2 real tasks.
# Only promote from train so hash-assigned test tasks stay untouched.
# Only promote from train so hash-assigned test tasks stay untouched,
# UNLESS train is also empty, in which case we must borrow from test.
if len(real) >= 2 and not any(t.split == "val" for t in real):
_promote_one(to="val", from_splits={"train"})
# Guarantee a train pool exists when possible; never borrow from test.
if not any(t.split == "train" for t in real):
logging.getLogger("skillopt_sleep").warning(
f"assign_splits: all {len(real)} real tasks hashed into test "
f"(val_fraction={val_fraction}, test_fraction={test_fraction}, seed={seed}). "
"Borrowing from test to satisfy non-empty val."
)
_promote_one(to="val", from_splits={"test"})
else:
_promote_one(to="val", from_splits={"train"})

# Guarantee a train pool exists when possible; never borrow from test
# UNLESS val has only 1 task (so taking from val would re-empty the gate)
# and test tasks are available.
if not any(t.split == "train" for t in tasks) and len(real) >= 2:
_promote_one(to="train", from_splits={"val"})
val_count = sum(1 for t in real if t.split == "val")
if val_count <= 1 and any(t.split == "test" for t in real):
logging.getLogger("skillopt_sleep").warning(
f"assign_splits: pulling from test to satisfy non-empty train "
f"because val has only {val_count} real task "
f"(val_fraction={val_fraction}, test_fraction={test_fraction}, seed={seed})."
)
_promote_one(to="train", from_splits={"test"})
else:
_promote_one(to="train", from_splits={"val"})
return tasks


Expand Down
14 changes: 14 additions & 0 deletions tests/test_split_hardening_2x3.py
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,20 @@ def test_hash_assigned_test_not_demoted_for_val_top_up(self):
if t.id in test_ids:
self.assertEqual(t.split, "test")

def test_val_and_train_pull_from_test_when_empty(self):
"""When all real tasks hash to test, val and train borrow from test to satisfy non-empty guarantees."""
# Seed 42 with this configuration was observed to put all tasks into test
tasks = assign_splits(
[_task(f"t{i}", f"task {i}") for i in range(5)],
val_fraction=0.10,
test_fraction=0.80,
seed=42,
)
splits = {t.split for t in tasks}
self.assertIn("val", splits, "val must not be empty even if it needs to borrow from test")
self.assertIn("train", splits, "train must not be empty even if it needs to borrow from test")
self.assertIn("test", splits, "test must still contain remaining tasks")


class Pass1ApproachCFractionBoundaries(unittest.TestCase):
"""Pass 1 / approach C: reject invalid fraction knobs early."""
Expand Down