From 32d6d95983c0faf09e4e42ba18009b91404066a6 Mon Sep 17 00:00:00 2001 From: wazzowsky Date: Tue, 24 Jun 2025 12:02:44 +0200 Subject: [PATCH] Update validator.py ## Problem Validators occasionally assign zero weights to high-performing miners due to a metadata scope bug in the score normalization process. This causes good datasets to randomly "fall off" the leaderboard despite maintaining their quality. ## Root Cause The validator code has a subtle but critical bug in `neurons/validator.py`: 1. **First loop (lines 385-421)**: Evaluates miners and assigns `metadata` variable for each UID - Can break early when `should_set_weights()` returns True - The `metadata` variable gets overwritten each iteration 2. **Second loop (lines 470-491)**: Normalizes scores for ALL UIDs in `uids_to_eval` - Uses `metadata.id.competition_id` for score computation - **BUG**: This `metadata` references only the LAST evaluated UID from the first loop ### Example scenario: - `uids_to_eval = [10, 20, 30, 40, 50]` - First loop breaks after UID 30 (due to approaching epoch boundary) - `metadata` now contains UID 30's data - Second loop processes ALL UIDs [10, 20, 30, 40, 50] - UIDs 10, 20, 40, 50 all use UID 30's metadata (wrong!) - This causes `compute_score()` to return 0 when competition IDs don't match ## Solution This PR implements a minimal fix: 1. Add `metadata_per_uid = {}` dictionary to store metadata for each UID 2. Store metadata for each UID: `metadata_per_uid[uid] = metadata` 3. Skip normalization for UIDs that weren't evaluated 4. Use correct metadata for each UID: `uid_metadata = metadata_per_uid[uid]` ## Changes - Only 10 lines added to `neurons/validator.py` - No breaking changes - Preserves existing logic while fixing the scope issue ## Testing To verify this fix: 1. Add debug logging to see which metadata is used for each UID 2. Monitor that unevaluated UIDs are properly skipped 3. Confirm no more zero scores for valid datasets ## Impact This fix ensures: - Miners with good datasets maintain consistent scores - No more random weight drops to zero - Fair evaluation across all validators --- neurons/validator.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/neurons/validator.py b/neurons/validator.py index fa3a25f..c27ff3e 100644 --- a/neurons/validator.py +++ b/neurons/validator.py @@ -247,6 +247,7 @@ async def run_step(self): raw_scores_this_epoch = {} block_per_uid = {} + metadata_per_uid = {} # Store metadata for each UID for uid in uids_to_eval: bt.logging.info(f"Evaluating UID: {uid}") bt.logging.info( @@ -255,6 +256,7 @@ async def run_step(self): metadata = retrieve_model_metadata( self.subtensor, self.config.netuid, self.metagraph.hotkeys[uid] ) + metadata_per_uid[uid] = metadata # Store metadata for this specific UID if self.should_set_weights(): bt.logging.info( @@ -399,6 +401,12 @@ async def run_step(self): normalized_scores_this_epoch = {} for uid in uids_to_eval: current_raw_score = raw_scores_this_epoch.get(uid) + + # Skip UIDs that were not evaluated (no metadata) + if uid not in metadata_per_uid: + bt.logging.debug(f"UID {uid} was not evaluated, skipping normalization") + continue + if current_raw_score is not None: bt.logging.debug( f"Computing normalized score for UID {uid} with raw score {current_raw_score}" @@ -409,6 +417,9 @@ async def run_step(self): ) normalized_score = constants.DEFAULT_SCORE else: + # Use the correct metadata for this UID + uid_metadata = metadata_per_uid[uid] + normalized_score = compute_score( current_raw_score, competition.bench, @@ -416,7 +427,7 @@ async def run_step(self): competition.maxb, competition.pow, competition.bheight, - metadata.id.competition_id, + uid_metadata.id.competition_id, # Use correct metadata competition.id, ) normalized_scores_this_epoch[uid] = normalized_score @@ -482,4 +493,4 @@ async def run(self): if __name__ == "__main__": - asyncio.run(Validator().run()) \ No newline at end of file + asyncio.run(Validator().run())