evaluation.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import argparse
import hashlib
import json
import os
from concurrent.futures import ThreadPoolExecutor, as_completed
from datetime import datetime
from typing import Any, Dict, List, Optional, Tuple

from reward_function import get_config_model_name, score_trajectory


MAX_WORKERS = 8
VERSION = None
PLATFORM_TO_DATASET_TAG = {
    "AndroidWorld": "AW",
    "WindowsAgentArena": "WAA",
    "OSWorld": "OSWorld",
    "WebArena": "WebArena",
    "MacOS": "MacOS",
}
DATASET_SORT_ORDER = {
    "OSWorld": 0,
    "AW": 1,
    "WAA": 2,
    "WebArena": 3,
    "MacOS": 4,
}


###############################################################################
# Utilities
###############################################################################
def now_iso() -> str:
    return datetime.now().isoformat(timespec="seconds")


def ensure_output_dir(path: str) -> None:
    os.makedirs(path, exist_ok=True)


def read_jsonl(path: str) -> List[Dict[str, Any]]:
    items: List[Dict[str, Any]] = []
    with open(path, "r", encoding="utf-8") as f:
        for line_no, line in enumerate(f, 1):
            line = line.strip()
            if not line:
                continue
            try:
                obj = json.loads(line)
            except json.JSONDecodeError as exc:
                raise ValueError(f"Invalid JSONL at {path}:{line_no}: {exc}")
            if isinstance(obj, dict):
                items.append(obj)
    return items


def load_benchmark_items(path: str) -> List[Dict[str, Any]]:
    lower_path = path.lower()
    if lower_path.endswith(".jsonl"):
        return read_jsonl(path)

    with open(path, "r", encoding="utf-8") as f:
        data = json.load(f)

    if isinstance(data, dict):
        samples = data.get("samples")
        if isinstance(samples, list):
            return [item for item in samples if isinstance(item, dict)]
        raise ValueError(f"Unsupported JSON schema in {path}: expected top-level 'samples' list")

    if isinstance(data, list):
        return [item for item in data if isinstance(item, dict)]

    raise ValueError(f"Unsupported benchmark file format: {path}")


def resolve_path(maybe_path: str, base_dir: str) -> str:
    if not isinstance(maybe_path, str) or not maybe_path:
        return ""
    if os.path.isabs(maybe_path):
        return maybe_path
    return os.path.abspath(os.path.join(base_dir, maybe_path))


def normalize_trajectory_dir(path: str) -> str:
    if not path:
        return ""
    if os.path.isdir(path):
        return path
    if os.path.isfile(path):
        return os.path.dirname(path)
    if os.path.basename(path).lower() == "trajectory.json":
        return os.path.dirname(path)
    return path


def dataset_sort_key(dataset_tag: str) -> Tuple[int, str]:
    return (DATASET_SORT_ORDER.get(dataset_tag, 999), dataset_tag)


def load_summary_bool(path: str) -> Dict[str, Any]:
    if os.path.exists(path):
        with open(path, "r", encoding="utf-8") as f:
            data = json.load(f)
            if isinstance(data, dict):
                if "tasks" not in data or not isinstance(data["tasks"], dict):
                    data["tasks"] = {}
                return data
    return {"tasks": {}}


def write_summary_bool(path: str, summary: Dict[str, Any]) -> None:
    summary["updated_at"] = now_iso()
    with open(path, "w", encoding="utf-8") as f:
        json.dump(summary, f, ensure_ascii=False, indent=2)


def update_summary_bool(path: str, summary: Dict[str, Any], key: str, value: bool) -> None:
    summary.setdefault("tasks", {})[key] = bool(value)
    write_summary_bool(path, summary)


def load_summary_full(path: str) -> Dict[str, Any]:
    if os.path.exists(path):
        with open(path, "r", encoding="utf-8") as f:
            data = json.load(f)
            if isinstance(data, dict):
                if "tasks" not in data or not isinstance(data["tasks"], dict):
                    data["tasks"] = {}
                return data
    return {"tasks": {}}


def write_summary_full(path: str, summary: Dict[str, Any]) -> None:
    summary["updated_at"] = now_iso()
    with open(path, "w", encoding="utf-8") as f:
        json.dump(summary, f, ensure_ascii=False, indent=2)


def update_summary_full(path: str, summary: Dict[str, Any], key: str, record: Dict[str, Any]) -> None:
    summary.setdefault("tasks", {})[key] = record
    write_summary_full(path, summary)


def compute_metrics(tp: int, fp: int, tn: int, fn: int) -> Dict[str, float]:
    total = tp + fp + tn + fn
    accuracy = (tp + tn) / total if total else 0.0
    precision = tp / (tp + fp) if (tp + fp) else 0.0
    recall = tp / (tp + fn) if (tp + fn) else 0.0
    f1 = (2 * precision * recall / (precision + recall)) if (precision + recall) else 0.0
    return {
        "accuracy": accuracy,
        "precision": precision,
        "recall": recall,
        "f1": f1,
    }


def accumulate_confusion_counts(
    tp: int,
    fp: int,
    tn: int,
    fn: int,
    *,
    gt_success: bool,
    pred_success: bool,
) -> Tuple[int, int, int, int]:
    if pred_success and gt_success:
        tp += 1
    elif pred_success and (not gt_success):
        fp += 1
    elif (not pred_success) and (not gt_success):
        tn += 1
    else:
        fn += 1
    return tp, fp, tn, fn


def build_task_output_dir(
    *,
    run_output_dir: str,
    dataset_tag: str,
    record_key: str,
    entry_meta: Dict[str, Any],
) -> str:
    task_slug = os.path.splitext(
        _task_detail_filename(dataset_tag, record_key, entry_meta)
    )[0]
    return os.path.join(run_output_dir, "task_logs", dataset_tag, task_slug)


def build_task_record(
    *,
    traj_display: str,
    gt_success: bool,
    pred_success: bool,
    matched: bool,
    dataset_tag: str,
    entry_meta: Dict[str, Any],
    result: Dict[str, Any],
    eval_error: Optional[Exception] = None,
) -> Dict[str, Any]:
    full_record = {
        "trajectory_path": traj_display,
        "gt_success": gt_success,
        "pred_success": pred_success,
        "matched": matched,
        "final_decision": result.get("final_decision"),
        "final_raw": result.get("final_raw"),
        "final_output": result.get("final_output"),
        "log_path": result.get("log_path"),
        "compact_log_path": result.get("compact_log_path"),
        "updated_at": now_iso(),
        "mode": "completion",
        "dataset": dataset_tag,
        "sample_id": entry_meta.get("sample_id"),
        "platform": entry_meta.get("platform"),
        "task_id": entry_meta.get("task_id"),
        "application": entry_meta.get("application"),
    }
    if eval_error is not None:
        full_record["error"] = {
            "type": type(eval_error).__name__,
            "message": str(eval_error),
        }
    return full_record


def _counts_from_records(records: List[Dict[str, Any]]) -> Dict[str, int]:
    tp = fp = tn = fn = 0
    for rec in records:
        gt = bool(rec.get("gt_success", False))
        pred = bool(rec.get("pred_success", False))
        if pred and gt:
            tp += 1
        elif pred and (not gt):
            fp += 1
        elif (not pred) and (not gt):
            tn += 1
        else:
            fn += 1
    return {"tp": tp, "fp": fp, "tn": tn, "fn": fn, "total": tp + fp + tn + fn}


def _flatten_summary_records(summary_full_path: str, default_dataset: str) -> List[Dict[str, Any]]:
    if not os.path.exists(summary_full_path):
        return []
    data = load_summary_full(summary_full_path)
    tasks = data.get("tasks", {})
    if not isinstance(tasks, dict):
        return []

    records: List[Dict[str, Any]] = []
    for key, rec in tasks.items():
        if not isinstance(rec, dict):
            continue
        item = dict(rec)
        item["record_key"] = key
        item["dataset"] = item.get("dataset") or default_dataset
        item["gt_success"] = bool(item.get("gt_success", False))
        item["pred_success"] = bool(item.get("pred_success", False))
        item["matched"] = bool(item.get("matched", item["gt_success"] == item["pred_success"]))
        records.append(item)
    return records


def _build_group_breakdown(records: List[Dict[str, Any]], field: str) -> Dict[str, Dict[str, Any]]:
    grouped: Dict[str, List[Dict[str, Any]]] = {}
    for rec in records:
        raw_key = rec.get(field)
        key = str(raw_key).strip() if raw_key is not None else ""
        key = key or "UNKNOWN"
        grouped.setdefault(key, []).append(rec)

    breakdown: Dict[str, Dict[str, Any]] = {}
    for key in sorted(grouped.keys()):
        rows = grouped[key]
        counts = _counts_from_records(rows)
        matched_count = sum(1 for row in rows if bool(row.get("matched", False)))
        completed_count = sum(1 for row in rows if str(row.get("final_decision", "")).strip() == "completed")
        breakdown[key] = {
            "counts": counts,
            "metrics": compute_metrics(counts["tp"], counts["fp"], counts["tn"], counts["fn"]),
            "match_rate": (matched_count / counts["total"]) if counts["total"] else 0.0,
            "completed_rate": (completed_count / counts["total"]) if counts["total"] else 0.0,
        }
    return breakdown


def _safe_token(value: Any, default: str = "UNKNOWN", max_len: int = 40) -> str:
    text = str(value).strip() if value is not None else ""
    if not text:
        text = default
    cleaned = "".join(ch if ch.isalnum() or ch in ("-", "_") else "_" for ch in text)
    cleaned = cleaned.strip("_") or default
    return cleaned[:max_len]


def _task_detail_filename(dataset_tag: str, record_key: str, record: Dict[str, Any]) -> str:
    digest = hashlib.sha1(str(record_key).encode("utf-8")).hexdigest()[:12]
    sample_id = _safe_token(record.get("sample_id"), default="sample")
    task_id = _safe_token(record.get("task_id"), default="task")
    platform = _safe_token(record.get("platform"), default="platform")
    dataset = _safe_token(dataset_tag, default="dataset")
    return f"{dataset}_{platform}_{sample_id}_{task_id}_{digest}.json"


def write_task_detail_record(
    *,
    run_output_dir: str,
    dataset_tag: str,
    record_key: str,
    record: Dict[str, Any],
) -> str:
    detailed_dir = os.path.join(run_output_dir, "detailed", dataset_tag)
    ensure_output_dir(detailed_dir)

    payload = dict(record)
    payload["record_key"] = record_key
    payload["dataset"] = payload.get("dataset") or dataset_tag
    payload["written_at"] = now_iso()

    out_path = os.path.join(detailed_dir, _task_detail_filename(dataset_tag, record_key, payload))
    with open(out_path, "w", encoding="utf-8") as f:
        json.dump(payload, f, ensure_ascii=False, indent=2)
    return out_path


def sync_task_detail_records(run_output_dir: str, dataset_tag: str, summary_full_path: str) -> int:
    if not os.path.exists(summary_full_path):
        return 0
    summary = load_summary_full(summary_full_path)
    tasks = summary.get("tasks", {})
    if not isinstance(tasks, dict):
        return 0

    written = 0
    for record_key, record in tasks.items():
        if not isinstance(record, dict):
            continue
        write_task_detail_record(
            run_output_dir=run_output_dir,
            dataset_tag=dataset_tag,
            record_key=record_key,
            record=record,
        )
        written += 1
    return written


def discover_dataset_output_map(run_output_dir: str) -> Dict[str, str]:
    dataset_output_map: Dict[str, str] = {}
    if not os.path.isdir(run_output_dir):
        return dataset_output_map

    for name in os.listdir(run_output_dir):
        out_dir = os.path.join(run_output_dir, name)
        if not os.path.isdir(out_dir):
            continue
        if os.path.exists(os.path.join(out_dir, "summary_full.json")):
            dataset_output_map[name] = out_dir
    return dataset_output_map


def write_live_detailed_metrics(
    *,
    run_output_dir: str,
    model_name: str,
    run_name: str,
    benchmark_source: str,
    config_path: str,
) -> str:
    ensure_output_dir(run_output_dir)

    dataset_output_map = discover_dataset_output_map(run_output_dir)
    all_records: List[Dict[str, Any]] = []
    dataset_breakdown: Dict[str, Dict[str, Any]] = {}
    for dataset_tag in sorted(dataset_output_map.keys(), key=dataset_sort_key):
        out_dir = dataset_output_map[dataset_tag]
        summary_full_path = os.path.join(out_dir, "summary_full.json")
        records = _flatten_summary_records(summary_full_path, dataset_tag)
        all_records.extend(records)

        counts = _counts_from_records(records)
        matched_count = sum(1 for row in records if bool(row.get("matched", False)))
        dataset_breakdown[dataset_tag] = {
            "counts": counts,
            "metrics": compute_metrics(counts["tp"], counts["fp"], counts["tn"], counts["fn"]),
            "match_rate": (matched_count / counts["total"]) if counts["total"] else 0.0,
        }

    all_counts = _counts_from_records(all_records)
    matched_total = sum(1 for row in all_records if bool(row.get("matched", False)))
    completed_total = sum(1 for row in all_records if str(row.get("final_decision", "")).strip() == "completed")
    live_payload = {
        "updated_at": now_iso(),
        "agent_name": model_name,
        "model_name": model_name,
        "run_name": run_name,
        "version": VERSION,
        "benchmark_source": benchmark_source,
        "config_path": config_path,
        "counts": all_counts,
        "metrics": compute_metrics(all_counts["tp"], all_counts["fp"], all_counts["tn"], all_counts["fn"]),
        "match_rate": (matched_total / all_counts["total"]) if all_counts["total"] else 0.0,
        "completed_rate": (completed_total / all_counts["total"]) if all_counts["total"] else 0.0,
        "datasets": dataset_breakdown,
        "breakdown_by_platform": _build_group_breakdown(all_records, "platform"),
    }

    live_path = os.path.join(run_output_dir, "detailed_live_metrics.json")
    with open(live_path, "w", encoding="utf-8") as f:
        json.dump(live_payload, f, ensure_ascii=False, indent=2)
    return live_path


def write_detailed_report(
    *,
    run_output_dir: str,
    dataset_output_map: Dict[str, str],
    model_name: str,
    run_name: str,
    benchmark_source: str,
    config_path: str,
) -> None:
    ensure_output_dir(run_output_dir)

    all_records: List[Dict[str, Any]] = []
    dataset_breakdown: Dict[str, Dict[str, Any]] = {}
    for dataset_tag in sorted(dataset_output_map.keys(), key=dataset_sort_key):
        out_dir = dataset_output_map[dataset_tag]
        summary_full_path = os.path.join(out_dir, "summary_full.json")
        records = _flatten_summary_records(summary_full_path, dataset_tag)
        all_records.extend(records)

        counts = _counts_from_records(records)
        dataset_breakdown[dataset_tag] = {
            "output_dir": out_dir,
            "summary_full_path": summary_full_path,
            "counts": counts,
            "metrics": compute_metrics(counts["tp"], counts["fp"], counts["tn"], counts["fn"]),
            "match_rate": (
                sum(1 for row in records if bool(row.get("matched", False))) / counts["total"]
                if counts["total"] else 0.0
            ),
        }

    all_records.sort(key=lambda x: (str(x.get("dataset", "")), str(x.get("record_key", ""))))
    all_counts = _counts_from_records(all_records)
    decision_counts: Dict[str, int] = {}
    for rec in all_records:
        decision = str(rec.get("final_decision", "")).strip() or "UNKNOWN"
        decision_counts[decision] = decision_counts.get(decision, 0) + 1

    mismatch_records = [
        {
            "dataset": rec.get("dataset"),
            "sample_id": rec.get("sample_id"),
            "task_id": rec.get("task_id"),
            "application": rec.get("application"),
            "trajectory_path": rec.get("trajectory_path"),
            "gt_success": bool(rec.get("gt_success", False)),
            "pred_success": bool(rec.get("pred_success", False)),
            "final_decision": rec.get("final_decision"),
            "record_key": rec.get("record_key"),
        }
        for rec in all_records
        if not bool(rec.get("matched", False))
    ]

    detailed_report = {
        "updated_at": now_iso(),
        "agent_name": model_name,
        "model_name": model_name,
        "run_name": run_name,
        "version": VERSION,
        "benchmark_source": benchmark_source,
        "config_path": config_path,
        "counts": all_counts,
        "metrics": compute_metrics(all_counts["tp"], all_counts["fp"], all_counts["tn"], all_counts["fn"]),
        "match_rate": (
            sum(1 for row in all_records if bool(row.get("matched", False))) / all_counts["total"]
            if all_counts["total"] else 0.0
        ),
        "decision_counts": decision_counts,
        "datasets": dataset_breakdown,
        "breakdown_by_platform": _build_group_breakdown(all_records, "platform"),
        "breakdown_by_application": _build_group_breakdown(all_records, "application"),
        "breakdown_by_task_id": _build_group_breakdown(all_records, "task_id"),
        "mismatch_records": mismatch_records,
    }

    report_json_path = os.path.join(run_output_dir, "detailed_report.json")
    with open(report_json_path, "w", encoding="utf-8") as f:
        json.dump(detailed_report, f, ensure_ascii=False, indent=2)

    report_jsonl_path = os.path.join(run_output_dir, "detailed_records.jsonl")
    with open(report_jsonl_path, "w", encoding="utf-8") as f:
        for rec in all_records:
            f.write(json.dumps(rec, ensure_ascii=False) + "\n")

    print(f"[ALL] Detailed report saved to: {report_json_path}")
    print(f"[ALL] Detailed records saved to: {report_jsonl_path}")


###############################################################################
# Benchmark runner
###############################################################################
def run_one_benchmark(
    *,
    selected_trajs_path: str,
    output_dir: str,
    dataset_tag: str,
    model_name: str,
    run_name: str,
    config_path: str,
    run_output_dir: str,
    benchmark_source: str,
    selected_items: Optional[List[Dict[str, Any]]] = None,
    selected_base_dir: Optional[str] = None,
) -> None:
    selected_path = os.path.abspath(selected_trajs_path)
    selected_dir = os.path.abspath(selected_base_dir) if selected_base_dir else os.path.dirname(selected_path)

    ensure_output_dir(output_dir)

    summary_bool_file = os.path.join(output_dir, "summary.json")
    summary_full_file = os.path.join(output_dir, "summary_full.json")
    metrics_file = os.path.join(output_dir, "metrics.json")
    synced = sync_task_detail_records(run_output_dir, dataset_tag, summary_full_file)
    if synced:
        print(f"[{dataset_tag}] Synced {synced} task detail files to {os.path.join(run_output_dir, 'detailed', dataset_tag)}")
    write_live_detailed_metrics(
        run_output_dir=run_output_dir,
        model_name=model_name,
        run_name=run_name,
        benchmark_source=benchmark_source,
        config_path=config_path,
    )

    if selected_items is None:
        selected_items = load_benchmark_items(selected_path)
    if not selected_items:
        print(f"[{dataset_tag}] No entries found in {selected_path}")
        return

    summary_bool = load_summary_bool(summary_bool_file)
    summary_full = load_summary_full(summary_full_file)
    processed_keys = set(summary_full.get("tasks", {}).keys())

    jobs: List[Tuple[str, str, bool, str, Dict[str, Any]]] = []
    for entry in selected_items:
        traj_path_raw = entry.get("trajectory_path") or entry.get("path") or ""
        gt_success = bool(entry.get("success", False))

        if not isinstance(traj_path_raw, str) or not traj_path_raw.strip():
            continue
        traj_path_raw = traj_path_raw.strip()

        traj_abs_raw = resolve_path(traj_path_raw, selected_dir)
        traj_abs = normalize_trajectory_dir(traj_abs_raw)
        if not traj_abs:
            continue

        key_abs = traj_abs
        if key_abs in processed_keys or traj_abs_raw in processed_keys:
            continue

        traj_display = traj_path_raw if not os.path.isabs(traj_path_raw) else traj_abs_raw
        entry_meta = {
            "sample_id": entry.get("sample_id"),
            "platform": entry.get("platform"),
            "task_id": entry.get("task_id"),
            "application": entry.get("application"),
        }
        jobs.append((traj_abs, traj_display, gt_success, key_abs, entry_meta))

    skipped = len(selected_items) - len(jobs)
    print(f"[{dataset_tag}] Selected entries: {len(selected_items)} | Skipped: {skipped} | To run: {len(jobs)}")
    print(f"[{dataset_tag}] Output dir: {output_dir}")
    print(
        f"[{dataset_tag}] Model: {model_name} | RunName: {run_name} | "
        f"Mode: completion | MaxWorkers: {MAX_WORKERS}"
    )
    print(f"[{dataset_tag}] Config: {config_path}")

    tp = fp = tn = fn = 0
    for rec in summary_full.get("tasks", {}).values():
        if not isinstance(rec, dict):
            continue
        gt = bool(rec.get("gt_success", False))
        pred = bool(rec.get("pred_success", False))
        if pred and gt:
            tp += 1
        elif pred and not gt:
            fp += 1
        elif (not pred) and (not gt):
            tn += 1
        else:
            fn += 1

    total_prev = tp + fp + tn + fn
    if total_prev:
        m0 = compute_metrics(tp, fp, tn, fn)
        print(f"[{dataset_tag}] Resumed: total={total_prev} acc={m0['accuracy']:.4f}")

    if not jobs:
        metrics = compute_metrics(tp, fp, tn, fn)
        payload = {
            "updated_at": now_iso(),
            "dataset": dataset_tag,
            "mode": "completion",
            "agent_name": model_name,
            "model_name": model_name,
            "run_name": run_name,
            "counts": {"tp": tp, "fp": fp, "tn": tn, "fn": fn, "total": tp + fp + tn + fn},
            "metrics": metrics,
        }
        with open(metrics_file, "w", encoding="utf-8") as f:
            json.dump(payload, f, ensure_ascii=False, indent=2)
        write_live_detailed_metrics(
            run_output_dir=run_output_dir,
            model_name=model_name,
            run_name=run_name,
            benchmark_source=benchmark_source,
            config_path=config_path,
        )
        print(f"[{dataset_tag}] No new jobs. Metrics refreshed at {metrics_file}")
        return

    with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
        future_to_meta = {}
        for traj_abs, traj_display, gt_success, key_abs, entry_meta in jobs:
            task_output_dir = build_task_output_dir(
                run_output_dir=run_output_dir,
                dataset_tag=dataset_tag,
                record_key=key_abs,
                entry_meta=entry_meta,
            )
            future = executor.submit(
                score_trajectory,
                traj_dir=traj_abs,
                agent_name=None,
                config_path=config_path,
                output_dir=task_output_dir,
            )
            future_to_meta[future] = (key_abs, traj_abs, traj_display, gt_success, entry_meta)

        for future in as_completed(future_to_meta):
            key_abs, traj_abs, traj_display, gt_success, entry_meta = future_to_meta[future]
            gt_success = bool(gt_success)
            try:
                result = future.result()
                pred_success = bool(result.get("reward", 0) == 1)
                matched = (pred_success == gt_success)
                tp, fp, tn, fn = accumulate_confusion_counts(
                    tp,
                    fp,
                    tn,
                    fn,
                    gt_success=gt_success,
                    pred_success=pred_success,
                )
                update_summary_bool(summary_bool_file, summary_bool, key_abs, matched)
                full_record = build_task_record(
                    traj_display=traj_display,
                    gt_success=gt_success,
                    pred_success=pred_success,
                    matched=matched,
                    dataset_tag=dataset_tag,
                    entry_meta=entry_meta,
                    result=result,
                )
                update_summary_full(summary_full_file, summary_full, key_abs, full_record)
                write_task_detail_record(
                    run_output_dir=run_output_dir,
                    dataset_tag=dataset_tag,
                    record_key=key_abs,
                    record=full_record,
                )

            except Exception as exc:
                # Count infrastructure/evaluator failures conservatively as mismatches
                # so they cannot disappear from the benchmark denominator.
                pred_success = not gt_success
                matched = False
                tp, fp, tn, fn = accumulate_confusion_counts(
                    tp,
                    fp,
                    tn,
                    fn,
                    gt_success=gt_success,
                    pred_success=pred_success,
                )
                failed_result = {
                    "final_decision": "error",
                    "final_raw": "",
                    "final_output": {
                        "error_type": type(exc).__name__,
                        "error_message": str(exc),
                    },
                    "log_path": "",
                    "compact_log_path": "",
                }
                update_summary_bool(summary_bool_file, summary_bool, key_abs, matched)
                full_record = build_task_record(
                    traj_display=traj_display,
                    gt_success=gt_success,
                    pred_success=pred_success,
                    matched=matched,
                    dataset_tag=dataset_tag,
                    entry_meta=entry_meta,
                    result=failed_result,
                    eval_error=exc,
                )
                update_summary_full(summary_full_file, summary_full, key_abs, full_record)
                write_task_detail_record(
                    run_output_dir=run_output_dir,
                    dataset_tag=dataset_tag,
                    record_key=key_abs,
                    record=full_record,
                )
                print(f"[{dataset_tag}][ERROR] {traj_abs} failed: {exc}")

            metrics = compute_metrics(tp, fp, tn, fn)
            total = tp + fp + tn + fn
            print(
                f"[{dataset_tag}][Progress] {total} processed | "
                f"Acc={metrics['accuracy']:.4f} P={metrics['precision']:.4f} "
                f"R={metrics['recall']:.4f} F1={metrics['f1']:.4f} | "
                f"TP={tp} FP={fp} TN={tn} FN={fn}"
            )

            payload = {
                "updated_at": now_iso(),
                "dataset": dataset_tag,
                "mode": "completion",
                "agent_name": model_name,
                "model_name": model_name,
                "run_name": run_name,
                "counts": {"tp": tp, "fp": fp, "tn": tn, "fn": fn, "total": total},
                "metrics": metrics,
            }
            with open(metrics_file, "w", encoding="utf-8") as f:
                json.dump(payload, f, ensure_ascii=False, indent=2)
            write_live_detailed_metrics(
                run_output_dir=run_output_dir,
                model_name=model_name,
                run_name=run_name,
                benchmark_source=benchmark_source,
                config_path=config_path,
            )

    print(f"[{dataset_tag}] Done. Metrics saved to: {metrics_file}")
    print(f"[{dataset_tag}] Summaries saved to: {summary_bool_file} and {summary_full_file}")


###############################################################################
# Main
###############################################################################
def main() -> None:
    parser = argparse.ArgumentParser(description="Run fixed benchmark evaluation")
    parser.add_argument(
        "--output-dir",
        default="",
        help="Output directory for evaluation artifacts. Defaults to ./benchmark_result/<model_name>/",
    )
    parser.add_argument(
        "--benchmark-file",
        required=True,
        help="Single benchmark file (.jsonl or .json). Supports OmniGUIRewardBench.json.",
    )
    parser.add_argument("--result-root", default="benchmark_result", help="Root dir for outputs")
    parser.add_argument(
        "--config",
        default="config.yaml",
        help="Path to config yaml. Defaults to ./config.yaml (current working directory).",
    )
    args = parser.parse_args()

    result_root = os.path.abspath(args.result_root)
    benchmark_file = str(args.benchmark_file).strip()
    config_path = os.path.abspath(str(args.config).strip() or "config.yaml")
    if not os.path.exists(config_path):
        raise FileNotFoundError(f"config file not found: {config_path}")
    model_name = get_config_model_name(config_path)
    raw_output_dir = str(args.output_dir).strip()
    if raw_output_dir:
        run_output_dir = os.path.abspath(raw_output_dir)
    elif VERSION:
        run_output_dir = os.path.join(result_root, f"{model_name}_v{VERSION}")
    else:
        run_output_dir = os.path.join(result_root, model_name)
    run_name = os.path.basename(run_output_dir.rstrip(os.sep)) or model_name
    dataset_output_map: Dict[str, str] = {}

    selected_path = os.path.abspath(benchmark_file)
    if not os.path.exists(selected_path):
        raise FileNotFoundError(f"benchmark file not found: {selected_path}")

    selected_items = load_benchmark_items(selected_path)
    if not selected_items:
        print(f"[ALL] No entries found in {selected_path}")
        return

    selected_dir = os.path.dirname(selected_path)
    fallback_dataset = os.path.splitext(os.path.basename(selected_path))[0]
    grouped: Dict[str, List[Dict[str, Any]]] = {}
    for entry in selected_items:
        platform_raw = entry.get("platform")
        platform = platform_raw.strip() if isinstance(platform_raw, str) else ""
        dataset_tag = PLATFORM_TO_DATASET_TAG.get(platform, platform or fallback_dataset)
        grouped.setdefault(dataset_tag, []).append(entry)

    for dataset_tag in sorted(grouped.keys(), key=dataset_sort_key):
        output_dir = os.path.join(run_output_dir, dataset_tag)
        run_one_benchmark(
            selected_trajs_path=selected_path,
            output_dir=output_dir,
            dataset_tag=dataset_tag,
            model_name=model_name,
            run_name=run_name,
            config_path=config_path,
            run_output_dir=run_output_dir,
            benchmark_source=selected_path,
            selected_items=grouped[dataset_tag],
            selected_base_dir=selected_dir,
        )
        dataset_output_map[dataset_tag] = output_dir

    write_detailed_report(
        run_output_dir=run_output_dir,
        dataset_output_map=dataset_output_map,
        model_name=model_name,
        run_name=run_name,
        benchmark_source=selected_path,
        config_path=config_path,
    )


if __name__ == "__main__":
    main()