agentopt/examples/selection/daemon/basic.py at main · AgentOptimizer/agentopt · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
"""
Example: same selection script, local proxy vs. ``agentopt serve`` daemon.

The user-facing API does not change between local and daemon modes; the
``AGENTOPT_GATEWAY_URL`` env var is the entire switch.

Local mode (default — spins up an in-process mitmproxy per session)::

    python examples/selection/daemon/basic.py

Daemon mode (one long-lived gateway shared by every script and language)::

    # Terminal 1 — start the daemon (Ctrl-C to stop).
    agentopt serve

    # Terminal 2 — same script, just point at the daemon.
    AGENTOPT_GATEWAY_URL=http://127.0.0.1:9000 \\
        python examples/selection/daemon/basic.py

The selector script (below) is byte-identical between the two runs.

Prerequisites:
    1. ``pip install agentopt-py openai``
    2. ``export OPENAI_API_KEY=...``
"""

from __future__ import annotations

import os

from dotenv import load_dotenv

load_dotenv()

from openai import OpenAI

from agentopt import ModelSelector


# ---------------------------------------------------------------------------
# Step 1: Define your agent class.
# ---------------------------------------------------------------------------


class MyAgent:
    """One-shot OpenAI chat completion."""

    def __init__(self, models):
        self.client = OpenAI()
        self.model = models["agent"]

    def run(self, input_data: str) -> str:
        resp = self.client.chat.completions.create(
            model=self.model,
            messages=[
                {"role": "system", "content": "Answer in one word."},
                {"role": "user", "content": input_data},
            ],
        )
        return resp.choices[0].message.content or ""


# ---------------------------------------------------------------------------
# Step 2: Dataset + scoring.
# ---------------------------------------------------------------------------


dataset = [
    ("What is the capital of France?", "Paris"),
    ("What is 2 + 2?", "4"),
    ("What color is the sky on a clear day?", "blue"),
]


def eval_fn(expected: str, actual: str) -> float:
    return 1.0 if expected.lower() in str(actual).lower() else 0.0


# ---------------------------------------------------------------------------
# Step 3: Run.  Nothing here knows about local vs. daemon mode.
# ---------------------------------------------------------------------------


if __name__ == "__main__":
    gateway = os.environ.get("AGENTOPT_GATEWAY_URL")
    print(f"Mode: {'remote → ' + gateway if gateway else 'local (in-process proxy)'}")

    selector = ModelSelector(
        agent=MyAgent,
        models={"agent": ["gpt-4o-mini", "gpt-4.1-nano"]},
        eval_fn=eval_fn,
        dataset=dataset,
        method="brute_force",
    )
    results = selector.select_best(parallel=False)
    results.print_summary()

    best = results.get_best_combo()
    if best:
        print(f"\nBest model: {best}")