|
| 1 | +"""Azure provider-usage accounting regressions: single-owner charging. |
| 2 | +
|
| 3 | +A backend that reports provider usage (AzureOpenAI / AzureResponses) must be |
| 4 | +charged exactly once in ``_cached_call`` — with the provider's own token count, |
| 5 | +never double-charged by the ``len//4`` length estimate, and never charged on a |
| 6 | +cache hit. (The maintainer's reproduction: a 30-token provider usage was being |
| 7 | +recorded as a 110-token length estimate because ``_call`` recorded usage and |
| 8 | +``_cached_call`` then recorded the length estimate on top.) |
| 9 | +
|
| 10 | +Also covers the OpenCode error path routing through ``_record_delta``. |
| 11 | +""" |
| 12 | +from __future__ import annotations |
| 13 | + |
| 14 | +from types import SimpleNamespace |
| 15 | +from unittest import mock |
| 16 | + |
| 17 | +from skillopt_sleep.backend import AzureOpenAIBackend, AzureResponsesBackend, OpenCodeCliBackend |
| 18 | + |
| 19 | + |
| 20 | +class _ChatResp: |
| 21 | + def __init__(self, text, prompt_tokens, completion_tokens): |
| 22 | + self.choices = [SimpleNamespace(message=SimpleNamespace(content=text))] |
| 23 | + self.usage = SimpleNamespace(prompt_tokens=prompt_tokens, completion_tokens=completion_tokens) |
| 24 | + |
| 25 | + |
| 26 | +class _FakeChatClient: |
| 27 | + """Scripted chat.completions.create returning a _ChatResp or raising.""" |
| 28 | + |
| 29 | + def __init__(self, replies): |
| 30 | + self.replies = list(replies) |
| 31 | + self.calls = [] |
| 32 | + |
| 33 | + def create(self, **kwargs): |
| 34 | + self.calls.append(kwargs) |
| 35 | + item = self.replies.pop(0) |
| 36 | + if isinstance(item, Exception): |
| 37 | + raise item |
| 38 | + return item |
| 39 | + |
| 40 | + |
| 41 | +class _ResponsesResp: |
| 42 | + def __init__(self, text, input_tokens, output_tokens): |
| 43 | + self.output_text = text |
| 44 | + self.usage = SimpleNamespace(input_tokens=input_tokens, output_tokens=output_tokens) |
| 45 | + |
| 46 | + |
| 47 | +class _FakeResponsesClient: |
| 48 | + def __init__(self, replies): |
| 49 | + self.replies = list(replies) |
| 50 | + self.calls = [] |
| 51 | + |
| 52 | + def create(self, **kwargs): |
| 53 | + self.calls.append(kwargs) |
| 54 | + item = self.replies.pop(0) |
| 55 | + if isinstance(item, Exception): |
| 56 | + raise item |
| 57 | + return item |
| 58 | + |
| 59 | + |
| 60 | +def _azure_chat(replies): |
| 61 | + be = AzureOpenAIBackend(deployment="gpt-5.5") |
| 62 | + be._client = SimpleNamespace(chat=SimpleNamespace(completions=_FakeChatClient(replies))) |
| 63 | + return be |
| 64 | + |
| 65 | + |
| 66 | +def _azure_responses(replies): |
| 67 | + be = AzureResponsesBackend(deployment="gpt-5.5", endpoints=["https://t.openai.azure.com/"]) |
| 68 | + fake = SimpleNamespace(responses=_FakeResponsesClient(replies)) |
| 69 | + be._next_endpoint = lambda: be.endpoints[0] |
| 70 | + be._client_for = lambda ep: fake |
| 71 | + return be |
| 72 | + |
| 73 | + |
| 74 | +def test_azure_chat_single_call_charges_exact_usage(): |
| 75 | + """A single Azure chat call must charge the provider usage, not len//4.""" |
| 76 | + be = _azure_chat([_ChatResp("ok", 10, 20)]) |
| 77 | + with mock.patch("time.sleep"): |
| 78 | + out = be._cached_call("k:1", "x" * 400) |
| 79 | + assert out == "ok" |
| 80 | + # provider usage = 30; len//4 of 400+2 would be ~100 — must NOT be that. |
| 81 | + assert be._tokens == 30, f"expected exact provider usage 30, got {be._tokens}" |
| 82 | + assert be.token_delta() == 30 |
| 83 | + |
| 84 | + |
| 85 | +def test_azure_chat_empty_retry_then_success_accumulates(): |
| 86 | + """Empty-response retry + success must accumulate usage across paid attempts.""" |
| 87 | + be = _azure_chat([_ChatResp("", 7, 0), _ChatResp("ok", 10, 20)]) |
| 88 | + with mock.patch("time.sleep"): |
| 89 | + out = be._cached_call("k:1", "hello") |
| 90 | + assert out == "ok" |
| 91 | + assert be._tokens == 7 + 30, f"expected accumulated 37, got {be._tokens}" |
| 92 | + assert be.token_delta() == 37 |
| 93 | + |
| 94 | + |
| 95 | +def test_azure_chat_cache_hit_does_not_charge(): |
| 96 | + """A cache hit must reset the call-local delta and leave the aggregate alone.""" |
| 97 | + be = _azure_chat([_ChatResp("ok", 10, 20)]) |
| 98 | + with mock.patch("time.sleep"): |
| 99 | + be._cached_call("k:1", "hello") |
| 100 | + assert be._tokens == 30 |
| 101 | + assert be.token_delta() == 30 |
| 102 | + with mock.patch("time.sleep"): |
| 103 | + out2 = be._cached_call("k:1", "hello") |
| 104 | + assert out2 == "ok" |
| 105 | + assert be._tokens == 30, "cache hit changed the aggregate" |
| 106 | + assert be.token_delta() == 0, "cache hit leaked the prior delta" |
| 107 | + |
| 108 | + |
| 109 | +def test_azure_responses_single_call_charges_exact_usage(): |
| 110 | + be = _azure_responses([_ResponsesResp("ok", 12, 18)]) |
| 111 | + with mock.patch("time.sleep"): |
| 112 | + out = be._cached_call("k:1", "x" * 400) |
| 113 | + assert out == "ok" |
| 114 | + assert be._tokens == 30, f"expected exact provider usage 30, got {be._tokens}" |
| 115 | + assert be.token_delta() == 30 |
| 116 | + |
| 117 | + |
| 118 | +def test_azure_responses_empty_retry_then_success_accumulates(): |
| 119 | + be = _azure_responses([_ResponsesResp("", 5, 0), _ResponsesResp("ok", 12, 18)]) |
| 120 | + with mock.patch("time.sleep"): |
| 121 | + out = be._cached_call("k:1", "hello") |
| 122 | + assert out == "ok" |
| 123 | + assert be._tokens == 5 + 30, f"expected accumulated 35, got {be._tokens}" |
| 124 | + assert be.token_delta() == 35 |
| 125 | + |
| 126 | + |
| 127 | +def test_azure_responses_cache_hit_does_not_charge(): |
| 128 | + be = _azure_responses([_ResponsesResp("ok", 12, 18)]) |
| 129 | + with mock.patch("time.sleep"): |
| 130 | + be._cached_call("k:1", "hello") |
| 131 | + assert be._tokens == 30 |
| 132 | + with mock.patch("time.sleep"): |
| 133 | + be._cached_call("k:1", "hello") |
| 134 | + assert be._tokens == 30 |
| 135 | + assert be.token_delta() == 0 |
| 136 | + |
| 137 | + |
| 138 | +def test_opencode_error_path_uses_record_delta(monkeypatch): |
| 139 | + """The OpenCode error path must route prompt-only cost through _record_delta.""" |
| 140 | + import contextlib |
| 141 | + from types import SimpleNamespace |
| 142 | + |
| 143 | + import skillopt_sleep.backend as bm |
| 144 | + from skillopt_sleep.backend import OpenCodeCliBackend |
| 145 | + |
| 146 | + b = OpenCodeCliBackend(model="", opencode_path="opencode", tool_replay=True) |
| 147 | + monkeypatch.setattr(bm, "_opencode_temporary_workspace", lambda *a, **k: contextlib.nullcontext()) |
| 148 | + |
| 149 | + def _fail(*args, **kwargs): |
| 150 | + raise bm.OpenCodeError("boom", prompt_chars=100) |
| 151 | + |
| 152 | + monkeypatch.setattr(bm, "_prepare_opencode_replay_project", _fail) |
| 153 | + |
| 154 | + task = SimpleNamespace(intent="intent", context_excerpt="ctx") |
| 155 | + out, called = b.attempt_with_tools(task, skill="s", memory="m", tools=["search"]) |
| 156 | + |
| 157 | + assert out == "" and called == [] |
| 158 | + assert b.token_delta() == 100 // 4, "OpenCode error path did not route through _record_delta" |
| 159 | + assert b._tokens == 100 // 4, f"expected 25, got {b._tokens}" |
0 commit comments