Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 26 additions & 3 deletions internal/ai/service/builtin_ai_account.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,12 @@ const (
builtinAIRefreshTokenKey = "refresh-token"
builtinAITokenTypeKey = "token-type"
builtinAITokenExpiresKey = "expires-at"
// builtinAIRefreshPendingKey marks an access token whose last refresh ended
// without a usable answer. The token may still authenticate for a while, but
// the rotated refresh token was never received, so the next call must retry
// the refresh instead of waiting for the access token to expire (which could
// be hours, far past the Gateway's replay grace window).
builtinAIRefreshPendingKey = "refresh-pending"

// builtinAIRefreshSkew refreshes slightly early so a request never leaves
// the desktop with a token that expires in flight.
Expand Down Expand Up @@ -96,10 +102,27 @@ func (s *Service) builtinAIHasLocalToken() bool {
}

func builtinAITokenFresh(bundle dailysecret.ProviderBundle) bool {
if strings.EqualFold(strings.TrimSpace(bundle.SensitiveHeaders[builtinAIRefreshPendingKey]), "true") {
return false
}
expiresAt, err := time.Parse(time.RFC3339, strings.TrimSpace(bundle.SensitiveHeaders[builtinAITokenExpiresKey]))
return err == nil && time.Until(expiresAt) > builtinAIRefreshSkew
}

// holdStaleToken keeps using the access token that is still valid but records
// that a refresh is owed. Without the mark the client would wait for expiry and
// then present a refresh token the Gateway rotated long ago.
func (s *Service) holdStaleToken(bundle dailysecret.ProviderBundle) dailysecret.ProviderBundle {
if bundle.SensitiveHeaders == nil {
bundle.SensitiveHeaders = map[string]string{}
}
bundle.SensitiveHeaders[builtinAIRefreshPendingKey] = "true"
// Best effort: if the store cannot be written the next call simply waits for
// expiry as before; the in-memory bundle still carries the flag for this run.
_ = s.saveBuiltinAIToken(bundle)
return bundle
}

func builtinAITokenStillValid(bundle dailysecret.ProviderBundle) bool {
expiresAt, err := time.Parse(time.RFC3339, strings.TrimSpace(bundle.SensitiveHeaders[builtinAITokenExpiresKey]))
return err == nil && time.Until(expiresAt) > 0
Expand Down Expand Up @@ -159,10 +182,10 @@ func (s *Service) ensureBuiltinAIAccessToken(config builtinAIConfig, force bool)
if err != nil {
return dailysecret.ProviderBundle{}, err
}
response, err := doBuiltinAIJSONRequest(http.MethodPost, config.tokenURL, "", body)
response, err := doBuiltinAIRefreshRequest(config.tokenURL, body)
if err != nil {
if !force && builtinAITokenStillValid(bundle) {
return bundle, nil
return s.holdStaleToken(bundle), nil
}
return dailysecret.ProviderBundle{}, fmt.Errorf("%w: %v", errBuiltinAINetwork, err)
}
Expand All @@ -173,7 +196,7 @@ func (s *Service) ensureBuiltinAIAccessToken(config builtinAIConfig, force bool)
return dailysecret.ProviderBundle{}, errBuiltinAIUnauthorized
default:
if !force && builtinAITokenStillValid(bundle) {
return bundle, nil
return s.holdStaleToken(bundle), nil
}
return dailysecret.ProviderBundle{}, fmt.Errorf("GoNavi AI refresh endpoint returned HTTP %d", response.statusCode)
}
Expand Down
74 changes: 73 additions & 1 deletion internal/ai/service/builtin_ai_account_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,14 +35,19 @@ func putBuiltinToken(t *testing.T, s *Service, access, refresh string, expires t
func stubBuiltinGateway(t *testing.T, handler func(*http.Request) (int, string)) {
t.Helper()
previous := builtinAIHTTPDo
previousBackoff := builtinAIRefreshBackoff
builtinAIRefreshBackoff = func(int) time.Duration { return 0 }
builtinAIHTTPDo = func(req *http.Request) (*http.Response, error) {
status, body := handler(req)
if status == 0 {
return nil, errors.New("dial tcp: connection refused")
}
return &http.Response{StatusCode: status, Body: io.NopCloser(strings.NewReader(body))}, nil
}
t.Cleanup(func() { builtinAIHTTPDo = previous })
t.Cleanup(func() {
builtinAIHTTPDo = previous
builtinAIRefreshBackoff = previousBackoff
})
}

func TestEnsureBuiltinAIAccessTokenRefreshesOnceForConcurrentCallers(t *testing.T) {
Expand Down Expand Up @@ -331,3 +336,70 @@ func TestBuiltinAIContextProfileIsFixedAndPinnedForTheAgent(t *testing.T) {
t.Fatalf("the hosted model has no reasoning mode; no level may be sent: %+v", pinned)
}
}

// The Gateway rotates the refresh token as soon as it receives the request, so
// a timeout can mean the answer was lost after a successful rotation. Retrying
// with the token still held must recover instead of signing the person out.
func TestRefreshRetriesWhenTheAnswerIsLost(t *testing.T) {
s := builtinTestService(t)
putBuiltinToken(t, s, "access-old", "refresh-old", time.Now().Add(-time.Minute))
var calls atomic.Int32
stubBuiltinGateway(t, func(req *http.Request) (int, string) {
if calls.Add(1) == 1 {
return 0, ""
}
return 200, `{"access_token":"access-new","refresh_token":"refresh-new","token_type":"Bearer","expires_in":3600}`
})
config, _ := loadBuiltinAIConfig()
bundle, err := s.ensureBuiltinAIAccessToken(config, false)
if err != nil {
t.Fatalf("a lost answer must be retried, not surfaced: %v", err)
}
if calls.Load() != 2 || bundle.APIKey != "access-new" || bundle.SensitiveHeaders[builtinAIRefreshTokenKey] != "refresh-new" {
t.Fatalf("calls=%d bundle=%+v", calls.Load(), bundle)
}
if strings.EqualFold(bundle.SensitiveHeaders[builtinAIRefreshPendingKey], "true") {
t.Fatal("a successful refresh must clear the pending mark")
}
}

// A refresh that keeps failing must not leave the client waiting for the access
// token to expire: by then the Gateway's replay window is long gone. The bundle
// is marked so the very next call retries the refresh.
func TestLostRefreshMarksTheTokenForImmediateRetry(t *testing.T) {
s := builtinTestService(t)
// Not fresh (inside the 60 s skew) but still valid: the old access token
// keeps working while the refresh is retried.
putBuiltinToken(t, s, "access-old", "refresh-old", time.Now().Add(30*time.Second))
var fail atomic.Bool
fail.Store(true)
var calls atomic.Int32
stubBuiltinGateway(t, func(req *http.Request) (int, string) {
calls.Add(1)
if fail.Load() {
return 0, ""
}
return 200, `{"access_token":"access-new","refresh_token":"refresh-new","token_type":"Bearer","expires_in":3600}`
})
config, _ := loadBuiltinAIConfig()
bundle, err := s.ensureBuiltinAIAccessToken(config, false)
if err != nil {
t.Fatalf("a transient failure must keep the credentials: %v", err)
}
if bundle.APIKey != "access-old" || calls.Load() != int32(builtinAIRefreshAttempts) {
t.Fatalf("bundle=%+v calls=%d", bundle, calls.Load())
}
if !strings.EqualFold(bundle.SensitiveHeaders[builtinAIRefreshPendingKey], "true") {
t.Fatal("a failed refresh must mark the bundle as pending")
}
stored, ok, err := s.builtinAIToken()
if err != nil || !ok || !strings.EqualFold(stored.SensitiveHeaders[builtinAIRefreshPendingKey], "true") {
t.Fatalf("the pending mark was not persisted: %+v %v", stored, err)
}

fail.Store(false)
bundle, err = s.ensureBuiltinAIAccessToken(config, false)
if err != nil || bundle.APIKey != "access-new" {
t.Fatalf("the pending mark did not trigger a retry: %+v %v", bundle, err)
}
}
48 changes: 47 additions & 1 deletion internal/ai/service/builtin_ai_http.go
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,17 @@ func (e builtinAIHTTPStatusError) Error() string {
}

func doBuiltinAIJSONRequest(method, endpoint, accessToken string, body []byte) (builtinAIHTTPResponse, error) {
request, err := http.NewRequestWithContext(context.Background(), method, endpoint, bytes.NewReader(body))
return doBuiltinAIJSONRequestWithTimeout(method, endpoint, accessToken, body, 0)
}

func doBuiltinAIJSONRequestWithTimeout(method, endpoint, accessToken string, body []byte, timeout time.Duration) (builtinAIHTTPResponse, error) {
ctx := context.Background()
var cancel context.CancelFunc
if timeout > 0 {
ctx, cancel = context.WithTimeout(ctx, timeout)
defer cancel()
}
request, err := http.NewRequestWithContext(ctx, method, endpoint, bytes.NewReader(body))
if err != nil {
return builtinAIHTTPResponse{}, err
}
Expand All @@ -70,6 +80,42 @@ func doBuiltinAIJSONRequest(method, endpoint, accessToken string, body []byte) (
return builtinAIHTTPResponse{statusCode: response.StatusCode, body: responseBody}, nil
}

const (
// builtinAIRefreshAttemptTimeout bounds one refresh attempt. A refresh is a
// single database round trip and normally answers in well under a second;
// 5 s keeps the whole retry loop close to the old single 20 s attempt.
builtinAIRefreshAttemptTimeout = 5 * time.Second
// builtinAIRefreshAttempts is how many times one logical refresh is sent.
// The Gateway rotates the token as soon as it receives the request, so a
// transport error does not prove the request failed: the answer may have
// been lost after a successful rotation. Retrying with the token we still
// hold is safe because the Gateway answers it from its grace window with a
// fresh pair instead of treating the replay as theft.
builtinAIRefreshAttempts = 3
)

// builtinAIRefreshBackoff is a variable so tests can make retries immediate.
var builtinAIRefreshBackoff = func(attempt int) time.Duration {
return time.Duration(attempt) * 2 * time.Second
}

// doBuiltinAIRefreshRequest sends one refresh grant, retrying transport
// failures. It never retries an HTTP answer: a 400/401 is final.
func doBuiltinAIRefreshRequest(endpoint string, body []byte) (builtinAIHTTPResponse, error) {
var lastErr error
for attempt := 0; attempt < builtinAIRefreshAttempts; attempt++ {
if attempt > 0 {
time.Sleep(builtinAIRefreshBackoff(attempt))
}
response, err := doBuiltinAIJSONRequestWithTimeout(http.MethodPost, endpoint, "", body, builtinAIRefreshAttemptTimeout)
if err == nil {
return response, nil
}
lastErr = err
}
return builtinAIHTTPResponse{}, lastErr
}

func fetchBuiltinAIQuota(config builtinAIConfig, accessToken string) (ai.BuiltinAIQuota, error) {
response, err := doBuiltinAIJSONRequest(http.MethodGet, config.quotaURL, accessToken, nil)
if err != nil {
Expand Down
Loading